diff --git a/src/fromager/commands/list_overrides.py b/src/fromager/commands/list_overrides.py index 714ec8a85..4d3a9ff20 100644 --- a/src/fromager/commands/list_overrides.py +++ b/src/fromager/commands/list_overrides.py @@ -70,13 +70,18 @@ def list_overrides( variant_info: dict[str, str] = {} for v in variants: v_info = ps.variants.get(v) - if v_info: - if v_info.pre_built: - variant_info[str(v)] = "pre-built" - else: - variant_info[str(v)] = "yes" - else: + if v_info is None: variant_info[str(v)] = "" + continue + + is_prebuilt = v_info.pre_built + if v_info.source is not None: + # A variant `source:` block replaces the legacy settings, + # so its resolver decides whether wheels are prebuilt + # (e.g. provider: pypi-prebuilt). + is_prebuilt = v_info.source.resolves_prebuilt_wheel + + variant_info[str(v)] = "pre-built" if is_prebuilt else "yes" all_patches: PatchMap = pbi.get_all_patches() global_patches: list[pathlib.Path] = all_patches.get(None, []) @@ -86,7 +91,10 @@ def list_overrides( [v for v in all_patches.keys() if v is not None] ) - min_release_age = ps.resolver_dist.min_release_age + # Show the package-level source so the column does not change with + # --variant; variant-level sources appear in the variant columns. + source = ps.source.provider if ps.source is not None else "" + min_release_age = pbi.resolver_min_release_age min_release_age_str = ( str(min_release_age) if min_release_age is not None else "" ) @@ -97,6 +105,7 @@ def list_overrides( row_data = { "package": name, "version": "", + "source": source, "patches": patches_str, "plugin_hooks": plugin_hooks_str, "min_release_age": min_release_age_str, @@ -114,6 +123,7 @@ def list_overrides( row_data = { "package": name, "version": str(version), + "source": source, "patches": patches_str, "plugin_hooks": plugin_hooks_str, "min_release_age": min_release_age_str, @@ -149,7 +159,7 @@ def _export_csv( """Export data as CSV.""" # Define field names in the order we want them fieldnames = ( - ["package", "version", "patches", "min_release_age"] + ["package", "version", "source", "patches", "min_release_age"] + variants + ["plugin_hooks"] ) @@ -176,6 +186,7 @@ def _export_table( table = Table(title="Package Overrides") table.add_column("Package", justify="left", no_wrap=True) table.add_column("Version", justify="left", no_wrap=True) + table.add_column("Source", justify="left", no_wrap=True) table.add_column("Patches", justify="left", no_wrap=True) table.add_column("Min Release Age (days)", justify="left", no_wrap=True) @@ -186,7 +197,7 @@ def _export_table( table.add_column("Plugin", justify="left") column_keys = ( - ["package", "version", "patches", "min_release_age"] + ["package", "version", "source", "patches", "min_release_age"] + variants + ["plugin_hooks"] ) diff --git a/src/fromager/commands/package.py b/src/fromager/commands/package.py index db75ca6de..ef3c88a05 100644 --- a/src/fromager/commands/package.py +++ b/src/fromager/commands/package.py @@ -24,6 +24,7 @@ packagesettings, request_session, resolver, + sources, ) from ..candidate import Candidate, Cooldown @@ -151,8 +152,9 @@ def list_versions( - "package_name>=1.0" (versions >= 1.0) - "package_name==1.*" (versions matching 1.*) - This command uses the get_resolver_provider hook to retrieve a resolver - provider for the package in case there is a custom provider configured. + This command uses sources.get_source_provider() to retrieve a resolver + provider for the package, so both ``source:`` settings and the legacy + get_resolver_provider hook are honored. Distribution types: - "default": Use package settings for include_sdists/include_wheels @@ -160,6 +162,9 @@ def list_versions( - "wheel": Only include wheels - "both": Include both source distributions and wheels + An explicit distribution type cannot be combined with a ``source:`` + resolver config; version resolution follows the configured resolver. + Output formats: - "versions": one version per line (default) - "requirements": name==version per line (pip-installable pins) @@ -183,26 +188,32 @@ def list_versions( logger.info(f"Filtering versions with specifier: {req.specifier}") pbi = wkctx.package_build_info(req) - override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + + dist_type = DistributionType(distribution_type) + if pbi.source_resolver is not None and dist_type is not DistributionType.DEFAULT: + raise click.ClickException( + f"--distribution-type {dist_type.value} cannot be combined with the " + f"'{pbi.source_resolver.provider}' source resolver of {req.name}: " + "version resolution follows the source config" + ) include_sdists, include_wheels = parse_distribution_option( distribution_type, pbi, ) - provider = overrides.find_and_invoke( - req.name, - "get_resolver_provider", - resolver.default_resolver_provider, + provider = sources.get_source_provider( ctx=wkctx, req=req, + sdist_server_url=sdist_server_url, include_sdists=include_sdists, include_wheels=include_wheels, - sdist_server_url=override_sdist_server_url, ) # Get all available candidates from the provider (cooldown is NOT set on - # the provider so we receive every version that matches the specifier). + # the provider so we receive every version that matches the specifier; + # the detail view applies the cooldown separately). + provider.cooldown = None candidates = list( provider.find_matches( identifier=req.name, diff --git a/src/fromager/packagesettings/_pbi.py b/src/fromager/packagesettings/_pbi.py index 74d55a45b..5b3f60fb0 100644 --- a/src/fromager/packagesettings/_pbi.py +++ b/src/fromager/packagesettings/_pbi.py @@ -267,7 +267,18 @@ def resolver_min_release_age(self) -> int | None: Returns None (inherit global), 0 (disabled), or a positive integer (override days). The caller is responsible for converting to a :class:`~fromager.candidate.Cooldown` instance. + + For packages with a ``source:`` resolver, the resolver's + ``min_release_age`` takes precedence over the legacy + ``resolver_dist`` setting. """ + source_resolver = self.source_resolver + if source_resolver is not None: + source_min_release_age: int | None = getattr( + source_resolver, "min_release_age", None + ) + if source_min_release_age is not None: + return source_min_release_age return self._ps.resolver_dist.min_release_age @property diff --git a/src/fromager/sources.py b/src/fromager/sources.py index f7041415a..2f986044c 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -131,12 +131,19 @@ def get_source_provider( req: Requirement, sdist_server_url: str, req_type: RequirementType | None = None, + include_sdists: bool | None = None, + include_wheels: bool | None = None, ) -> resolver.BaseProvider: """Create a provider for resolving source distributions. Returns a provider configured according to the package's resolver settings (sdist/wheel inclusion and platform matching). + *include_sdists* and *include_wheels* override the legacy package + settings for packages without a ``source:`` resolver; they are ignored + for packages with one because the resolver defines its own inclusion + rules. + Packages with a ``source:`` resolver use that resolver's provider configuration and ignore the caller's URL, with a warning unless it is the default PyPI URL. Packages without one retain the @@ -168,6 +175,11 @@ def get_source_provider( override_sdist_server_url = pbi.resolver_sdist_server_url(sdist_server_url) + if include_sdists is None: + include_sdists = pbi.resolver_include_sdists + if include_wheels is None: + include_wheels = pbi.resolver_include_wheels + provider = typing.cast( resolver.BaseProvider, overrides.find_and_invoke( @@ -176,8 +188,8 @@ def get_source_provider( resolver.default_resolver_provider, ctx=ctx, req=req, - include_sdists=pbi.resolver_include_sdists, - include_wheels=pbi.resolver_include_wheels, + include_sdists=include_sdists, + include_wheels=include_wheels, sdist_server_url=override_sdist_server_url, req_type=req_type, ignore_platform=pbi.resolver_ignore_platform, diff --git a/tests/test_list_overrides.py b/tests/test_list_overrides.py index df7edf72d..48bd3ffa9 100644 --- a/tests/test_list_overrides.py +++ b/tests/test_list_overrides.py @@ -275,3 +275,89 @@ def test_list_overrides_min_release_age( # test-other-pkg has no min_release_age override — should be empty test_other_pkg = next(item for item in data if item["package"] == "test-other-pkg") assert test_other_pkg["min_release_age"] == "" + + +def test_list_overrides_details_source( + testdata_path: pathlib.Path, cli_runner: CliRunner +) -> None: + """--details includes the source provider for source-config packages.""" + overrides_dir = testdata_path / "context" / "overrides" + settings_file = overrides_dir / "settings.yaml" + settings_dir = overrides_dir / "settings" + patches_dir = overrides_dir / "patches" + + result = cli_runner.invoke( + fromager, + [ + "--settings-file", + str(settings_file), + "--settings-dir", + str(settings_dir), + "--patches-dir", + str(patches_dir), + "list-overrides", + "--details", + "--format", + "json", + ], + ) + assert result.exit_code == 0 + + json_output = _extract_json_from_output(result.stdout) + data = json.loads(json_output) + + # test-source-pkg has a source: block with a github-tag-git provider + test_source_pkg = next( + item for item in data if item["package"] == "test-source-pkg" + ) + assert test_source_pkg["source"] == "github-tag-git" + # min_release_age is read from the source config + assert test_source_pkg["min_release_age"] == "14" + # the rocm variant resolves a prebuilt wheel through its variant source + assert test_source_pkg["rocm"] == "pre-built" + + # legacy packages have no source config — the field is blank + test_other_pkg = next(item for item in data if item["package"] == "test-other-pkg") + assert test_other_pkg["source"] == "" + + +def test_list_overrides_details_source_variant( + testdata_path: pathlib.Path, cli_runner: CliRunner +) -> None: + """--variant does not change the package-level source column.""" + overrides_dir = testdata_path / "context" / "overrides" + settings_file = overrides_dir / "settings.yaml" + settings_dir = overrides_dir / "settings" + patches_dir = overrides_dir / "patches" + + result = cli_runner.invoke( + fromager, + [ + "--variant", + "rocm", + "--settings-file", + str(settings_file), + "--settings-dir", + str(settings_dir), + "--patches-dir", + str(patches_dir), + "list-overrides", + "--details", + "--format", + "json", + ], + ) + assert result.exit_code == 0 + + json_output = _extract_json_from_output(result.stdout) + data = json.loads(json_output) + + test_source_pkg = next( + item for item in data if item["package"] == "test-source-pkg" + ) + # the source column shows the package-level setting, not the rocm + # variant's pypi-prebuilt resolver + assert test_source_pkg["source"] == "github-tag-git" + # min_release_age comes from the variant's source resolver + assert test_source_pkg["min_release_age"] == "3" + assert test_source_pkg["rocm"] == "pre-built" diff --git a/tests/test_list_versions.py b/tests/test_list_versions.py index 09ff31040..3f665419f 100644 --- a/tests/test_list_versions.py +++ b/tests/test_list_versions.py @@ -377,18 +377,171 @@ def test_list_versions_output_file_requirements_format( # --ignore-per-package-overrides # --------------------------------------------------------------------------- +_ALTERNATE_INDEX_URL = "https://alternate.test/simple/test-pkg/" -def test_list_versions_ignore_per_package_overrides( + +def _invoke_with_source_settings( + cli_runner: CliRunner, + settings_dir: pathlib.Path, + extra_args: list[str] | None = None, + min_release_age: int = 0, +) -> typing.Any: + """Invoke ``package list-versions`` for a package with a ``source:`` resolver. + + The configured resolver uses an alternate index URL; only that URL is + mocked, so a request to pypi.org fails the test. + """ + args: list[str] = ["--settings-dir", str(settings_dir)] + if min_release_age: + args.extend(["--min-release-age", str(min_release_age)]) + args.extend(["package", "list-versions"]) + if extra_args: + args.extend(extra_args) + args.append("test-pkg") + + with requests_mock.Mocker() as m: + m.get( + _ALTERNATE_INDEX_URL, + json=_PYPI_JSON_RESPONSE, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + return cli_runner.invoke(fromager, args) + + +def _write_source_settings( + settings_dir: pathlib.Path, + extra_source_settings: str = "", +) -> None: + """Write a pypi-sdist source config using the alternate index URL.""" + settings_dir.mkdir(parents=True, exist_ok=True) + (settings_dir / "test_pkg.yaml").write_text( + "source:\n" + " provider: pypi-sdist\n" + f" index_url: {(_ALTERNATE_INDEX_URL.removesuffix('test-pkg/'))}\n" + + extra_source_settings + ) + + +def test_list_versions_source_config_provider( + cli_runner: CliRunner, + tmp_path: pathlib.Path, +) -> None: + """A ``source:`` package lists versions from its configured provider.""" + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir) + + result = _invoke_with_source_settings(cli_runner, settings_dir) + assert result.exit_code == 0 + # Versions come from the alternate index, not pypi.org. + versions = [line.strip() for line in result.stdout.split("\n") if line.strip()] + assert "1.2.2" in versions + assert "1.3.2" in versions + assert "2.0.0" in versions + + +def test_list_versions_source_rejects_distribution_type( + cli_runner: CliRunner, + tmp_path: pathlib.Path, +) -> None: + """An explicit --distribution-type is rejected for source-config packages.""" + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir) + + for dist_type in ("sdist", "wheel", "both"): + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--distribution-type", dist_type], + ) + assert result.exit_code != 0 + assert f"--distribution-type {dist_type}" in result.output + assert "source resolver" in result.output + + +def test_list_versions_source_min_release_age( cli_runner: CliRunner, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """--ignore-per-package-overrides uses global cooldown, ignoring per-package override. + """Cooldown status uses the source resolver's min_release_age.""" + _freeze_cooldown_bootstrap_time(monkeypatch) - Set up a per-package override of min_release_age=0 (exempt) and verify - that --ignore-per-package-overrides still shows cooldown status based - on the global --min-release-age. - """ + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir, " min_release_age: 3\n") + + # The source override (3 days) wins over the global value (2 days): + # 2.0.0 is 2 days old, so it is blocked. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json"], + min_release_age=2, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "blocked" + assert status_by_version["1.3.2"] == "available" + + # --ignore-per-package-overrides uses the global value (2 days): + # 2.0.0 is exactly 2 days old, so it is available. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json", "--ignore-per-package-overrides"], + min_release_age=2, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "available" + assert status_by_version["1.3.2"] == "available" + + +def test_list_versions_source_min_release_age_zero( + cli_runner: CliRunner, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A source min_release_age of 0 disables the cooldown.""" + _freeze_cooldown_bootstrap_time(monkeypatch) + + settings_dir = tmp_path / "settings" + _write_source_settings(settings_dir, " min_release_age: 0\n") + + # The source override (0) disables the cooldown entirely. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json"], + min_release_age=_COOLDOWN_7_DAYS, + ) + assert result.exit_code == 0 + for row in json.loads(_extract_json_from_output(result.stdout)): + assert row["cooldown"] == "" + + # --ignore-per-package-overrides restores the global cooldown. + result = _invoke_with_source_settings( + cli_runner, + settings_dir, + extra_args=["--format", "json", "--ignore-per-package-overrides"], + min_release_age=_COOLDOWN_7_DAYS, + ) + assert result.exit_code == 0 + status_by_version = { + row["version"]: row["cooldown"] + for row in json.loads(_extract_json_from_output(result.stdout)) + } + assert status_by_version["2.0.0"] == "blocked" + assert status_by_version["1.3.2"] == "available" + + +def _freeze_cooldown_bootstrap_time(monkeypatch: pytest.MonkeyPatch) -> None: + """Pin Cooldown.bootstrap_time to _BOOTSTRAP_TIME for deterministic ages.""" original_init = candidate.Cooldown.__init__ def patched_init( @@ -399,6 +552,20 @@ def patched_init( monkeypatch.setattr(candidate.Cooldown, "__init__", patched_init) + +def test_list_versions_ignore_per_package_overrides( + cli_runner: CliRunner, + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """--ignore-per-package-overrides uses global cooldown, ignoring per-package override. + + Set up a per-package override of min_release_age=0 (exempt) and verify + that --ignore-per-package-overrides still shows cooldown status based + on the global --min-release-age. + """ + _freeze_cooldown_bootstrap_time(monkeypatch) + # Create per-package override with min_release_age=0 (exempted) settings_dir = tmp_path / "settings" settings_dir.mkdir(parents=True) diff --git a/tests/test_packagesettings.py b/tests/test_packagesettings.py index bfd14c726..6a37a8266 100644 --- a/tests/test_packagesettings.py +++ b/tests/test_packagesettings.py @@ -37,6 +37,7 @@ TEST_RELATED_PKG = "test-pkg-library" TEST_PREBUILT_PKG = "test-prebuilt-pkg" TEST_COOLDOWN_PKG = "test-cooldown-pkg" +TEST_SOURCE_PKG = "test-source-pkg" FULL_EXPECTED: dict[str, typing.Any] = { "annotations": { @@ -552,6 +553,7 @@ def test_settings_overrides(testdata_context: context.WorkContext) -> None: TEST_RELATED_PKG, TEST_PREBUILT_PKG, TEST_COOLDOWN_PKG, + TEST_SOURCE_PKG, } @@ -603,11 +605,14 @@ def test_settings_list(testdata_context: context.WorkContext) -> None: TEST_OTHER_PKG, TEST_RELATED_PKG, TEST_PREBUILT_PKG, + TEST_SOURCE_PKG, } assert testdata_context.settings.list_pre_built() == {TEST_PREBUILT_PKG} assert testdata_context.settings.variant_changelog() == [] testdata_context.settings.variant = Variant("rocm") - assert testdata_context.settings.list_pre_built() == {TEST_PKG} + # TEST_SOURCE_PKG's rocm variant resolves a prebuilt wheel via its + # variant-level `source:` resolver. + assert testdata_context.settings.list_pre_built() == {TEST_PKG, TEST_SOURCE_PKG} assert testdata_context.settings.variant_changelog() == [ "setuptools upgraded to 82.0.0" ] diff --git a/tests/testdata/context/overrides/settings/test_source_pkg.yaml b/tests/testdata/context/overrides/settings/test_source_pkg.yaml new file mode 100644 index 000000000..b6c74abe4 --- /dev/null +++ b/tests/testdata/context/overrides/settings/test_source_pkg.yaml @@ -0,0 +1,10 @@ +source: + provider: github-tag-git + project_url: https://github.com/python-wheel-build/fromager + min_release_age: 14 +variants: + rocm: + source: + provider: pypi-prebuilt + index_url: https://wheel.test/simple + min_release_age: 3