diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index 6b4252cd..93efafa6 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -137,6 +137,8 @@ def bootstrap( and optional version constraints. """ + resolver.cooldown_report.clear() + logger.info(f"cache wheel server url: {cache_wheel_server_url}") to_build = _get_requirements_from_args(toplevel, requirements_files) @@ -239,6 +241,8 @@ def bootstrap( f"Could not produce a pip compatible constraints file. Please review {constraints_filename} for more details" ) + resolver.cooldown_report.write_to(wkctx.work_dir / "cooldown-skipped-versions.json") + logger.debug("match_py_req LRU cache: %r", resolver.match_py_req.cache_info()) metrics.summarize(wkctx, "Bootstrapping") diff --git a/src/fromager/commands/build.py b/src/fromager/commands/build.py index 204f01ac..cd3effd2 100644 --- a/src/fromager/commands/build.py +++ b/src/fromager/commands/build.py @@ -101,6 +101,7 @@ def build( separately. """ + resolver.cooldown_report.clear() wkctx.wheel_server_url = wheel_server_url server.start_wheel_server(wkctx) req = Requirement(f"{dist_name}=={dist_version}") @@ -122,6 +123,7 @@ def build( force=True, cache_wheel_server_url=None, ) + resolver.cooldown_report.write_to(wkctx.work_dir / "cooldown-skipped-versions.json") print(entry.wheel_filename) diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index e1b78a8b..8d35707f 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -5,11 +5,15 @@ # from __future__ import annotations +import dataclasses import datetime import functools +import json import logging import os +import pathlib import re +import threading import typing from collections.abc import Iterable from operator import attrgetter @@ -59,6 +63,82 @@ ) +@dataclasses.dataclass(frozen=True, slots=True) +class CooldownBlockedEntry: + """Record of a single package version blocked by the cooldown policy.""" + + package: str + version: str + upload_time: str | None + cooldown_min_age_days: int + provider: str + + +class CooldownReport: + """Accumulate cooldown-blocked versions across a resolution run.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._entries: list[CooldownBlockedEntry] = [] + self._seen: set[tuple[str, str]] = set() + + def record(self, entry: CooldownBlockedEntry) -> None: + """Record a blocked entry, deduplicating by (package, version).""" + key = (entry.package, entry.version) + with self._lock: + if key not in self._seen: + self._seen.add(key) + self._entries.append(entry) + + def entries(self) -> list[CooldownBlockedEntry]: + """Return a copy of all recorded blocked entries.""" + with self._lock: + return list(self._entries) + + def clear(self) -> None: + """Remove all recorded entries and reset the deduplication set.""" + with self._lock: + self._entries.clear() + self._seen.clear() + + def write_to(self, path: pathlib.Path) -> None: + """Write JSON report to *path*. + + The file is always created so downstream tooling can rely on its + presence. When no versions were blocked the ``packages`` dict is + empty and ``total_blocked`` is 0. + """ + entries = self.entries() + packages: dict[str, list[dict[str, typing.Any]]] = {} + for e in entries: + packages.setdefault(e.package, []).append( + { + "version": e.version, + "upload_time": e.upload_time, + "cooldown_min_age_days": e.cooldown_min_age_days, + "provider": e.provider, + } + ) + report = { + "generated_at": datetime.datetime.now(datetime.UTC).isoformat(), + "total_blocked": len(entries), + "packages": packages, + } + with open(path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + if entries: + logger.info( + "cooldown skipped %d version(s) across %d package(s); " + "report written to %s", + len(entries), + len(packages), + path, + ) + + +cooldown_report = CooldownReport() + + @functools.lru_cache(maxsize=200) def match_py_req(py_req: str, *, python_version: Version = PYTHON_VERSION) -> bool: """Python version requirement lookup with LRU cache @@ -824,10 +904,29 @@ def find_matches( candidates.remove(b) versions = ", ".join(str(b.version) for b in blocked) logger.info( - "cooldown blocked %d version(s): %s", + "%s: cooldown blocked %d version(s): %s", + identifier, len(blocked), versions, ) + for b in blocked: + cooldown_report.record( + CooldownBlockedEntry( + package=b.name, + version=str(b.version), + upload_time=( + b.upload_time.isoformat() + if b.upload_time is not None + else None + ), + cooldown_min_age_days=( + self.cooldown.min_age.days + if self.cooldown is not None + else 0 + ), + provider=self.get_provider_description(), + ) + ) if not candidates: raise resolvelib.resolvers.ResolverException( self._get_no_match_error_message(identifier, requirements) diff --git a/tests/test_cooldown.py b/tests/test_cooldown.py index 0d47cc5f..e6fd8465 100644 --- a/tests/test_cooldown.py +++ b/tests/test_cooldown.py @@ -6,6 +6,7 @@ """ import datetime +import json import logging import pathlib import re @@ -84,7 +85,7 @@ @pytest.fixture(autouse=True) def clear_resolver_cache() -> typing.Generator[None, None, None]: - """Clear the class-level resolver cache before each test. + """Clear the class-level resolver cache and cooldown report before each test. BaseProvider.resolver_cache is a ClassVar that persists across test instances. Without clearing it, candidates fetched in one test are reused @@ -93,7 +94,9 @@ def clear_resolver_cache() -> typing.Generator[None, None, None]: """ resolver.BaseProvider.clear_cache() resolver.BaseProvider._cooldown_unsupported_warned.clear() + resolver.cooldown_report.clear() yield + resolver.cooldown_report.clear() def test_cooldown_filters_recent_version( @@ -965,3 +968,121 @@ def test_resolve_package_cooldown_toplevel_compound_specifier_not_exempt( ctx, Requirement("test-pkg==1.0,>0.9"), req_type=RequirementType.TOP_LEVEL ) assert result is _COOLDOWN + + +# --------------------------------------------------------------------------- +# CooldownReport tests +# --------------------------------------------------------------------------- + + +def test_cooldown_report_records_blocked_version() -> None: + """Blocked versions are recorded in the module-level cooldown report.""" + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, cooldown=_COOLDOWN) + rslvr = resolvelib.Resolver(provider, resolvelib.BaseReporter()) + rslvr.resolve([Requirement("test-pkg")]) + + entries = resolver.cooldown_report.entries() + assert len(entries) == 1 + entry = entries[0] + assert entry.package == "test-pkg" + assert entry.version == "2.0.0" + assert entry.upload_time == "2026-03-24T00:00:00+00:00" + assert entry.cooldown_min_age_days == 7 + + +def test_cooldown_report_empty_when_disabled() -> None: + """No entries recorded when cooldown is not configured.""" + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, cooldown=None) + rslvr = resolvelib.Resolver(provider, resolvelib.BaseReporter()) + rslvr.resolve([Requirement("test-pkg")]) + + assert resolver.cooldown_report.entries() == [] + + +def test_cooldown_report_write_to_json(tmp_path: pathlib.Path) -> None: + """write_to() produces valid JSON with the expected structure.""" + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, cooldown=_COOLDOWN) + rslvr = resolvelib.Resolver(provider, resolvelib.BaseReporter()) + rslvr.resolve([Requirement("test-pkg")]) + + output = tmp_path / "cooldown-skipped-versions.json" + resolver.cooldown_report.write_to(output) + + assert output.exists() + report = json.loads(output.read_text()) + assert report["total_blocked"] == 1 + assert "test-pkg" in report["packages"] + assert report["packages"]["test-pkg"][0]["version"] == "2.0.0" + assert "generated_at" in report + + +def test_cooldown_report_empty_file_when_no_blocked(tmp_path: pathlib.Path) -> None: + """write_to() creates a file with zero blocked entries when report is empty.""" + output = tmp_path / "cooldown-skipped-versions.json" + resolver.cooldown_report.write_to(output) + assert output.exists() + report = json.loads(output.read_text()) + assert report["total_blocked"] == 0 + assert report["packages"] == {} + + +def test_cooldown_report_deduplicates() -> None: + """Repeated find_matches calls for the same package record only once.""" + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider( + include_sdists=True, cooldown=_COOLDOWN, use_resolver_cache=False + ) + req = Requirement("test-pkg") + # Simulate resolvelib backtracking — call find_matches twice + provider.find_matches( + identifier="test-pkg", + requirements={"test-pkg": [req]}, + incompatibilities={"test-pkg": []}, + ) + provider.find_matches( + identifier="test-pkg", + requirements={"test-pkg": [req]}, + incompatibilities={"test-pkg": []}, + ) + + entries = resolver.cooldown_report.entries() + assert len(entries) == 1 + + +def test_cooldown_report_clear() -> None: + """clear() removes all recorded entries.""" + resolver.cooldown_report.record( + resolver.CooldownBlockedEntry( + package="pkg", + version="1.0.0", + upload_time=None, + cooldown_min_age_days=7, + provider="test", + ) + ) + assert len(resolver.cooldown_report.entries()) == 1 + resolver.cooldown_report.clear() + assert resolver.cooldown_report.entries() == []