fix(resolver): apply release-age cooldown to pre-built wheel resolution

Remove the sdist-only guard from resolver.py and sources.py so that
cooldown applies regardless of whether wheels or sdists are being
resolved. Also set provider.cooldown in resolve_all_prebuilt_wheels()
since that path bypasses resolver.resolve() entirely.

Adds an E2E test (test_bootstrap_cooldown_prebuilt.sh) that configures
stevedore as pre_built and verifies cooldown blocks the 5.4.0 wheel in
favour of 5.3.0.

Co-Authored-By: Claude <claude@anthropic.com>

Author: ryanpetrello

docs/how-tos/release-age-cooldown.rst

@@ -41,43 +41,50 @@ A value of ``0`` (the default) disables the check entirely.
 Scope
 -----
 
-The cooldown applies to **sdist resolution** — selecting which version of a
-package to build from source, including transitive dependencies. It does not
-apply to:
+The cooldown applies to both **sdist resolution** and **pre-built wheel
+resolution** — any candidate whose ``upload-time`` is more recent than the
+cutoff is rejected, regardless of whether it is an sdist or a prebuilt wheel.
 
-* Wheel-only lookups, including cache servers (``--cache-wheel-server-url``) and
-  packages configured as ``pre_built: true`` in variant settings. These use a
-  different trust model and are not subject to the cooldown regardless of which
-  server they are fetched from.
-* Packages resolved from Git URLs that do not provide timestamp metadata.
+The following are **not** subject to the cooldown:
 
-Note that sdist resolution from a private package index depends on
+* Fromager's internal build and cache wheel servers. These are not used for
+  version selection — they are checked only for already-resolved pinned
+  versions — so the cooldown has no insertion point.
+* Packages resolved from Git URLs. Git timestamps are set by the client, not
+  the server, and cannot be trusted for cooldown enforcement.
+
+Resolution from a private package index (sdist or wheel) depends on
 ``upload-time`` being present in the index's PEP 691 JSON responses. If the
-index does not provide that metadata, candidates will be rejected under the
-fail-closed policy described below.
+index does not provide that metadata, candidates are rejected under the
+fail-closed policy described below. Use ``resolver_dist.min_release_age: 0``
+to bypass cooldown for packages from indexes that structurally cannot supply
+timestamps.
 
 
 Fail-Closed Behavior
 --------------------
 
-If a candidate has no ``upload-time`` metadata — which can occur with older
-PyPI Simple HTML responses — it is rejected when a cooldown is active. Fromager
-uses the `PEP 691 JSON Simple API`_ when fetching package metadata, which
-reliably includes upload timestamps.
+If a candidate has no ``upload-time`` metadata — whether it is an sdist or a
+wheel — it is rejected when a cooldown is active. Fromager uses the
+`PEP 691 JSON Simple API`_ when fetching package metadata, which reliably
+includes upload timestamps for PyPI.org.
 
 .. _PEP 691 JSON Simple API: https://peps.python.org/pep-0691/
 
+For indexes that only implement the `PEP 503`_ HTML API and cannot supply
+timestamps, use the per-package ``resolver_dist.min_release_age: 0`` override
+to bypass the cooldown for affected packages rather than disabling it globally.
+
+.. _PEP 503: https://peps.python.org/pep-0503/
+
 .. note::
 
    If you are writing a ``get_resolver_provider`` plugin that uses
    :class:`~fromager.resolver.PyPIProvider` with a private index that only
-   implements the `PEP 503`_ HTML API (and therefore cannot supply upload
-   timestamps), pass ``supports_upload_time=False`` to ``PyPIProvider``. This
-   switches the provider from fail-closed to warn-and-skip, so candidates
-   without upload timestamps are skipped with a warning rather than causing
-   resolution to fail.
-
-.. _PEP 503: https://peps.python.org/pep-0503/
+   implements the PEP 503 HTML API, pass ``supports_upload_time=False`` to
+   ``PyPIProvider``. This switches the provider from fail-closed to
+   warn-and-skip, so candidates without upload timestamps are skipped with a
+   warning rather than causing resolution to fail.
 
 Example
 -------

e2e/ci_bootstrap_suite.sh

@@ -39,6 +39,7 @@ run_test "bootstrap_cooldown_transitive"
 run_test "bootstrap_cooldown_gitlab"
 run_test "bootstrap_cooldown_github"
 run_test "bootstrap_cooldown_override"
+run_test "bootstrap_cooldown_prebuilt"
 
 test_section "bootstrap git URL tests"
 run_test "bootstrap_git_url"

e2e/cooldown_prebuilt_settings/stevedore.yaml

@@ -0,0 +1,3 @@
+variants:
+  cpu:
+    pre_built: true

e2e/test_bootstrap_cooldown_github.sh

@@ -43,7 +43,7 @@ find "$OUTDIR/wheels-repo/" -name '*.whl'
 
 pass=true
 
-# Resolution must succeed despite the 9999-day cooldown — GitHub timestamps
+# Resolution must succeed despite the large cooldown — GitHub timestamps
 # are not yet supported, so the cooldown is skipped rather than enforced.
 if ! find "$OUTDIR/wheels-repo/downloads/" -name 'stevedore-*.whl' | grep -q .; then
   echo "FAIL: no stevedore wheel found — resolution should have succeeded despite cooldown" 1>&2

e2e/test_bootstrap_cooldown_prebuilt.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Tests that --min-release-age rejects pre-built wheel candidates published
+# within the cooldown window and falls back to an older stevedore version.
+# This exercises the resolve_all_prebuilt_wheels() code path in wheels.py,
+# which is taken when a package is configured with pre_built: true.
+#
+# Release timeline (wheel upload times, UTC):
+#
+#   stevedore 5.3.0   2024-08-22  (the expected fallback)
+#   stevedore 5.4.0   2024-11-20  (blocked by cooldown)
+#   stevedore 5.5.0+  future      (all blocked by cooldown)
+#
+# We use the same MIN_AGE anchor as test_bootstrap_cooldown.sh: the age of
+# stevedore 5.4.0 (released 2024-11-20) plus a 1-day buffer, ensuring 5.4.0
+# is always just inside the cooldown window while 5.3.0 (released ~90 days
+# earlier) always clears it.
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+MIN_AGE=$(python3 -c "
+from datetime import date
+age = (date.today() - date(2024, 11, 20)).days
+print(age + 1)
+")
+
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$SCRIPTDIR/cooldown_prebuilt_settings" \
+  --min-release-age="$MIN_AGE" \
+  bootstrap 'stevedore'
+
+pass=true
+
+# stevedore 5.4.0's wheel is blocked; the resolver must fall back to 5.3.0.
+if ! grep -q "new toplevel dependency stevedore resolves to 5.3.0" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: expected stevedore to resolve to 5.3.0 but it did not" 1>&2
+  pass=false
+fi
+
+# The wheel must have been downloaded as a pre-built (not built from source).
+if ! grep -q "uses a pre-built wheel" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: stevedore was not downloaded as a pre-built wheel" 1>&2
+  pass=false
+fi
+
+# Wheel file must exist in the prebuilt directory.
+if ! find "$OUTDIR/wheels-repo/prebuilt/" -name 'stevedore-5.3.0*.whl' | grep -q .; then
+  echo "FAIL: stevedore-5.3.0 wheel not found in wheels-repo/prebuilt" 1>&2
+  pass=false
+fi
+
+# No stevedore sdist should have been downloaded — it is pre_built only.
+if find "$OUTDIR/sdists-repo/" \( -name 'stevedore*.tar.gz' -o -name 'stevedore*.zip' \) 2>/dev/null | grep -q .; then
+  echo "FAIL: stevedore sdist found in sdists-repo — should be pre-built only" 1>&2
+  pass=false
+fi
+
+$pass

src/fromager/packagesettings/_models.py

@@ -178,7 +178,7 @@ class ResolverDist(pydantic.BaseModel):
     0: disable the release-age cooldown for this package.
     Positive integer: override the cooldown with this many days.
 
-    .. versionadded:: 0.81
+    .. versionadded:: 0.82
     """
 
     @pydantic.model_validator(mode="after")

src/fromager/resolver.py

@@ -104,14 +104,7 @@ def resolve(
         req_type=req_type,
         ignore_platform=ignore_platform,
     )
-    # Apply cooldown settings to sdist resolution only. Wheel-only lookups
-    # (cache servers, pre_built packages) use a different trust model.
-    pbi = ctx.package_build_info(req)
-    provider.cooldown = (
-        resolve_package_cooldown(ctx.cooldown, pbi.resolver_min_release_age)
-        if include_sdists
-        else None
-    )
+    provider.cooldown = resolve_package_cooldown(ctx, req)
     results = find_all_matching_from_provider(provider, req)
     return results[0]
 
@@ -143,19 +136,20 @@ def default_resolver_provider(
 
 
 def resolve_package_cooldown(
-    global_cooldown: Cooldown | None,
-    per_package_days: int | None,
+    ctx: context.WorkContext,
+    req: Requirement,
 ) -> Cooldown | None:
     """Compute the effective cooldown for a single package.
 
     Args:
-        global_cooldown: The run-wide cooldown from ``--min-release-age``.
-        per_package_days: The ``resolver_dist.min_release_age`` setting for
-            this package (None = inherit, 0 = disabled, positive = override).
+        ctx: The current work context (provides the global cooldown).
+        req: The package requirement being resolved.
 
     Returns:
         The cooldown to pass to the provider, or ``None`` if disabled.
     """
+    per_package_days = ctx.package_build_info(req).resolver_min_release_age
+    global_cooldown = ctx.cooldown
     if per_package_days is None:
         return global_cooldown
     if per_package_days == 0:
@@ -470,6 +464,7 @@ def get_project_from_pypi(
 class BaseProvider(ExtrasProvider):
     resolver_cache: typing.ClassVar[ResolverCache] = {}
     provider_description: typing.ClassVar[str]
+    _cooldown_unsupported_warned: typing.ClassVar[set[str]] = set()
 
     def __init__(
         self,
@@ -486,16 +481,11 @@ def __init__(
 
         # cooldown specific settings
         self.cooldown = cooldown
-        self.cooldown_unsupported_previously_warned = False
-        """
-        Does this provider supply upload timestamps for candidates?
-
-        Defaults to False (safe/unknown). Subclasses that reliably populate
-        upload_time on every candidate should set this to True in their __init__.
-
-        When a cooldown is active and this is False, the cooldown check is
-        skipped with a warning rather than failing closed.
-        """
+        # Does this provider supply upload timestamps for candidates?
+        # Defaults to False (safe/unknown). Subclasses that reliably populate
+        # upload_time on every candidate should set this to True in their __init__.
+        # When a cooldown is active and this is False, the cooldown check is
+        # skipped with a warning rather than failing closed.
         self.supports_upload_time: bool = False
 
     @property
@@ -607,47 +597,62 @@ def is_satisfied_by(self, requirement: Requirement, candidate: Candidate) -> boo
                 )
             return False
 
-        # Release-age cooldown: reject candidates published too recently.
-        if self.cooldown is not None:
-            if candidate.upload_time is None:
-                if not self.supports_upload_time:
-                    # Provider does not yet support timestamp retrieval (e.g. GitHub).
-                    # Warn once per provider instance (i.e. per package) rather than
-                    # once per candidate.
-                    if not self.cooldown_unsupported_previously_warned:
-                        self.cooldown_unsupported_previously_warned = True
-                        logger.warning(
-                            "%s: release-age cooldown cannot be enforced — upload "
-                            "timestamp support is not yet implemented for %s; "
-                            "cooldown check skipped",
-                            requirement.name,
-                            self.get_provider_description(),
-                        )
-                    return True
-                # Provider should supply timestamps but this candidate is missing one.
-                # Fail closed: we cannot verify the age, so reject it.
-                if DEBUG_RESOLVER:
-                    logger.debug(
-                        "%s: skipping %s — upload_time unknown, required for cooldown",
-                        requirement.name,
-                        candidate.version,
-                    )
-                return False
-            cutoff = self.cooldown.bootstrap_time - self.cooldown.min_age
-            if candidate.upload_time > cutoff:
-                if DEBUG_RESOLVER:
-                    age = self.cooldown.bootstrap_time - candidate.upload_time
-                    logger.debug(
-                        "%s: skipping %s uploaded %s ago (cooldown: %s)",
-                        requirement.name,
-                        candidate.version,
-                        age,
-                        self.cooldown.min_age,
-                    )
-                return False
+        if self.is_blocked_by_cooldown(candidate):
+            return False
 
         return True
 
+    def is_blocked_by_cooldown(self, candidate: Candidate) -> bool:
+        """Return True if the candidate is rejected by the release-age cooldown."""
+
+        # a cooldown is not specified...
+        if self.cooldown is None:
+            return False
+
+        # the target candidate doesn't provide a valid upload timestamp
+        if candidate.upload_time is None:
+            if not self.supports_upload_time:
+                # this provider does not yet support timestamp retrieval (e.g. GitHub).
+                # Warn once per package name across all provider instances.
+                if candidate.name not in BaseProvider._cooldown_unsupported_warned:
+                    BaseProvider._cooldown_unsupported_warned.add(candidate.name)
+                    logger.warning(
+                        "%s: release-age cooldown cannot be enforced — upload "
+                        "timestamp support is not yet implemented for %s; "
+                        "cooldown check skipped",
+                        candidate.name,
+                        self.get_provider_description(),
+                    )
+                return False
+            # this provider is expected to supply timestamps,
+            # but this candidate is missing one.
+            # Fail closed: we cannot verify the age of this candidate, so reject it.
+            if DEBUG_RESOLVER:
+                logger.debug(
+                    "%s: skipping %s — upload_time unknown, required for cooldown",
+                    candidate.name,
+                    candidate.version,
+                )
+            return True
+
+        # cooldowns are enabled, and this candidate has a valid upload timestamp
+        # so we can do the math to determine whether or not the candidate should
+        # be blocked/skipped
+        cutoff = self.cooldown.bootstrap_time - self.cooldown.min_age
+        if candidate.upload_time > cutoff:
+            # if this candidate is "too new", block/skip it
+            if DEBUG_RESOLVER:
+                age = self.cooldown.bootstrap_time - candidate.upload_time
+                logger.debug(
+                    "%s: skipping %s uploaded %s ago (cooldown: %s)",
+                    candidate.name,
+                    candidate.version,
+                    age,
+                    self.cooldown.min_age,
+                )
+            return True
+        return False
+
     def get_dependencies(self, candidate: Candidate) -> list[Requirement]:
         # return candidate.dependencies
         return []
@@ -751,7 +756,7 @@ def __init__(
         use_resolver_cache: bool = True,
         override_download_url: str | None = None,
         cooldown: Cooldown | None = None,
-        supports_upload_time: bool = True,
+        supports_upload_time: bool | None = None,
     ):
         super().__init__(
             constraints=constraints,
@@ -760,13 +765,10 @@ def __init__(
             cooldown=cooldown,
         )
 
-        # not all PyPI indexes reliably support
-        # https://peps.python.org/pep-0691/, which is necessary
-        # to reliably supply upload times for packages
-        #
-        # consumers of this provider which specify a custom sdist_server_url
-        # that only provides support for https://peps.python.org/pep-0503/
-        # Simple Repository API should specify support_upload_time=False
+        # Only pypi.org reliably supports PEP 691 upload timestamps.
+        # Default to True for pypi.org, False for all other indexes.
+        if supports_upload_time is None:
+            supports_upload_time = sdist_server_url.startswith(PYPI_SERVER_URL)
         self.supports_upload_time = supports_upload_time
         self.include_sdists = include_sdists
         self.include_wheels = include_wheels

src/fromager/sources.py

@@ -153,11 +153,7 @@ def get_source_provider(
             ignore_platform=pbi.resolver_ignore_platform,
         ),
     )
-    provider.cooldown = (
-        resolver.resolve_package_cooldown(ctx.cooldown, pbi.resolver_min_release_age)
-        if pbi.resolver_include_sdists
-        else None
-    )
+    provider.cooldown = resolver.resolve_package_cooldown(ctx, req)
     return provider
 
 

src/fromager/wheels.py

@@ -549,6 +549,12 @@ def resolve_all_prebuilt_wheels(
             provider = get_prebuilt_wheel_provider(
                 ctx=ctx, req=req, wheel_server_url=url, req_type=req_type
             )
+            provider.cooldown = resolver.resolve_package_cooldown(ctx, req)
+            # The local fromager wheel server is PEP 503-only and serves
+            # packages that were already resolved and vetted earlier in the
+            # same run. Don't fail-closed on missing upload_time there.
+            if ctx.wheel_server_url and url == ctx.wheel_server_url:
+                provider.supports_upload_time = False
 
             # Get all matching candidates from provider
             results = resolver.find_all_matching_from_provider(provider, req)