From 0eeb77b22b4def8f68c9a7dfaa6e09b4987c46e0 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Mon, 31 Aug 2026 15:14:32 +0200 Subject: [PATCH 1/7] feat: Probe native wheels on all platforms via YAML policy Run native_import_guard.yaml probes on every test-wheels-install runner so SIGABRT and ABI mismatches fail CI before upload --- README.md | 31 ++-- _helper_functions.py | 41 ----- native_import_guard.py | 243 +++++++++++++++++++++++++++++ native_import_guard.yaml | 48 ++++-- scripts/run_native_import_probe.py | 24 +-- test_build_wheels.py | 28 ++++ test_native_import_guard.py | 156 +++++++++++++++++- test_wheels_install.py | 25 ++- 8 files changed, 506 insertions(+), 90 deletions(-) create mode 100644 native_import_guard.py diff --git a/README.md b/README.md index aa46d39..a4baad8 100644 --- a/README.md +++ b/README.md @@ -143,30 +143,35 @@ The syntax can be also converted into a sentence: "For assembled **main requirem ### native_import_guard.yaml -File for **native ARMv7 import checks** after wheels are installed in CI ([`test_wheels_install.py`](./test_wheels_install.py)). It does not change the requirements list. +File for **native import checks** after wheels are installed in CI ([`test_wheels_install.py`](./test_wheels_install.py), all test-wheels-install runners). It does not change the requirements list. -For every `name` there is: +The syntax matches [`exclude_list.yaml`](./exclude_list.yaml) / [`include_list.yaml`](./include_list.yaml): “After installing the wheel for `package_name` (on `platform`, for `python`, at `version`), run `imports` — or `skip` — and fail CI if import crashes or errors.” -* `imports` — Python statement(s) to run after install (list of strings; multiline allowed) +Top-level options: -native_import_guard template: +* `probe_unlisted` — if true (default), every other **platform** wheel is probed with `import ` from the wheel’s `top_level.txt` +* `skip_pure_any` — if true (default), skip `*-none-any.whl` +* `skip_top_level` — names ignored when generating that default import (`test`, `tests`, `testing`) - packages: - - name: '' - imports: - - import _cffi_backend +Each `packages:` entry: -The syntax can be converted into a sentence: "After installing the wheel for `name` on ARMv7, run `imports` and fail CI if import crashes or errors." +* `package_name` — PyPI distribution name (legacy key `name` is accepted) +* `imports` — Python statement(s) to run after install (list; multiline allowed) +* `skip` — if true, do not probe when the filters match +* `platform` / `python` / `version` — optional, same tokens as `exclude_list.yaml` (omitted = all) example: ```yaml - - name: 'cffi' - imports: - - import _cffi_backend +probe_unlisted: true +skip_pure_any: true +packages: + - package_name: cffi + imports: + - import _cffi_backend ``` -This would mean: load the real C extension for `cffi` on ARMv7 (not `import cffi` alone). The same names are referenced in [`repair_wheels.py`](./repair_wheels.py) for ARMv7 manylinux repair policy. +This would mean: load the real C extension for `cffi` (not `import cffi` alone). The same names are referenced in [`repair_wheels.py`](./repair_wheels.py) for ARMv7 manylinux repair policy. ### build_requirements.txt diff --git a/_helper_functions.py b/_helper_functions.py index a83a84a..261ecd4 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -14,7 +14,6 @@ import sys import zipfile -from dataclasses import dataclass from pathlib import Path from typing import Any from typing import Dict @@ -30,8 +29,6 @@ from urllib.request import Request from urllib.request import urlopen -import yaml - from colorama import Fore from colorama import Style from packaging.requirements import InvalidRequirement @@ -53,7 +50,6 @@ FORCE_NO_BINARY_LINUX_FILE = "force_no_binary_linux.txt" EXCLUDE_LIST_PATH = "exclude_list.yaml" -NATIVE_IMPORT_GUARD_PATH = "native_import_guard.yaml" PYPI_SIMPLE_INDEX = "https://pypi.org/simple/" DEFAULT_WHEEL_DIR = "downloaded_wheels" @@ -401,43 +397,6 @@ def load_force_no_binary_linux_names(repo_root: Path | None = None) -> list[str] return out -@dataclass(frozen=True) -class NativeImportGuardEntry: - """Import probes for native ARMv7 wheels (``test_wheels_install.py``).""" - - imports: tuple[str, ...] - - -_NATIVE_IMPORT_GUARD_CACHE: dict[Path, dict[str, NativeImportGuardEntry]] = {} - - -def native_import_guard_by_name( - repo_root: Path | None = None, -) -> dict[str, NativeImportGuardEntry]: - """Map canonical distribution name → guard config (``native_import_guard.yaml``).""" - root = (repo_root if repo_root is not None else _REPO_ROOT).resolve() - cached = _NATIVE_IMPORT_GUARD_CACHE.get(root) - if cached is not None: - return cached - path = root / NATIVE_IMPORT_GUARD_PATH - data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - by_name: dict[str, NativeImportGuardEntry] = {} - for entry in data.get("packages") or []: - if not isinstance(entry, dict): - continue - name, imports = entry.get("name"), entry.get("imports") - if not name or not imports: - continue - stmts = tuple(str(s).strip() for s in imports if str(s).strip()) - if not stmts: - continue - by_name[canonicalize_name(str(name))] = NativeImportGuardEntry(imports=stmts) - if not by_name: - raise ValueError(f"{path}: no packages with imports defined") - _NATIVE_IMPORT_GUARD_CACHE[root] = by_name - return by_name - - _MANYLINUX_GLIBC_TAG = re.compile(r"manylinux_(\d+)_(\d+)", re.IGNORECASE) diff --git a/native_import_guard.py b/native_import_guard.py new file mode 100644 index 0000000..170c68a --- /dev/null +++ b/native_import_guard.py @@ -0,0 +1,243 @@ +# +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Load ``native_import_guard.yaml`` and resolve import-probe statements for a wheel.""" + +from __future__ import annotations + +import sys +import zipfile + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion +from packaging.version import Version + +from _helper_functions import exclude_entry_applies_to_platform +from _helper_functions import get_current_platform +from _helper_functions import parse_wheel_name + +_REPO_ROOT = Path(__file__).resolve().parent +NATIVE_IMPORT_GUARD_PATH = "native_import_guard.yaml" + +DEFAULT_SKIP_TOP_LEVEL_IMPORTS = frozenset({"test", "tests", "testing"}) + + +def _yaml_scalar_or_list(raw: Any) -> list[str]: + if raw is None or raw is False or raw == "": + return [] + if isinstance(raw, list): + return [str(item).strip() for item in raw if str(item).strip()] + return [str(raw).strip()] if str(raw).strip() else [] + + +def _version_matches_yaml_spec(raw: Any, version: Version) -> bool: + """Match ``version`` / ``python`` YAML filters (string or list of PEP 440 specifiers). + + A list of only ``==`` / ``===`` specs is OR (e.g. ``['==3.8', '==3.9']``). + Otherwise specs are AND (e.g. ``['>=1.0', '<2']``). + """ + items = _yaml_scalar_or_list(raw) + if not items: + return True + specs = [SpecifierSet(item) for item in items] + operators = [spec.operator for spec_set in specs for spec in spec_set] + if operators and all(op in ("==", "===") for op in operators): + return any(version in spec_set for spec_set in specs) + return version in SpecifierSet(",".join(items)) + + +@dataclass(frozen=True) +class NativeImportGuardEntry: + """One ``packages:`` row from ``native_import_guard.yaml``.""" + + name: str + imports: tuple[str, ...] + skip: bool + raw: dict[str, Any] + + +@dataclass(frozen=True) +class NativeImportGuardConfig: + """Loaded ``native_import_guard.yaml`` (policy + per-package rows).""" + + probe_unlisted: bool + skip_pure_any: bool + skip_top_level: frozenset[str] + packages: tuple[NativeImportGuardEntry, ...] + + +_NATIVE_IMPORT_GUARD_CACHE: dict[Path, NativeImportGuardConfig] = {} + + +def _native_import_entry_matches( + entry: NativeImportGuardEntry, + *, + current_platform: str, + wheel_version: str | None, + python_version: str, +) -> bool: + if not exclude_entry_applies_to_platform(entry.raw, current_platform): + return False + raw_version = entry.raw.get("version") + if raw_version: + if not wheel_version: + return False + try: + if not _version_matches_yaml_spec(raw_version, Version(wheel_version)): + return False + except InvalidVersion: + return False + raw_python = entry.raw.get("python") + if raw_python: + try: + if not _version_matches_yaml_spec(raw_python, Version(python_version)): + return False + except InvalidVersion: + return False + return True + + +def load_native_import_guard(repo_root: Path | None = None) -> NativeImportGuardConfig: + """Load ``native_import_guard.yaml`` (cached per repo root).""" + root = (repo_root if repo_root is not None else _REPO_ROOT).resolve() + cached = _NATIVE_IMPORT_GUARD_CACHE.get(root) + if cached is not None: + return cached + path = root / NATIVE_IMPORT_GUARD_PATH + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + skip_top = _yaml_scalar_or_list(data.get("skip_top_level")) + packages: list[NativeImportGuardEntry] = [] + for entry in data.get("packages") or []: + if not isinstance(entry, dict): + continue + name = entry.get("package_name") or entry.get("name") + if not name: + continue + raw_imports = entry.get("imports") or [] + stmts = tuple(str(s).strip() for s in raw_imports if str(s).strip()) + skip = bool(entry.get("skip")) + if not stmts and not skip: + continue + packages.append( + NativeImportGuardEntry( + name=canonicalize_name(str(name)), + imports=stmts, + skip=skip, + raw=entry, + ) + ) + if not packages: + raise ValueError(f"{path}: no packages with imports or skip defined") + config = NativeImportGuardConfig( + probe_unlisted=bool(data.get("probe_unlisted", True)), + skip_pure_any=bool(data.get("skip_pure_any", True)), + skip_top_level=frozenset(skip_top) if skip_top else DEFAULT_SKIP_TOP_LEVEL_IMPORTS, + packages=tuple(packages), + ) + _NATIVE_IMPORT_GUARD_CACHE[root] = config + return config + + +def native_import_guard_by_name( + repo_root: Path | None = None, +) -> dict[str, NativeImportGuardEntry]: + """Map canonical name → first YAML row that defines ``imports`` (custom probes).""" + by_name: dict[str, NativeImportGuardEntry] = {} + for entry in load_native_import_guard(repo_root).packages: + if entry.imports and entry.name not in by_name: + by_name[entry.name] = entry + return by_name + + +def is_pure_any_wheel_name(wheel_name: str) -> bool: + """True for ``*-none-any.whl`` (no native extension to probe).""" + return "-none-any.whl" in wheel_name.lower() + + +def wheel_top_level_modules( + wheel_path: Path | str, + skip_top_level: frozenset[str] | None = None, +) -> tuple[str, ...]: + """Read ``*.dist-info/top_level.txt`` from a wheel (import names).""" + skip = skip_top_level if skip_top_level is not None else DEFAULT_SKIP_TOP_LEVEL_IMPORTS + path = Path(wheel_path) + try: + with zipfile.ZipFile(path) as zf: + matches = [n for n in zf.namelist() if n.endswith(".dist-info/top_level.txt")] + if not matches: + return () + text = zf.read(matches[0]).decode("utf-8", errors="replace") + except (OSError, zipfile.BadZipFile): + return () + names: list[str] = [] + for line in text.splitlines(): + name = line.strip() + if name and name not in skip: + names.append(name) + return tuple(names) + + +def default_native_import_statements( + wheel_path: Path | str, + skip_top_level: frozenset[str] | None = None, +) -> tuple[str, ...]: + """``import `` statements for a native wheel without custom YAML imports.""" + tops = wheel_top_level_modules(wheel_path, skip_top_level) + if tops: + return tuple(f"import {mod}" for mod in tops) + parsed = parse_wheel_name(Path(wheel_path).name) + if not parsed: + return () + return (f"import {parsed[0].replace('-', '_')}",) + + +def resolve_native_import_statements( + wheel_path: Path | str, + *, + config: NativeImportGuardConfig | None = None, + current_platform: str | None = None, + python_version: str | None = None, +) -> tuple[str, ...] | None: + """Import statements for a wheel, or ``None`` to skip the probe. + + First matching YAML row (name + platform / python / version) wins: ``skip: true`` + skips; ``imports`` override the default. Unlisted platform wheels use + ``top_level.txt`` when ``probe_unlisted`` is true. + """ + path = Path(wheel_path) + parsed = parse_wheel_name(path.name) + if not parsed: + return None + cfg = config if config is not None else load_native_import_guard() + if cfg.skip_pure_any and is_pure_any_wheel_name(path.name): + return None + plat = current_platform if current_platform is not None else get_current_platform() + py_ver = python_version if python_version is not None else f"{sys.version_info.major}.{sys.version_info.minor}" + for entry in cfg.packages: + if entry.name != parsed[0]: + continue + if not _native_import_entry_matches( + entry, + current_platform=plat, + wheel_version=parsed[1], + python_version=py_ver, + ): + continue + if entry.skip: + return None + if entry.imports: + return entry.imports + break + if not cfg.probe_unlisted: + return None + stmts = default_native_import_statements(path, cfg.skip_top_level) + return stmts or None diff --git a/native_import_guard.yaml b/native_import_guard.yaml index 727cc28..4058015 100644 --- a/native_import_guard.yaml +++ b/native_import_guard.yaml @@ -1,21 +1,43 @@ -# Native ARMv7 wheels: import probes run by test_wheels_install.py after pip install (ARMv7 CI only). -# ``import cffi`` alone is not enough — a broken auditwheel manylinux tag can segfault only in _cffi_backend. +# Native import probes after pip install (test_wheels_install.py, every test-wheels-install runner). +# Does not change the requirements list. +# +# "After installing the wheel for package_name (on platform, for python, at version), +# run imports — or skip — and fail CI if import crashes or errors." +# +# Top-level: +# probe_unlisted — probe platform wheels not listed below via import (default true) +# skip_pure_any — skip *-none-any.whl (default true) +# skip_top_level — top_level.txt names ignored when generating default imports +# +# Each packages: entry (same filter tokens as exclude_list.yaml / include_list.yaml): +# package_name — PyPI distribution name (legacy key ``name`` is also accepted) +# imports — Python statement(s) in one subprocess (list; multiline allowed). Optional if skip. +# skip — true: do not probe when filters match (optional) +# platform — win32 | darwin | linux | linux_x86_64 | linux_arm64 | linux_armv7 | +# macos_x86_64 | macos_arm64 (string or list). Omitted = all platforms. +# python — python_version specifier(s), e.g. '>=3.10' or ['==3.8', '==3.9'] +# version — package version specifier(s), e.g. '>=49' or ['>=1.0', '<2'] # -# Each entry: -# name — PyPI distribution name (packaging canonical names) -# imports — Python statements executed in one subprocess (multiline allowed) +# ``import cffi`` alone is not enough — a broken auditwheel manylinux tag can segfault only in _cffi_backend. + +probe_unlisted: true +skip_pure_any: true +skip_top_level: + - test + - tests + - testing packages: - - name: cffi + - package_name: cffi imports: - import _cffi_backend - - name: argon2-cffi-bindings + - package_name: argon2-cffi-bindings imports: - from _argon2_cffi_bindings import _ffi - - name: argon2-cffi + - package_name: argon2-cffi imports: - import argon2 - - name: cryptography + - package_name: cryptography imports: - | try: @@ -24,9 +46,13 @@ packages: from cryptography.hazmat.bindings._openssl import ffi - from cryptography import x509 - print("cryptography import OK", x509.__name__) - - name: bcrypt + - package_name: bcrypt imports: - import bcrypt - - name: pynacl + - package_name: pynacl imports: - import nacl.bindings + - package_name: psutil + imports: + - import psutil + - print("psutil import OK", psutil.__version__) diff --git a/scripts/run_native_import_probe.py b/scripts/run_native_import_probe.py index b951108..7d51bdd 100644 --- a/scripts/run_native_import_probe.py +++ b/scripts/run_native_import_probe.py @@ -15,31 +15,23 @@ from pathlib import Path _ROOT = Path(__file__).resolve().parent.parent -_GUARD_PATH = _ROOT / "native_import_guard.yaml" +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) def _guard_import_blocks(package_names: list[str]) -> dict[str, str]: """Return canonical name → import probe source for requested packages.""" - import yaml - from packaging.utils import canonicalize_name - data = yaml.safe_load(_GUARD_PATH.read_text(encoding="utf-8")) or {} + from native_import_guard import load_native_import_guard + wanted = {canonicalize_name(name) for name in package_names} blocks: dict[str, str] = {} - for entry in data.get("packages") or []: - if not isinstance(entry, dict): - continue - name = entry.get("name") - imports = entry.get("imports") - if not name or not imports: - continue - canon = canonicalize_name(str(name)) - if canon not in wanted: + for entry in load_native_import_guard(_ROOT).packages: + if entry.name not in wanted or not entry.imports: continue - stmts = [str(stmt).strip() for stmt in imports if str(stmt).strip()] - if stmts: - blocks[canon] = "\n".join(stmts) + if entry.name not in blocks: + blocks[entry.name] = "\n".join(entry.imports) return blocks diff --git a/test_build_wheels.py b/test_build_wheels.py index 09c14b6..19d0153 100644 --- a/test_build_wheels.py +++ b/test_build_wheels.py @@ -319,6 +319,34 @@ def test_run_native_import_probes_fails_on_openssl_mismatch(self, mock_probe): self.assertFalse(wheel.exists()) mock_probe.assert_called_once() + @patch("test_wheels_install.run_import_probes", return_value=(True, "ok")) + def test_run_native_import_probes_defaults_for_unguarded_platform_wheel(self, mock_probe): + import zipfile + + from test_wheels_install import _run_native_import_probes + + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "greenlet-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl" + with zipfile.ZipFile(wheel, "w") as zf: + zf.writestr("greenlet-3.1.0.dist-info/top_level.txt", "greenlet\n") + failed, failures = _run_native_import_probes([wheel]) + self.assertEqual(failed, 0) + self.assertEqual(failures, []) + mock_probe.assert_called_once() + self.assertEqual(mock_probe.call_args[0][0], ("import greenlet",)) + + @patch("test_wheels_install.run_import_probes") + def test_run_native_import_probes_skips_pure_any_wheel(self, mock_probe): + from test_wheels_install import _run_native_import_probes + + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "six-1.16.0-py2.py3-none-any.whl" + wheel.write_bytes(b"") + failed, failures = _run_native_import_probes([wheel]) + self.assertEqual(failed, 0) + self.assertEqual(failures, []) + mock_probe.assert_not_called() + class TestPruneWheelsForArtifact(unittest.TestCase): """``prune_wheels_not_for_current_python`` keeps per-matrix wheels for CI artifacts.""" diff --git a/test_native_import_guard.py b/test_native_import_guard.py index cba4f58..a902c5e 100644 --- a/test_native_import_guard.py +++ b/test_native_import_guard.py @@ -6,9 +6,44 @@ from __future__ import annotations +import tempfile import unittest +import zipfile -from _helper_functions import native_import_guard_by_name +from pathlib import Path + +from native_import_guard import DEFAULT_SKIP_TOP_LEVEL_IMPORTS +from native_import_guard import NativeImportGuardConfig +from native_import_guard import NativeImportGuardEntry +from native_import_guard import default_native_import_statements +from native_import_guard import is_pure_any_wheel_name +from native_import_guard import load_native_import_guard +from native_import_guard import native_import_guard_by_name +from native_import_guard import resolve_native_import_statements + + +def _cfg( + *packages: NativeImportGuardEntry, + probe_unlisted: bool = True, + skip_pure_any: bool = True, +) -> NativeImportGuardConfig: + return NativeImportGuardConfig( + probe_unlisted=probe_unlisted, + skip_pure_any=skip_pure_any, + skip_top_level=DEFAULT_SKIP_TOP_LEVEL_IMPORTS, + packages=packages, + ) + + +def _write_wheel(tmp: str, filename: str, top_level: str = "") -> Path: + wheel = Path(tmp) / filename + if top_level: + with zipfile.ZipFile(wheel, "w") as zf: + info = filename.replace(".whl", "").rsplit("-", 3)[0] + zf.writestr(f"{info}.dist-info/top_level.txt", top_level) + else: + wheel.write_bytes(b"not-a-zip") + return wheel class TestNativeImportGuard(unittest.TestCase): @@ -22,6 +57,125 @@ def test_loads_cryptography_fallback(self) -> None: self.assertIn("from cryptography import x509", code) self.assertIn("cryptography import OK", code) + def test_loads_psutil(self) -> None: + guarded = native_import_guard_by_name() + entry = guarded["psutil"] + code = "\n".join(entry.imports) + self.assertIn("import psutil", code) + self.assertIn("psutil import OK", code) + + def test_yaml_uses_package_name_and_probe_unlisted(self) -> None: + cfg = load_native_import_guard() + self.assertTrue(cfg.probe_unlisted) + self.assertTrue(cfg.skip_pure_any) + self.assertEqual(cfg.skip_top_level, DEFAULT_SKIP_TOP_LEVEL_IMPORTS) + names = {entry.name for entry in cfg.packages} + self.assertIn("cffi", names) + self.assertIn("psutil", names) + + def test_pure_any_wheel_name(self) -> None: + self.assertTrue(is_pure_any_wheel_name("six-1.16.0-py2.py3-none-any.whl")) + self.assertFalse(is_pure_any_wheel_name("psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl")) + + def test_default_imports_from_top_level_txt(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "greenlet-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl" + with zipfile.ZipFile(wheel, "w") as zf: + zf.writestr("greenlet-3.1.0.dist-info/top_level.txt", "greenlet\ntest\n") + self.assertEqual(default_native_import_statements(wheel), ("import greenlet",)) + + def test_default_imports_fallback_to_distribution_name(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_10_9_x86_64.whl" + wheel.write_bytes(b"not-a-zip") + self.assertEqual( + default_native_import_statements(wheel), + ("import ruamel_yaml_clib",), + ) + + def test_resolve_custom_imports(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel(tmp, "cffi-1.17.1-cp311-cp311-manylinux_2_28_x86_64.whl") + cfg = _cfg( + NativeImportGuardEntry( + name="cffi", + imports=("import _cffi_backend",), + skip=False, + raw={"package_name": "cffi"}, + ) + ) + self.assertEqual( + resolve_native_import_statements(wheel, config=cfg, current_platform="linux_x86_64"), + ("import _cffi_backend",), + ) + + def test_resolve_skip_on_platform(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel( + tmp, + "foo-1.0.0-cp311-cp311-linux_armv7l.whl", + top_level="foo\n", + ) + cfg = _cfg( + NativeImportGuardEntry( + name="foo", + imports=(), + skip=True, + raw={"package_name": "foo", "skip": True, "platform": "linux_armv7"}, + ) + ) + self.assertIsNone(resolve_native_import_statements(wheel, config=cfg, current_platform="linux_armv7")) + self.assertEqual( + resolve_native_import_statements(wheel, config=cfg, current_platform="macos_x86_64"), + ("import foo",), + ) + + def test_resolve_python_and_version_filters(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel(tmp, "bar-2.0.0-cp311-cp311-manylinux_2_28_x86_64.whl") + cfg = _cfg( + NativeImportGuardEntry( + name="bar", + imports=("import bar_ext",), + skip=False, + raw={"package_name": "bar", "python": ">=3.10", "version": ">=2"}, + ) + ) + self.assertEqual( + resolve_native_import_statements( + wheel, + config=cfg, + current_platform="linux_x86_64", + python_version="3.11", + ), + ("import bar_ext",), + ) + self.assertEqual( + resolve_native_import_statements( + wheel, + config=cfg, + current_platform="linux_x86_64", + python_version="3.9", + ), + ("import bar",), + ) + + def test_resolve_probe_unlisted_false_skips_unknown(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel( + tmp, + "greenlet-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", + top_level="greenlet\n", + ) + cfg = _cfg(probe_unlisted=False) + self.assertIsNone(resolve_native_import_statements(wheel, config=cfg, current_platform="macos_x86_64")) + + def test_resolve_skips_pure_any(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel(tmp, "six-1.16.0-py2.py3-none-any.whl", top_level="six\n") + cfg = _cfg() + self.assertIsNone(resolve_native_import_statements(wheel, config=cfg)) + if __name__ == "__main__": unittest.main() diff --git a/test_wheels_install.py b/test_wheels_install.py index 385346f..7ea25f2 100644 --- a/test_wheels_install.py +++ b/test_wheels_install.py @@ -14,9 +14,10 @@ do not carry wheels for other Python versions. CI downloads ``wheels-repaired-`` per matrix row (not the full merge) so ARMv7 vs ARMv7 Legacy binaries are not mixed. -After ``pip install``, :func:`_run_native_import_probes` imports native extensions listed -in ``native_import_guard.yaml`` (e.g. ``from cryptography import x509``) so wheels that -install cleanly but fail at load time (OpenSSL symbol mismatches) fail CI before upload. +After ``pip install``, :func:`_run_native_import_probes` applies +``native_import_guard.yaml`` (custom imports, skip filters, and default +``import `` when ``probe_unlisted`` is true) so wheels that +install cleanly but fail at load time (OpenSSL / SIGABRT) fail CI before upload. Wheels are ZIP archives (PEP 427). pip opens them with the zipfile module; a BadZipFile / "Bad magic number" error means the bytes on disk are not a valid @@ -39,11 +40,12 @@ from _helper_functions import EXCLUDE_LIST_PATH from _helper_functions import armv7_wheel_matches_forced_plat from _helper_functions import get_current_platform -from _helper_functions import native_import_guard_by_name from _helper_functions import parse_wheel_name from _helper_functions import print_color from _helper_functions import should_exclude_wheel from _helper_functions import wheel_archive_is_readable +from native_import_guard import load_native_import_guard +from native_import_guard import resolve_native_import_statements from native_import_probe import run_import_probes from yaml_list_adapter import YAMLListAdapter @@ -100,8 +102,12 @@ def _should_skip_native_import_probe(dist_name: str) -> bool: def _run_native_import_probes(installed_wheels: list[Path]) -> tuple[int, list[tuple[str, str]]]: - """Import native extensions after install; catch load-time ABI/OpenSSL mismatches.""" - guarded = native_import_guard_by_name() + """Import native extensions after install; catch load-time ABI/OpenSSL mismatches. + + Resolution is ``native_import_guard.yaml`` (custom imports, skip filters, + ``probe_unlisted`` default ``import ``). + """ + config = load_native_import_guard() native_failed = 0 failed_wheels: list[tuple[str, str]] = [] if not installed_wheels: @@ -110,7 +116,10 @@ def _run_native_import_probes(installed_wheels: list[Path]) -> tuple[int, list[t print_color("---------- NATIVE IMPORT PROBES ----------") for wheel_path in installed_wheels: parsed = parse_wheel_name(wheel_path.name) - if not parsed or parsed[0] not in guarded: + if not parsed: + continue + stmts = resolve_native_import_statements(wheel_path, config=config) + if stmts is None: continue if _should_skip_native_import_probe(parsed[0]): print_color( @@ -118,7 +127,7 @@ def _run_native_import_probes(installed_wheels: list[Path]) -> tuple[int, list[t Fore.YELLOW, ) continue - ok, msg = run_import_probes(guarded[parsed[0]].imports) + ok, msg = run_import_probes(stmts) if ok: print_color(f"-- {wheel_path.name}", Fore.GREEN) if msg: From f7f460f13da50658b6b09b987449f20f1525fe78 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Mon, 31 Aug 2026 15:16:53 +0200 Subject: [PATCH 2/7] fix: Prefer PyPI macOS wheels so CI tags cannot steal pip Keep portable macosx tags on the extra-index Local macos-15 builds were winning over official PyPI wheels (for example psutil 7.2.2 SIGABRT) --- README.md | 6 + _helper_functions.py | 276 +-------------------------- build_wheels.py | 4 +- build_wheels_from_file.py | 4 +- repair_wheels.py | 31 ++- test_build_wheels.py | 102 ++++++++-- wheel_compat_policy.py | 390 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 517 insertions(+), 296 deletions(-) create mode 100644 wheel_compat_policy.py diff --git a/README.md b/README.md index a4baad8..e56a89a 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,12 @@ Mitigations in this repo: Upstream [dropped x86_64 macOS support](https://github.com/pyca/cryptography/compare/48.0.1...49.0.0) in 49+; this path is maintained locally until ESP-IDF drops Intel Mac. Scheduled CI runs [`validate_cryptography_macos_intel_wheel.sh`](./scripts/validate_cryptography_macos_intel_wheel.sh) after macOS Intel builds and after [`delocate`](./repair_wheels.py) repair. For a local isolated rebuild smoke test, use [`spike_cryptography_macos_intel_openssl4.sh`](./scripts/spike_cryptography_macos_intel_openssl4.sh). +### macOS: prefer PyPI wheels (avoid CI ``macosx_*`` tag stealing) + +A local sdist build on GitHub `macos-15-*` runners often stamps a **higher** `macosx_*` tag (`macosx_15_0_x86_64`, …) than the official cibuildwheel tag (`macosx_10_9_x86_64`, `macosx_11_0_arm64`). Pip prefers the highest compatible tag, so the extra-index CI copy wins over PyPI. For **psutil 7.2.2** that locally compiled `_psutil_osx.abi3.so` SIGABRT’d ESP-IDF `export.sh` / `test_pytest_macos`. + +macOS `pip wheel` therefore passes **`--only-binary :all:`** (Intel **cryptography** still uses `--no-binary cryptography` for the OpenSSL 4 rebuild above). [`repair_wheels.py`](./repair_wheels.py) **skips delocate** except that Intel cryptography rebuild, and **prunes** any higher `macosx_*` sibling for the same distribution/version/arch so it cannot be uploaded. After install, every platform wheel is import-probed. Leftover higher-tag objects already on the extra-index (for example [psutil 7.2.2](https://dl.espressif.com/pypi/psutil/)) must be deleted from the bucket; a later upload only overwrites the same filename. + ## Activity Diagram The main file is `build-wheels-platforms.yml` which is scheduled to run periodically to build Python wheels for any requirement of all [ESP-IDF]-supported versions. diff --git a/_helper_functions.py b/_helper_functions.py index 261ecd4..0665803 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -10,7 +10,6 @@ import os import platform import re -import subprocess import sys import zipfile @@ -502,6 +501,7 @@ def get_no_binary_args(requirement_name: str) -> list: canonicalize_name("argon2-cffi-bindings"), canonicalize_name("cryptography"), canonicalize_name("protobuf"), + canonicalize_name("psutil"), canonicalize_name("pydantic-core"), canonicalize_name("pynacl"), canonicalize_name("rpds-py"), @@ -568,272 +568,11 @@ def bounded_pin_without_find_links_skip( return False, "" -def pip_wheel_standard_args(find_links_dir: Path | str = DEFAULT_WHEEL_DIR) -> list[str]: - """Shared ``pip wheel`` flags used by ``build_wheels.py`` and ``build_wheels_from_file.py``.""" - wheel_dir = str(find_links_dir) - return [ - "--find-links", - wheel_dir, - "--find-links", - PYPI_SIMPLE_INDEX, - "--wheel-dir", - wheel_dir, - "--no-cache-dir", - "--no-build-isolation", - ] - - -def pip_wheel_invocation_args( - requirement_name: str, - find_links_dir: Path | str = DEFAULT_WHEEL_DIR, -) -> list[str]: - """``pip wheel`` flags for a single requirement (may drop ``--no-build-isolation`` on ARMv7).""" - args = list(pip_wheel_standard_args(find_links_dir)) - if is_linux_armv7_runner(): - # cffi 2.x sdists fail metadata prep under host setuptools + --no-build-isolation - # (project.license validation), including transitive cffi pulls (e.g. esptool → cryptography). - # PEP 517 isolation matches build_requirements install on ARMv7 Docker. - del requirement_name # per-requirement env still via armv7_pip_wheel_subprocess_env() - args = [arg for arg in args if arg != "--no-build-isolation"] - return args - - -def _wheel_has_manylinux_228_tag(wheel_name: str) -> bool: - """True when the wheel filename includes a ``manylinux_2_28`` platform tag.""" - return (2, 28) in manylinux_glibc_tags_in_name(wheel_name) - - -def _requirement_exact_version(req: Requirement) -> str | None: - """Return the pinned version when the requirement uses ``==``.""" - for spec in req.specifier: - if spec.operator == "==": - return str(spec.version) - return None - - -def _wheel_highest_manylinux_glibc(wheel_name: str) -> tuple[int, int] | None: - """Return the highest ``manylinux_M_N`` glibc tag embedded in a wheel filename.""" - tags = manylinux_glibc_tags_in_name(wheel_name) - return max(tags) if tags else None - - -def prune_ci_manylinux_newer_than_228( - package_name: str, - package_version: str, - *, - wheel_dir: Path | str = DEFAULT_WHEEL_DIR, -) -> int: - """Drop CI-built manylinux wheels newer than ``2_28`` after mirroring PyPI ``2_28``. - - Ubuntu 24.04 rebuilds link against newer system OpenSSL (e.g. SM4 symbols) and - auditwheel tags ``manylinux_2_34_*``. Pip prefers those over mirrored ``2_28`` - wheels when both are on the index, breaking consumers on older OpenSSL (ESP-IDF - Docker, esptool PyInstaller on focal, etc.). - """ - dest = Path(wheel_dir) - canonical = canonicalize_name(package_name) - version = str(package_version) - removed = 0 - for path in dest.glob("*.whl"): - parsed = parse_wheel_name(path.name) - if not parsed or parsed[0] != canonical or parsed[1] != version: - continue - glibc = _wheel_highest_manylinux_glibc(path.name) - if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): - print_color( - f"-- removed {path.name} (CI manylinux_{glibc[0]}_{glibc[1]}; PyPI 2_28 mirror kept)", - Fore.YELLOW, - ) - path.unlink() - removed += 1 - return removed - - -def prune_ci_manylinux_newer_than_228_when_228_mirror_present( - *, - wheel_dir: Path | str = DEFAULT_WHEEL_DIR, - package_names: frozenset[str] | None = None, -) -> int: - """Drop manylinux > ``2_28`` when a ``2_28`` wheel exists for the same dist+version. - - Used after merging matrix artifacts and after ``repair_wheels`` so CI or auditwheel - outputs cannot coexist with PyPI ``manylinux_2_28`` mirrors on the upload index. - """ - dest = Path(wheel_dir) - by_dist_version: dict[tuple[str, str], list[Path]] = {} - for path in dest.rglob("*.whl"): - parsed = parse_wheel_name(path.name) - if not parsed: - continue - if package_names is not None: - pkg_key = parsed[0].replace("-", "_") - if pkg_key not in package_names: - continue - by_dist_version.setdefault(parsed, []).append(path) - - removed = 0 - for (dist, version), paths in by_dist_version.items(): - if not any(_wheel_has_manylinux_228_tag(p.name) for p in paths): - continue - for path in paths: - glibc = _wheel_highest_manylinux_glibc(path.name) - if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): - print_color( - f"-- removed {path.name} (manylinux_{glibc[0]}_{glibc[1]}; " - f"keeping manylinux_2_28 for {dist}=={version})", - Fore.YELLOW, - ) - path.unlink() - removed += 1 - return removed - - -def should_skip_linux_auditwheel_for_pypi_mirror(wheel_name: str) -> bool: - """Skip auditwheel on PyPI ``manylinux_2_28`` mirrors (repair retags to host ``2_34``).""" - machine = platform.machine().lower() - if machine not in ("x86_64", "amd64", "aarch64", "arm64"): - return False - parsed = parse_wheel_name(wheel_name) - if not parsed: - return False - pkg_key = parsed[0].replace("-", "_") - if pkg_key not in _force_no_binary_linux_normalized(_REPO_ROOT): - return False - return _wheel_has_manylinux_228_tag(wheel_name) - - -def _mirrored_manylinux228_version(package_name: str, wheel_dir: Path) -> str | None: - """Return the version string of a ``manylinux_2_28`` wheel mirrored for ``package_name``.""" - canonical = canonicalize_name(package_name) - for path in wheel_dir.glob("*.whl"): - parsed = parse_wheel_name(path.name) - if not parsed or parsed[0] != canonical: - continue - if _wheel_has_manylinux_228_tag(path.name): - return parsed[1] - return None - - -def _prune_mirrored_manylinux228_ci_builds( - req: Requirement, - *, - wheel_dir: Path, -) -> int: - """Remove CI manylinux > ``2_28`` after mirroring PyPI ``2_28`` for ``req``.""" - mirrored_version = _mirrored_manylinux228_version(req.name, wheel_dir) - if not mirrored_version: - mirrored_version = _requirement_exact_version(req) - if not mirrored_version: - return 0 - pruned = prune_ci_manylinux_newer_than_228(req.name, mirrored_version, wheel_dir=wheel_dir) - if pruned: - print_color( - f"-- pruned {pruned} CI manylinux wheel(s) newer than 2_28 for {req.name}=={mirrored_version}", - Fore.YELLOW, - ) - return pruned - - def _requirement_for_pip_download(req: Requirement) -> str: """Return ``name`` + ``specifier`` only; ``pip download`` rejects PEP 508 markers.""" return f"{req.name}{req.specifier}" -def mirror_pypi_manylinux228_wheel( - requirement_line: str, - *, - wheel_dir: Path | str = DEFAULT_WHEEL_DIR, -) -> bool: - """Download a ``manylinux_2_28`` wheel from PyPI for older glibc/OpenSSL consumers. - - CI rebuilds on Ubuntu 24.04 produce ``manylinux_2_34_*`` wheels (glibc >= 2.34, - newer system OpenSSL). Downstream tools (ESP-IDF Docker, esptool PyInstaller on - Ubuntu 20.04) need PyPI's ``manylinux_2_28`` wheel instead. After a successful - mirror, locally rebuilt ``manylinux`` wheels newer than ``2_28`` for the same - version are removed so pip cannot prefer the incompatible CI build. - - ARMv7 builds use piwheels and different manylinux lineage tags; no mirroring here. - """ - if platform.system() != "Linux" or is_linux_armv7_runner(): - return False - machine = platform.machine().lower() - pip_platform = { - "x86_64": "manylinux_2_28_x86_64", - "amd64": "manylinux_2_28_x86_64", - "aarch64": "manylinux_2_28_aarch64", - "arm64": "manylinux_2_28_aarch64", - }.get(machine) - if not pip_platform: - return False - - line = requirement_line.strip() - if not line: - return False - try: - req = Requirement(line) - except InvalidRequirement: - return False - - download_line = _requirement_for_pip_download(req) - - pkg_key = canonicalize_name(req.name).replace("-", "_") - if pkg_key not in _force_no_binary_linux_normalized(_REPO_ROOT): - return False - - py_major, py_minor = sys.version_info[:2] - dest = Path(wheel_dir) - dest.mkdir(exist_ok=True) - cp_abi = f"cp{py_major}{py_minor}" - abi_candidates = [cp_abi, "abi3"] if cp_abi != "abi3" else ["abi3"] - - for abi in abi_candidates: - out = subprocess.run( - [ - sys.executable, - "-m", - "pip", - "download", - download_line, - "--only-binary", - ":all:", - "--dest", - str(dest), - "--no-deps", - "--platform", - pip_platform, - "--python-version", - f"{py_major}.{py_minor}", - "--implementation", - "cp", - "--abi", - abi, - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - if out.stdout: - print(out.stdout.decode("utf-8", errors="replace")) - if out.returncode == 0: - print_color( - f"-- mirrored {pip_platform} wheel for {download_line} from PyPI (abi={abi}; older glibc consumers)", - Fore.YELLOW, - ) - _prune_mirrored_manylinux228_ci_builds(req, wheel_dir=dest) - return True - - if out.stderr: - print_color(out.stderr.decode("utf-8", errors="replace"), Fore.YELLOW) - print_color( - f"-- could not mirror {pip_platform} wheel for {download_line} from PyPI", - Fore.YELLOW, - ) - return False - - -# Backward-compatible alias (same behavior; not cryptography-specific). -mirror_pypi_manylinux228_x86_64_wheel = mirror_pypi_manylinux228_wheel - - def _find_links_versions_too_new_for_pin(versions: list[Version], req: Requirement) -> bool: """True when every find-links wheel is excluded by an upper bound (obsolete pin; skip sdist).""" if not req.specifier: @@ -853,19 +592,6 @@ def _find_links_versions_too_new_for_pin(versions: list[Version], req: Requireme return False -def pip_wheel_or_mirror_success( - requirement_line: str, - pip_returncode: int, - *, - wheel_dir: Path | str = DEFAULT_WHEEL_DIR, -) -> bool: - """True when ``pip wheel`` succeeded or a PyPI ``manylinux_2_28`` mirror was fetched.""" - if pip_returncode == 0: - mirror_pypi_manylinux228_wheel(requirement_line, wheel_dir=wheel_dir) - return True - return mirror_pypi_manylinux228_wheel(requirement_line, wheel_dir=wheel_dir) - - def find_links_wheel_build_skip( req: Requirement, find_links_dir: Path | str = DEFAULT_WHEEL_DIR, diff --git a/build_wheels.py b/build_wheels.py index fd4e441..f1062a3 100644 --- a/build_wheels.py +++ b/build_wheels.py @@ -38,13 +38,13 @@ from _helper_functions import is_linux_armv7_runner from _helper_functions import macos_intel_cryptography_rebuild_instead_of_find_links_skip from _helper_functions import merge_requirements -from _helper_functions import pip_wheel_invocation_args -from _helper_functions import pip_wheel_or_mirror_success from _helper_functions import print_color from _helper_functions import remove_find_links_wheels_for_package from emit_sdist_requirements import SDIST_REQUIREMENTS_FILE from emit_sdist_requirements import compute_sdist_requirements from emit_sdist_requirements import write_sdist_requirements_file +from wheel_compat_policy import pip_wheel_invocation_args +from wheel_compat_policy import pip_wheel_or_mirror_success from yaml_list_adapter import YAMLListAdapter # GLOBAL VARIABLES diff --git a/build_wheels_from_file.py b/build_wheels_from_file.py index 2744d82..9159de8 100644 --- a/build_wheels_from_file.py +++ b/build_wheels_from_file.py @@ -29,11 +29,11 @@ from _helper_functions import get_pip_wheel_extra_args from _helper_functions import is_linux_armv7_runner from _helper_functions import macos_intel_cryptography_rebuild_instead_of_find_links_skip -from _helper_functions import pip_wheel_invocation_args -from _helper_functions import pip_wheel_or_mirror_success from _helper_functions import print_color from _helper_functions import pypi_requires_python_preflight_skip from _helper_functions import remove_find_links_wheels_for_package +from wheel_compat_policy import pip_wheel_invocation_args +from wheel_compat_policy import pip_wheel_or_mirror_success def _force_interpreter_no_binary_args(requirement_line: str) -> list[str]: diff --git a/repair_wheels.py b/repair_wheels.py index aff25da..d610f42 100644 --- a/repair_wheels.py +++ b/repair_wheels.py @@ -39,9 +39,11 @@ from _helper_functions import armv7_wheel_matches_forced_plat from _helper_functions import parse_wheel_name from _helper_functions import print_color -from _helper_functions import prune_ci_manylinux_newer_than_228_when_228_mirror_present -from _helper_functions import should_skip_linux_auditwheel_for_pypi_mirror from _helper_functions import wheel_archive_is_readable +from wheel_compat_policy import prune_ci_macos_newer_than_pypi_mirror +from wheel_compat_policy import prune_ci_manylinux_newer_than_228_when_228_mirror_present +from wheel_compat_policy import should_skip_linux_auditwheel_for_pypi_mirror +from wheel_compat_policy import should_skip_macos_delocate_for_pypi_mirror def _stderr_indicates_bad_zip(error_msg: str) -> bool: @@ -397,6 +399,15 @@ def main() -> None: ) wheels = _dedupe_wheel_paths(wheels_dir) + if current_platform == "Darwin": + pruned = prune_ci_macos_newer_than_pypi_mirror(wheel_dir=wheels_dir) + if pruned: + print_color( + f"Pre-repair: pruned {pruned} macOS wheel(s) newer than the PyPI macosx tag", + Fore.YELLOW, + ) + wheels = _dedupe_wheel_paths(wheels_dir) + repaired_count: int = 0 skipped_count: int = 0 deleted_count: int = 0 @@ -481,6 +492,14 @@ def main() -> None: skipped_count += 1 continue + if current_platform == "Darwin" and should_skip_macos_delocate_for_pypi_mirror(wheel.name): + print_color( + " -> Skipping delocate (keep PyPI macOS tag; local retag can steal pip preference)", + Fore.YELLOW, + ) + skipped_count += 1 + continue + # Clean temp directory for old_wheel in temp_dir.glob("*.whl"): old_wheel.unlink() @@ -736,6 +755,14 @@ def main() -> None: Fore.YELLOW, ) + if current_platform == "Darwin": + pruned = prune_ci_macos_newer_than_pypi_mirror(wheel_dir=wheels_dir) + if pruned: + print_color( + f"Pruned {pruned} macOS wheel(s) newer than the PyPI macosx tag", + Fore.YELLOW, + ) + if current_platform == "Linux" and current_arch == "armv7l": pruned = _prune_manylinux_armv7_when_linux_tag_present(wheels_dir) if pruned: diff --git a/test_build_wheels.py b/test_build_wheels.py index 19d0153..c552a1b 100644 --- a/test_build_wheels.py +++ b/test_build_wheels.py @@ -19,7 +19,6 @@ from packaging.requirements import Requirement from packaging.utils import canonicalize_name -from _helper_functions import _mirrored_manylinux228_version from _helper_functions import armv7_pip_wheel_subprocess_env from _helper_functions import armv7_rebuild_instead_of_find_links_skip from _helper_functions import bounded_pin_without_find_links_skip @@ -32,15 +31,19 @@ from _helper_functions import get_no_binary_args from _helper_functions import macos_intel_cryptography_rebuild_instead_of_find_links_skip from _helper_functions import merge_requirements -from _helper_functions import mirror_pypi_manylinux228_wheel -from _helper_functions import pip_wheel_or_mirror_success -from _helper_functions import prune_ci_manylinux_newer_than_228 -from _helper_functions import prune_ci_manylinux_newer_than_228_when_228_mirror_present from _helper_functions import pypi_requires_python_preflight_skip from _helper_functions import remove_find_links_wheels_for_package -from _helper_functions import should_skip_linux_auditwheel_for_pypi_mirror from build_wheels import _add_into_requirements from build_wheels import get_used_idf_branches +from wheel_compat_policy import _mirrored_manylinux228_version +from wheel_compat_policy import get_macos_only_binary_args +from wheel_compat_policy import mirror_pypi_manylinux228_wheel +from wheel_compat_policy import pip_wheel_or_mirror_success +from wheel_compat_policy import prune_ci_macos_newer_than_pypi_mirror +from wheel_compat_policy import prune_ci_manylinux_newer_than_228 +from wheel_compat_policy import prune_ci_manylinux_newer_than_228_when_228_mirror_present +from wheel_compat_policy import should_skip_linux_auditwheel_for_pypi_mirror +from wheel_compat_policy import should_skip_macos_delocate_for_pypi_mirror from yaml_list_adapter import YAMLListAdapter @@ -401,27 +404,95 @@ def test_armv7_cryptography_no_deps(self, _armv7: object) -> None: class TestPipWheelInvocationArgs(unittest.TestCase): - @patch("_helper_functions.is_linux_armv7_runner", return_value=True) + @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=True) def test_armv7_cffi_uses_build_isolation(self, _armv7: object) -> None: - from _helper_functions import pip_wheel_invocation_args + from wheel_compat_policy import pip_wheel_invocation_args args = pip_wheel_invocation_args("cffi>=1.15.0") self.assertNotIn("--no-build-isolation", args) - @patch("_helper_functions.is_linux_armv7_runner", return_value=True) + @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=True) def test_armv7_any_package_uses_build_isolation(self, _armv7: object) -> None: - from _helper_functions import pip_wheel_invocation_args + from wheel_compat_policy import pip_wheel_invocation_args args = pip_wheel_invocation_args("esptool~=4.9") self.assertNotIn("--no-build-isolation", args) - @patch("_helper_functions.is_linux_armv7_runner", return_value=False) + @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=False) def test_non_armv7_keeps_no_build_isolation(self, _armv7: object) -> None: - from _helper_functions import pip_wheel_invocation_args + from wheel_compat_policy import pip_wheel_invocation_args args = pip_wheel_invocation_args("cffi>=1.15.0") self.assertIn("--no-build-isolation", args) + @patch("wheel_compat_policy.platform.system", return_value="Darwin") + @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=False) + def test_macos_pip_wheel_only_binary_all(self, _armv7: object, _sys: object) -> None: + from wheel_compat_policy import pip_wheel_invocation_args + + args = pip_wheel_invocation_args("psutil") + only_at = args.index("--only-binary") + self.assertEqual(args[only_at + 1], ":all:") + + +class TestMacosPypiMirrorPolicy(unittest.TestCase): + @patch("wheel_compat_policy.platform.system", return_value="Darwin") + def test_only_binary_args_on_macos(self, _sys: object) -> None: + self.assertEqual(get_macos_only_binary_args(), ["--only-binary", ":all:"]) + + @patch("wheel_compat_policy.platform.system", return_value="Linux") + def test_only_binary_args_off_linux(self, _sys: object) -> None: + self.assertEqual(get_macos_only_binary_args(), []) + + @patch("_helper_functions.platform.machine", return_value="x86_64") + @patch("wheel_compat_policy.platform.system", return_value="Darwin") + def test_skip_delocate_for_pypi_macos_wheels(self, _sys: object, _mach: object) -> None: + self.assertTrue(should_skip_macos_delocate_for_pypi_mirror("psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl")) + self.assertTrue(should_skip_macos_delocate_for_pypi_mirror("cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl")) + + @patch("_helper_functions.platform.machine", return_value="x86_64") + @patch("_helper_functions.platform.system", return_value="Darwin") + @patch("wheel_compat_policy.platform.system", return_value="Darwin") + def test_keep_delocate_for_cryptography_intel(self, _sys: object, _hf_sys: object, _mach: object) -> None: + self.assertFalse( + should_skip_macos_delocate_for_pypi_mirror("cryptography-49.0.0-cp313-abi3-macosx_10_12_x86_64.whl") + ) + + @patch("_helper_functions.platform.machine", return_value="arm64") + @patch("_helper_functions.platform.system", return_value="Darwin") + @patch("wheel_compat_policy.platform.system", return_value="Darwin") + def test_skip_delocate_for_cryptography_arm(self, _sys: object, _hf_sys: object, _mach: object) -> None: + self.assertTrue( + should_skip_macos_delocate_for_pypi_mirror("cryptography-49.0.0-cp313-abi3-macosx_11_0_arm64.whl") + ) + + @patch("wheel_compat_policy.platform.system", return_value="Linux") + def test_skip_delocate_off_linux(self, _sys: object) -> None: + self.assertFalse(should_skip_macos_delocate_for_pypi_mirror("psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl")) + + def test_prune_keeps_lowest_macosx_tag_any_package(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep = links / "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl" + drop = links / "cffi-1.17.1-cp311-cp311-macosx_15_0_x86_64.whl" + other = links / "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl" + keep.write_bytes(b"pypi") + drop.write_bytes(b"local") + other.write_bytes(b"arm") + removed = prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links) + self.assertEqual(removed, 1) + self.assertTrue(keep.exists()) + self.assertFalse(drop.exists()) + self.assertTrue(other.exists()) + + def test_prune_noop_when_only_one_tag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + only = links / "psutil-7.2.2-cp36-abi3-macosx_15_0_x86_64.whl" + only.write_bytes(b"local") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 0) + self.assertTrue(only.exists()) + class TestParseWheelName(unittest.TestCase): """Test the parse_wheel_name function from _helper_functions.py.""" @@ -681,6 +752,7 @@ def test_skip_armv7_cffi_backed_packages(self): "tibs", "rpds-py", "cryptography", + "psutil", ): self.assertTrue(force_interpreter_skip_package(canonicalize_name(name))) @@ -810,17 +882,17 @@ def test_armv7_pip_wheel_env_leaves_global_for_other_packages(self, _mock_armv7) class TestPipWheelOrMirrorSuccess(unittest.TestCase): - @patch("_helper_functions.mirror_pypi_manylinux228_wheel", return_value=True) + @patch("wheel_compat_policy.mirror_pypi_manylinux228_wheel", return_value=True) def test_pip_success_still_mirrors_and_returns_true(self, mock_mirror): self.assertTrue(pip_wheel_or_mirror_success("greenlet>3.0.0", 0)) mock_mirror.assert_called_once() - @patch("_helper_functions.mirror_pypi_manylinux228_wheel", return_value=True) + @patch("wheel_compat_policy.mirror_pypi_manylinux228_wheel", return_value=True) def test_pip_failure_succeeds_when_mirror_works(self, mock_mirror): self.assertTrue(pip_wheel_or_mirror_success("pillow!=9.5.0", 1)) mock_mirror.assert_called_once() - @patch("_helper_functions.mirror_pypi_manylinux228_wheel", return_value=False) + @patch("wheel_compat_policy.mirror_pypi_manylinux228_wheel", return_value=False) def test_pip_failure_fails_when_mirror_fails(self, mock_mirror): self.assertFalse(pip_wheel_or_mirror_success("cffi==1.17.1", 1)) mock_mirror.assert_called_once() diff --git a/wheel_compat_policy.py b/wheel_compat_policy.py new file mode 100644 index 0000000..dfbf33c --- /dev/null +++ b/wheel_compat_policy.py @@ -0,0 +1,390 @@ +# +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# +"""macOS / manylinux tag policy and the ``pip wheel`` invocation that applies it.""" + +from __future__ import annotations + +import platform +import re +import subprocess +import sys + +from pathlib import Path + +from colorama import Fore +from packaging.requirements import InvalidRequirement +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from _helper_functions import _REPO_ROOT +from _helper_functions import DEFAULT_WHEEL_DIR +from _helper_functions import PYPI_SIMPLE_INDEX +from _helper_functions import _force_no_binary_linux_normalized +from _helper_functions import _requirement_for_pip_download +from _helper_functions import get_current_platform +from _helper_functions import is_linux_armv7_runner +from _helper_functions import manylinux_glibc_tags_in_name +from _helper_functions import parse_wheel_name +from _helper_functions import print_color + +# macOS: ``pip wheel --only-binary :all:`` so CI does not sdist-compile against the +# runner SDK (higher ``macosx_*`` tags steal pip preference; local ``.so`` can SIGABRT). +# ``--no-binary cryptography`` on Intel still wins (OpenSSL 4 rebuild). +_MACOSX_PLAT_TAG = re.compile(r"macosx_(\d+)_(\d+)_([a-z0-9_]+)", re.IGNORECASE) + + +def get_macos_only_binary_args() -> list[str]: + """Prefer PyPI wheels on macOS; do not sdist-build against the runner SDK.""" + if platform.system() != "Darwin": + return [] + return ["--only-binary", ":all:"] + + +def _macosx_deployment_and_arch(wheel_name: str) -> tuple[tuple[int, int], str] | None: + """Return ``((major, minor), arch)`` from a ``macosx_M_N_arch`` wheel filename.""" + match = _MACOSX_PLAT_TAG.search(wheel_name) + if not match: + return None + return (int(match.group(1)), int(match.group(2))), match.group(3).lower() + + +def should_skip_macos_delocate_for_pypi_mirror(wheel_name: str) -> bool: + """Skip delocate on macOS wheels except the Intel cryptography OpenSSL 4 rebuild. + + delocate on a PyPI copy can rewrite ``WHEEL`` / retag ``macosx_*`` upward so pip + prefers the CI artifact. cryptography on Intel must still be repaired (bundled OpenSSL 4). + """ + if platform.system() != "Darwin": + return False + if "macosx" not in wheel_name.lower(): + return False + parsed = parse_wheel_name(wheel_name) + if not parsed: + return False + if parsed[0] == canonicalize_name("cryptography") and get_current_platform() == "macos_x86_64": + return False + return True + + +def prune_ci_macos_newer_than_pypi_mirror( + *, + wheel_dir: Path | str = DEFAULT_WHEEL_DIR, +) -> int: + """Drop locally tagged macOS wheels when a lower ``macosx_*`` sibling exists. + + Pip prefers the highest compatible ``macosx_*`` tag. A CI sdist build on + ``macos-15-intel`` becomes ``macosx_15_0_x86_64`` and wins over the official + ``macosx_10_9_x86_64`` PyPI copy on the same extra-index. Applies to every + distribution (not only psutil). + """ + dest = Path(wheel_dir) + if not dest.is_dir(): + return 0 + groups: dict[tuple[str, str, str], list[tuple[tuple[int, int], Path]]] = {} + for path in dest.rglob("*.whl"): + parsed = parse_wheel_name(path.name) + if not parsed: + continue + tag = _macosx_deployment_and_arch(path.name) + if not tag: + continue + deployment, arch = tag + groups.setdefault((parsed[0], parsed[1], arch), []).append((deployment, path)) + + removed = 0 + for (dist, version, arch), items in groups.items(): + if len(items) < 2: + continue + lowest = min(deployment for deployment, _path in items) + for deployment, path in items: + if deployment <= lowest: + continue + print_color( + f"-- removed {path.name} (macosx_{deployment[0]}_{deployment[1]}_{arch}; " + f"keeping macosx_{lowest[0]}_{lowest[1]} for {dist}=={version})", + Fore.YELLOW, + ) + path.unlink() + removed += 1 + return removed + + +def pip_wheel_standard_args(find_links_dir: Path | str = DEFAULT_WHEEL_DIR) -> list[str]: + """Shared ``pip wheel`` flags used by ``build_wheels.py`` and ``build_wheels_from_file.py``.""" + wheel_dir = str(find_links_dir) + return [ + "--find-links", + wheel_dir, + "--find-links", + PYPI_SIMPLE_INDEX, + "--wheel-dir", + wheel_dir, + "--no-cache-dir", + "--no-build-isolation", + *get_macos_only_binary_args(), + ] + + +def pip_wheel_invocation_args( + requirement_name: str, + find_links_dir: Path | str = DEFAULT_WHEEL_DIR, +) -> list[str]: + """``pip wheel`` flags for a single requirement (may drop ``--no-build-isolation`` on ARMv7).""" + args = list(pip_wheel_standard_args(find_links_dir)) + if is_linux_armv7_runner(): + # cffi 2.x sdists fail metadata prep under host setuptools + --no-build-isolation + # (project.license validation), including transitive cffi pulls (e.g. esptool → cryptography). + # PEP 517 isolation matches build_requirements install on ARMv7 Docker. + del requirement_name # per-requirement env still via armv7_pip_wheel_subprocess_env() + args = [arg for arg in args if arg != "--no-build-isolation"] + return args + + +def _wheel_has_manylinux_228_tag(wheel_name: str) -> bool: + """True when the wheel filename includes a ``manylinux_2_28`` platform tag.""" + return (2, 28) in manylinux_glibc_tags_in_name(wheel_name) + + +def _requirement_exact_version(req: Requirement) -> str | None: + """Return the pinned version when the requirement uses ``==``.""" + for spec in req.specifier: + if spec.operator == "==": + return str(spec.version) + return None + + +def _wheel_highest_manylinux_glibc(wheel_name: str) -> tuple[int, int] | None: + """Return the highest ``manylinux_M_N`` glibc tag embedded in a wheel filename.""" + tags = manylinux_glibc_tags_in_name(wheel_name) + return max(tags) if tags else None + + +def prune_ci_manylinux_newer_than_228( + package_name: str, + package_version: str, + *, + wheel_dir: Path | str = DEFAULT_WHEEL_DIR, +) -> int: + """Drop CI-built manylinux wheels newer than ``2_28`` after mirroring PyPI ``2_28``. + + Ubuntu 24.04 rebuilds link against newer system OpenSSL (e.g. SM4 symbols) and + auditwheel tags ``manylinux_2_34_*``. Pip prefers those over mirrored ``2_28`` + wheels when both are on the index, breaking consumers on older OpenSSL (ESP-IDF + Docker, esptool PyInstaller on focal, etc.). + """ + dest = Path(wheel_dir) + canonical = canonicalize_name(package_name) + version = str(package_version) + removed = 0 + for path in dest.glob("*.whl"): + parsed = parse_wheel_name(path.name) + if not parsed or parsed[0] != canonical or parsed[1] != version: + continue + glibc = _wheel_highest_manylinux_glibc(path.name) + if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): + print_color( + f"-- removed {path.name} (CI manylinux_{glibc[0]}_{glibc[1]}; PyPI 2_28 mirror kept)", + Fore.YELLOW, + ) + path.unlink() + removed += 1 + return removed + + +def prune_ci_manylinux_newer_than_228_when_228_mirror_present( + *, + wheel_dir: Path | str = DEFAULT_WHEEL_DIR, + package_names: frozenset[str] | None = None, +) -> int: + """Drop manylinux > ``2_28`` when a ``2_28`` wheel exists for the same dist+version. + + Used after merging matrix artifacts and after ``repair_wheels`` so CI or auditwheel + outputs cannot coexist with PyPI ``manylinux_2_28`` mirrors on the upload index. + """ + dest = Path(wheel_dir) + by_dist_version: dict[tuple[str, str], list[Path]] = {} + for path in dest.rglob("*.whl"): + parsed = parse_wheel_name(path.name) + if not parsed: + continue + if package_names is not None: + pkg_key = parsed[0].replace("-", "_") + if pkg_key not in package_names: + continue + by_dist_version.setdefault(parsed, []).append(path) + + removed = 0 + for (dist, version), paths in by_dist_version.items(): + if not any(_wheel_has_manylinux_228_tag(p.name) for p in paths): + continue + for path in paths: + glibc = _wheel_highest_manylinux_glibc(path.name) + if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): + print_color( + f"-- removed {path.name} (manylinux_{glibc[0]}_{glibc[1]}; " + f"keeping manylinux_2_28 for {dist}=={version})", + Fore.YELLOW, + ) + path.unlink() + removed += 1 + return removed + + +def should_skip_linux_auditwheel_for_pypi_mirror(wheel_name: str) -> bool: + """Skip auditwheel on PyPI ``manylinux_2_28`` mirrors (repair retags to host ``2_34``).""" + machine = platform.machine().lower() + if machine not in ("x86_64", "amd64", "aarch64", "arm64"): + return False + parsed = parse_wheel_name(wheel_name) + if not parsed: + return False + pkg_key = parsed[0].replace("-", "_") + if pkg_key not in _force_no_binary_linux_normalized(_REPO_ROOT): + return False + return _wheel_has_manylinux_228_tag(wheel_name) + + +def _mirrored_manylinux228_version(package_name: str, wheel_dir: Path) -> str | None: + """Return the version string of a ``manylinux_2_28`` wheel mirrored for ``package_name``.""" + canonical = canonicalize_name(package_name) + for path in wheel_dir.glob("*.whl"): + parsed = parse_wheel_name(path.name) + if not parsed or parsed[0] != canonical: + continue + if _wheel_has_manylinux_228_tag(path.name): + return parsed[1] + return None + + +def _prune_mirrored_manylinux228_ci_builds( + req: Requirement, + *, + wheel_dir: Path, +) -> int: + """Remove CI manylinux > ``2_28`` after mirroring PyPI ``2_28`` for ``req``.""" + mirrored_version = _mirrored_manylinux228_version(req.name, wheel_dir) + if not mirrored_version: + mirrored_version = _requirement_exact_version(req) + if not mirrored_version: + return 0 + pruned = prune_ci_manylinux_newer_than_228(req.name, mirrored_version, wheel_dir=wheel_dir) + if pruned: + print_color( + f"-- pruned {pruned} CI manylinux wheel(s) newer than 2_28 for {req.name}=={mirrored_version}", + Fore.YELLOW, + ) + return pruned + + +def mirror_pypi_manylinux228_wheel( + requirement_line: str, + *, + wheel_dir: Path | str = DEFAULT_WHEEL_DIR, +) -> bool: + """Download a ``manylinux_2_28`` wheel from PyPI for older glibc/OpenSSL consumers. + + CI rebuilds on Ubuntu 24.04 produce ``manylinux_2_34_*`` wheels (glibc >= 2.34, + newer system OpenSSL). Downstream tools (ESP-IDF Docker, esptool PyInstaller on + Ubuntu 20.04) need PyPI's ``manylinux_2_28`` wheel instead. After a successful + mirror, locally rebuilt ``manylinux`` wheels newer than ``2_28`` for the same + version are removed so pip cannot prefer the incompatible CI build. + + ARMv7 builds use piwheels and different manylinux lineage tags; no mirroring here. + """ + if platform.system() != "Linux" or is_linux_armv7_runner(): + return False + machine = platform.machine().lower() + pip_platform = { + "x86_64": "manylinux_2_28_x86_64", + "amd64": "manylinux_2_28_x86_64", + "aarch64": "manylinux_2_28_aarch64", + "arm64": "manylinux_2_28_aarch64", + }.get(machine) + if not pip_platform: + return False + + line = requirement_line.strip() + if not line: + return False + try: + req = Requirement(line) + except InvalidRequirement: + return False + + download_line = _requirement_for_pip_download(req) + + pkg_key = canonicalize_name(req.name).replace("-", "_") + if pkg_key not in _force_no_binary_linux_normalized(_REPO_ROOT): + return False + + py_major, py_minor = sys.version_info[:2] + dest = Path(wheel_dir) + dest.mkdir(exist_ok=True) + cp_abi = f"cp{py_major}{py_minor}" + abi_candidates = [cp_abi, "abi3"] if cp_abi != "abi3" else ["abi3"] + + for abi in abi_candidates: + out = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + download_line, + "--only-binary", + ":all:", + "--dest", + str(dest), + "--no-deps", + "--platform", + pip_platform, + "--python-version", + f"{py_major}.{py_minor}", + "--implementation", + "cp", + "--abi", + abi, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if out.stdout: + print(out.stdout.decode("utf-8", errors="replace")) + if out.returncode == 0: + print_color( + f"-- mirrored {pip_platform} wheel for {download_line} from PyPI (abi={abi}; older glibc consumers)", + Fore.YELLOW, + ) + _prune_mirrored_manylinux228_ci_builds(req, wheel_dir=dest) + return True + + if out.stderr: + print_color(out.stderr.decode("utf-8", errors="replace"), Fore.YELLOW) + print_color( + f"-- could not mirror {pip_platform} wheel for {download_line} from PyPI", + Fore.YELLOW, + ) + return False + + +# Backward-compatible alias (same behavior; not cryptography-specific). +mirror_pypi_manylinux228_x86_64_wheel = mirror_pypi_manylinux228_wheel + + +def pip_wheel_or_mirror_success( + requirement_line: str, + pip_returncode: int, + *, + wheel_dir: Path | str = DEFAULT_WHEEL_DIR, +) -> bool: + """True when ``pip wheel`` succeeded or a PyPI ``manylinux_2_28`` mirror was fetched.""" + if pip_returncode == 0: + mirror_pypi_manylinux228_wheel(requirement_line, wheel_dir=wheel_dir) + prune_ci_macos_newer_than_pypi_mirror(wheel_dir=wheel_dir) + return True + mirrored = mirror_pypi_manylinux228_wheel(requirement_line, wheel_dir=wheel_dir) + prune_ci_macos_newer_than_pypi_mirror(wheel_dir=wheel_dir) + return mirrored From 13048727ce150e202e596e99a236775b8b650412 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Tue, 1 Sep 2026 11:31:05 +0200 Subject: [PATCH 3/7] feat: Added more "critical" native wheels into import guard --- native_import_guard.yaml | 13 +++++++++++++ test_native_import_guard.py | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/native_import_guard.yaml b/native_import_guard.yaml index 4058015..b0fbb6e 100644 --- a/native_import_guard.yaml +++ b/native_import_guard.yaml @@ -56,3 +56,16 @@ packages: imports: - import psutil - print("psutil import OK", psutil.__version__) + # Linux --no-binary rebuilds: top-level import can succeed without loading the C extension. + - package_name: pillow + imports: + - from PIL import Image + - package_name: pyyaml + imports: + - from yaml import CSafeLoader + - package_name: brotli + imports: + - import brotli + - package_name: pycryptodome + imports: + - from Crypto.Cipher import AES diff --git a/test_native_import_guard.py b/test_native_import_guard.py index a902c5e..760c4e3 100644 --- a/test_native_import_guard.py +++ b/test_native_import_guard.py @@ -72,6 +72,17 @@ def test_yaml_uses_package_name_and_probe_unlisted(self) -> None: names = {entry.name for entry in cfg.packages} self.assertIn("cffi", names) self.assertIn("psutil", names) + self.assertIn("pillow", names) + self.assertIn("pyyaml", names) + self.assertIn("brotli", names) + self.assertIn("pycryptodome", names) + + def test_loads_deeper_native_imports(self) -> None: + guarded = native_import_guard_by_name() + self.assertEqual(guarded["pillow"].imports, ("from PIL import Image",)) + self.assertEqual(guarded["pyyaml"].imports, ("from yaml import CSafeLoader",)) + self.assertEqual(guarded["brotli"].imports, ("import brotli",)) + self.assertEqual(guarded["pycryptodome"].imports, ("from Crypto.Cipher import AES",)) def test_pure_any_wheel_name(self) -> None: self.assertTrue(is_pure_any_wheel_name("six-1.16.0-py2.py3-none-any.whl")) From b9bb09c067f89338e4198087ebe3e30f59189eba Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Tue, 1 Sep 2026 11:52:25 +0200 Subject: [PATCH 4/7] fix: Fixed the import probe approach --- README.md | 8 +- _helper_functions.py | 43 ++- build_wheels.py | 63 ++--- build_wheels_from_file.py | 58 ++--- native_import_guard.py | 92 ++++++- native_import_guard.yaml | 16 +- scripts/run_native_import_probe.py | 31 +-- ...validate_cryptography_macos_intel_wheel.sh | 3 +- test_build_wheels.py | 198 +++++++++++++- test_build_wheels_from_file.py | 19 ++ test_native_import_guard.py | 86 +++++- wheel_compat_policy.py | 244 +++++++++++++++--- 12 files changed, 696 insertions(+), 165 deletions(-) diff --git a/README.md b/README.md index e56a89a..5d16445 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ The syntax matches [`exclude_list.yaml`](./exclude_list.yaml) / [`include_list.y Top-level options: -* `probe_unlisted` — if true (default), every other **platform** wheel is probed with `import ` from the wheel’s `top_level.txt` +* `probe_unlisted` — if true (default), every other **platform** wheel is probed with one `import ` from the wheel’s `top_level.txt` (skipped when that file is missing; installs use `--no-deps`) * `skip_pure_any` — if true (default), skip `*-none-any.whl` * `skip_top_level` — names ignored when generating that default import (`test`, `tests`, `testing`) @@ -158,7 +158,7 @@ Each `packages:` entry: * `package_name` — PyPI distribution name (legacy key `name` is accepted) * `imports` — Python statement(s) to run after install (list; multiline allowed) * `skip` — if true, do not probe when the filters match -* `platform` / `python` / `version` — optional, same tokens as `exclude_list.yaml` (omitted = all) +* `platform` / `python` / `version` — optional, same tokens as `exclude_list.yaml` (omitted = all). Unmatched filters do not skip the probe; the wheel is treated as unlisted (`probe_unlisted` default import). Use `skip: true` when a probe should not run. example: @@ -171,7 +171,7 @@ packages: - import _cffi_backend ``` -This would mean: load the real C extension for `cffi` (not `import cffi` alone). The same names are referenced in [`repair_wheels.py`](./repair_wheels.py) for ARMv7 manylinux repair policy. +This would mean: load the real C extension for `cffi` (not `import cffi` alone). [`test_wheels_install.py`](./test_wheels_install.py) runs these probes after `--no-deps` install. Unlisted platform wheels — including packages whose YAML row does not match the current platform / python / version — are probed only when `top_level.txt` has a valid import name (the distribution name is not guessed). ### build_requirements.txt @@ -214,7 +214,7 @@ Upstream [dropped x86_64 macOS support](https://github.com/pyca/cryptography/com A local sdist build on GitHub `macos-15-*` runners often stamps a **higher** `macosx_*` tag (`macosx_15_0_x86_64`, …) than the official cibuildwheel tag (`macosx_10_9_x86_64`, `macosx_11_0_arm64`). Pip prefers the highest compatible tag, so the extra-index CI copy wins over PyPI. For **psutil 7.2.2** that locally compiled `_psutil_osx.abi3.so` SIGABRT’d ESP-IDF `export.sh` / `test_pytest_macos`. -macOS `pip wheel` therefore passes **`--only-binary :all:`** (Intel **cryptography** still uses `--no-binary cryptography` for the OpenSSL 4 rebuild above). [`repair_wheels.py`](./repair_wheels.py) **skips delocate** except that Intel cryptography rebuild, and **prunes** any higher `macosx_*` sibling for the same distribution/version/arch so it cannot be uploaded. After install, every platform wheel is import-probed. Leftover higher-tag objects already on the extra-index (for example [psutil 7.2.2](https://dl.espressif.com/pypi/psutil/)) must be deleted from the bucket; a later upload only overwrites the same filename. +macOS `pip wheel` therefore passes **`--only-binary :all:`** (Intel **cryptography** still uses `--no-binary cryptography` for the OpenSSL 4 rebuild above). [`repair_wheels.py`](./repair_wheels.py) **skips delocate** except that Intel cryptography rebuild, and **prunes** any higher `macosx_*` sibling that competes for the same distribution, version, CPython (python+ABI family), and machine family (`x86_64` / `intel` / `universal2`, or `arm64` / `universal2`) so it cannot be uploaded. After install, every platform wheel is import-probed. Leftover higher-tag objects already on the extra-index (for example [psutil 7.2.2](https://dl.espressif.com/pypi/psutil/)) must be deleted from the bucket; a later upload only overwrites the same filename. ## Activity Diagram The main file is `build-wheels-platforms.yml` which is scheduled to run periodically to build Python wheels for any requirement of all [ESP-IDF]-supported versions. diff --git a/_helper_functions.py b/_helper_functions.py index 0665803..51f52d3 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -142,6 +142,28 @@ def _macos_x86_64_cryptography_versions_in_find_links( return versions +# cryptography 49+ dropped official macOS x86_64 wheels and is built here against OpenSSL 4. +# Older pins still have PyPI Intel wheels; their openssl-sys does not support OpenSSL 4. +_CRYPTOGRAPHY_MACOS_INTEL_OPENSSL4_SDIST_MIN = Version("49") + + +def cryptography_pin_needs_macos_intel_openssl4_sdist(req: Requirement) -> bool: + """True when this pin can resolve to cryptography 49+ (OpenSSL 4 Intel sdist).""" + spec = req.specifier + if not spec: + return True + if _CRYPTOGRAPHY_MACOS_INTEL_OPENSSL4_SDIST_MIN in spec: + return True + for item in spec: + if item.operator in (">=", ">", "==", "==="): + try: + if Version(str(item.version)) >= _CRYPTOGRAPHY_MACOS_INTEL_OPENSSL4_SDIST_MIN: + return True + except InvalidVersion: + continue + return False + + def macos_intel_cryptography_rebuild_instead_of_find_links_skip( req: Requirement, find_links_reason: str, @@ -151,12 +173,15 @@ def macos_intel_cryptography_rebuild_instead_of_find_links_skip( Official 49+ wheels dropped macOS x86_64. A matching older Intel wheel (or a Linux/Windows wheel seeded into find-links) must not skip the OpenSSL 4 build - for a newer PyPI release. Obsolete upper-bound pins still skip. + for a newer PyPI release. Pins that cannot reach 49+ (``<43``, ``<45``, + ``<49``) keep the PyPI Intel wheel: those sdists fail against OpenSSL 4. """ if get_current_platform() != "macos_x86_64": return False if canonicalize_name(req.name) != canonicalize_name("cryptography"): return False + if not cryptography_pin_needs_macos_intel_openssl4_sdist(req): + return False if "none match" in find_links_reason or "newer than obsolete pin" in find_links_reason: return False if "already has" not in find_links_reason: @@ -170,6 +195,8 @@ def macos_intel_cryptography_rebuild_instead_of_find_links_skip( latest_ver = Version(latest[0]) except InvalidVersion: return True + if latest_ver < _CRYPTOGRAPHY_MACOS_INTEL_OPENSSL4_SDIST_MIN: + return False local = _macos_x86_64_cryptography_versions_in_find_links(find_links_dir) if local and max(local) >= latest_ver: return False @@ -518,13 +545,19 @@ def force_interpreter_skip_package(canonical_dist_name: str) -> bool: def get_cryptography_macos_intel_pip_wheel_args(requirement_name: str) -> list[str]: - """``--no-binary cryptography`` on macOS Intel (OpenSSL 4 sdist build in CI).""" + """``--no-binary cryptography`` on macOS Intel for 49+ (OpenSSL 4 sdist; no PyPI x86_64).""" if get_current_platform() != "macos_x86_64": return [] - match = re.match(r"^([a-zA-Z0-9_.-]+)", str(requirement_name).strip()) - if not match: + line = str(requirement_name).strip() + if not line: + return [] + try: + req = Requirement(line) + except InvalidRequirement: + return [] + if canonicalize_name(req.name) != canonicalize_name("cryptography"): return [] - if canonicalize_name(match.group(1)) != canonicalize_name("cryptography"): + if not cryptography_pin_needs_macos_intel_openssl4_sdist(req): return [] return ["--no-binary", "cryptography"] diff --git a/build_wheels.py b/build_wheels.py index f1062a3..1bc66a3 100644 --- a/build_wheels.py +++ b/build_wheels.py @@ -6,8 +6,6 @@ import json import os import re -import subprocess -import sys from typing import Dict from typing import List @@ -43,8 +41,9 @@ from emit_sdist_requirements import SDIST_REQUIREMENTS_FILE from emit_sdist_requirements import compute_sdist_requirements from emit_sdist_requirements import write_sdist_requirements_file -from wheel_compat_policy import pip_wheel_invocation_args from wheel_compat_policy import pip_wheel_or_mirror_success +from wheel_compat_policy import prune_ci_macos_newer_than_pypi_mirror +from wheel_compat_policy import run_pip_wheel from yaml_list_adapter import YAMLListAdapter # GLOBAL VARIABLES @@ -348,6 +347,8 @@ def build_wheels(requirements: set, local_links: bool = True) -> dict: else: print_color(f"-- skip {requirement} ({reason})", Fore.YELLOW) skipped_wheels += 1 + # Find-links skip never runs pip_wheel_or_mirror_success. + prune_ci_macos_newer_than_pypi_mirror(wheel_dir=dir) continue skip, reason = bounded_pin_without_find_links_skip(requirement, dir) if skip: @@ -361,40 +362,28 @@ def build_wheels(requirements: set, local_links: bool = True) -> dict: f"-- removed {removed} find-links wheel(s) for {requirement.name} (ARMv7 sdist rebuild)", Fore.YELLOW, ) - # non classic requirement wheel build + # non classic requirement wheel build (e.g. '--only-binary cryptography' from IDF constraints) if non_classic_requirement: pattern = re.compile(r"(--[^ ]*)(.*)") match = pattern.search(non_classic_requirement[0]) - if match: - argument = match.group(1).strip() - arg_param = match.group(2).strip() - if arg_param in requirement.name: - out = subprocess.run( - [ - f"{sys.executable}", - "-m", - "pip", - "wheel", - f"{requirement}", - "--find-links", - f"{dir}", - "--find-links", - "https://pypi.org/simple/", - "--wheel-dir", - f"{dir}", - "--no-cache-dir", - "--no-build-isolation", - f"{argument}", - f"{arg_param}", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + argument = match.group(1).strip() if match else "" + arg_param = match.group(2).strip() if match else "" + if arg_param and arg_param in requirement.name: + extra = [argument, arg_param] if argument else [] + out = run_pip_wheel( + str(requirement), + extra, + find_links_dir=dir, env=armv7_pip_wheel_subprocess_env(str(requirement)), ) print(out.stdout.decode("utf-8", errors="replace")) if out.stderr: print_color(out.stderr.decode("utf-8", errors="replace"), Fore.RED) non_classic_requirement.remove(non_classic_requirement[0]) + if pip_wheel_or_mirror_success(str(requirement), out.returncode, wheel_dir=dir): + succeeded_wheels += 1 + else: + failed_wheels += 1 continue # requirement wheel build @@ -403,20 +392,10 @@ def build_wheels(requirements: set, local_links: bool = True) -> dict: crypto_intel_args = get_cryptography_macos_intel_pip_wheel_args(str(requirement)) extra_pip_args = get_pip_wheel_extra_args(str(requirement)) - out = subprocess.run( - [ - f"{sys.executable}", - "-m", - "pip", - "wheel", - f"{requirement}", - ] - + pip_wheel_invocation_args(str(requirement), dir) - + no_binary_args - + crypto_intel_args - + extra_pip_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + out = run_pip_wheel( + str(requirement), + no_binary_args + crypto_intel_args + extra_pip_args, + find_links_dir=dir, env=armv7_pip_wheel_subprocess_env(str(requirement)), ) diff --git a/build_wheels_from_file.py b/build_wheels_from_file.py index 9159de8..e80154d 100644 --- a/build_wheels_from_file.py +++ b/build_wheels_from_file.py @@ -8,8 +8,6 @@ import argparse import os import platform -import subprocess -import sys from pathlib import Path @@ -32,12 +30,19 @@ from _helper_functions import print_color from _helper_functions import pypi_requires_python_preflight_skip from _helper_functions import remove_find_links_wheels_for_package -from wheel_compat_policy import pip_wheel_invocation_args from wheel_compat_policy import pip_wheel_or_mirror_success +from wheel_compat_policy import prune_ci_macos_newer_than_pypi_mirror +from wheel_compat_policy import run_pip_wheel def _force_interpreter_no_binary_args(requirement_line: str) -> list[str]: - """Return pip --no-binary for this package so pip cannot reuse e.g. cp311-abi3 wheels on 3.13.""" + """Return pip --no-binary for this package so pip cannot reuse e.g. cp311-abi3 wheels on 3.13. + + Never used on macOS: ``--no-binary`` would override ``--only-binary :all:`` and + sdist-compile a higher ``macosx_*`` tag (the psutil 7.2.2 SIGABRT path). + """ + if platform.system() != "Linux": + return [] line = requirement_line.strip() if not line: return [] @@ -51,8 +56,8 @@ def _force_interpreter_no_binary_args(requirement_line: str) -> list[str]: def _apply_force_interpreter_binary(cli_flag: bool) -> bool: - """Linux/macOS only: forcing sdist builds for cryptography etc. is unreliable on Windows CI.""" - return cli_flag and platform.system() != "Windows" + """Linux only. Windows source builds are unreliable; macOS prefers PyPI ``--only-binary``.""" + return bool(cli_flag) and platform.system() == "Linux" def _pypi_preflight_skip_line(requirement_line: str) -> bool: @@ -89,6 +94,8 @@ def _dependent_requirement_skip_line(requirement_line: str, *, force_interpreter ) else: print_color(f"-- skip {requirement_line} ({reason})", Fore.YELLOW) + # Find-links skip never runs pip_wheel_or_mirror_success. + prune_ci_macos_newer_than_pypi_mirror() return True skip, reason = bounded_pin_without_find_links_skip(req) if skip: @@ -126,7 +133,8 @@ def _dependent_requirement_skip_line(requirement_line: str, *, force_interpreter help=( "For each requirement, pass --no-binary so pip builds a wheel for the current " "interpreter instead of reusing a compatible abi3 / older cpXY wheel from --find-links. " - "Ignored on Windows (source builds for e.g. cryptography are not used in CI there). " + "Ignored on Windows (source builds for e.g. cryptography are not used in CI there) " + "and on macOS (``--only-binary :all:`` keeps official macosx tags). " "Some packages are always skipped (e.g. cryptography, pydantic-core, argon2-cffi-bindings, PyNaCl, PyObjC)." ), ) @@ -175,21 +183,9 @@ def _dependent_requirement_skip_line(requirement_line: str, *, force_interpreter extra_pip_args = get_pip_wheel_extra_args(requirement) force_interpreter_args = _force_interpreter_no_binary_args(requirement) if force_interpreter else [] - out = subprocess.run( - [ - f"{sys.executable}", - "-m", - "pip", - "wheel", - requirement, - ] - + pip_wheel_invocation_args(requirement) - + no_binary_args - + crypto_intel_args - + extra_pip_args - + force_interpreter_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + out = run_pip_wheel( + requirement, + no_binary_args + crypto_intel_args + extra_pip_args + force_interpreter_args, env=armv7_pip_wheel_subprocess_env(requirement), ) @@ -238,21 +234,9 @@ def _dependent_requirement_skip_line(requirement_line: str, *, force_interpreter extra_pip_args = get_pip_wheel_extra_args(requirement) force_interpreter_args = _force_interpreter_no_binary_args(requirement) if force_interpreter else [] - out = subprocess.run( - [ - f"{sys.executable}", - "-m", - "pip", - "wheel", - f"{requirement}", - ] - + pip_wheel_invocation_args(requirement) - + no_binary_args - + crypto_intel_args - + extra_pip_args - + force_interpreter_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + out = run_pip_wheel( + requirement, + no_binary_args + crypto_intel_args + extra_pip_args + force_interpreter_args, env=armv7_pip_wheel_subprocess_env(requirement), ) diff --git a/native_import_guard.py b/native_import_guard.py index 170c68a..32022b6 100644 --- a/native_import_guard.py +++ b/native_import_guard.py @@ -16,6 +16,7 @@ import yaml +from packaging.specifiers import InvalidSpecifier from packaging.specifiers import SpecifierSet from packaging.utils import canonicalize_name from packaging.version import InvalidVersion @@ -48,11 +49,14 @@ def _version_matches_yaml_spec(raw: Any, version: Version) -> bool: items = _yaml_scalar_or_list(raw) if not items: return True - specs = [SpecifierSet(item) for item in items] - operators = [spec.operator for spec_set in specs for spec in spec_set] - if operators and all(op in ("==", "===") for op in operators): - return any(version in spec_set for spec_set in specs) - return version in SpecifierSet(",".join(items)) + try: + specs = [SpecifierSet(item) for item in items] + operators = [spec.operator for spec_set in specs for spec in spec_set] + if operators and all(op in ("==", "===") for op in operators): + return any(version in spec_set for spec_set in specs) + return version in SpecifierSet(",".join(items)) + except InvalidSpecifier as exc: + raise ValueError(f"invalid native_import_guard.yaml version/python specifier {items!r}: {exc}") from exc @dataclass(frozen=True) @@ -72,6 +76,7 @@ class NativeImportGuardConfig: probe_unlisted: bool skip_pure_any: bool skip_top_level: frozenset[str] + skip_name_prefixes: frozenset[str] packages: tuple[NativeImportGuardEntry, ...] @@ -115,6 +120,7 @@ def load_native_import_guard(repo_root: Path | None = None) -> NativeImportGuard path = root / NATIVE_IMPORT_GUARD_PATH data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} skip_top = _yaml_scalar_or_list(data.get("skip_top_level")) + skip_prefixes = [canonicalize_name(item) for item in _yaml_scalar_or_list(data.get("skip_name_prefixes"))] packages: list[NativeImportGuardEntry] = [] for entry in data.get("packages") or []: if not isinstance(entry, dict): @@ -141,6 +147,7 @@ def load_native_import_guard(repo_root: Path | None = None) -> NativeImportGuard probe_unlisted=bool(data.get("probe_unlisted", True)), skip_pure_any=bool(data.get("skip_pure_any", True)), skip_top_level=frozenset(skip_top) if skip_top else DEFAULT_SKIP_TOP_LEVEL_IMPORTS, + skip_name_prefixes=frozenset(skip_prefixes), packages=tuple(packages), ) _NATIVE_IMPORT_GUARD_CACHE[root] = config @@ -186,18 +193,35 @@ def wheel_top_level_modules( return tuple(names) +def _is_importable_module_name(name: str) -> bool: + """True when ``name`` is safe to pass to ``import`` (identifier or dotted identifiers).""" + return bool(name) and all(part.isidentifier() for part in name.split(".")) + + def default_native_import_statements( wheel_path: Path | str, skip_top_level: frozenset[str] | None = None, ) -> tuple[str, ...]: - """``import `` statements for a native wheel without custom YAML imports.""" - tops = wheel_top_level_modules(wheel_path, skip_top_level) - if tops: - return tuple(f"import {mod}" for mod in tops) - parsed = parse_wheel_name(Path(wheel_path).name) - if not parsed: + """One ``import `` for a native wheel without custom YAML imports. + + Installs in ``test_wheels_install.py`` use ``--no-deps``. Guessing the import + from the distribution name (when ``top_level.txt`` is missing) or importing + every top-level name often fails. Prefer the public name that matches the + distribution; otherwise the first public identifier. Skip the probe if none. + """ + tops = [name for name in wheel_top_level_modules(wheel_path, skip_top_level) if _is_importable_module_name(name)] + if not tops: return () - return (f"import {parsed[0].replace('-', '_')}",) + parsed = parse_wheel_name(Path(wheel_path).name) + dist_mod = parsed[0].replace("-", "_") if parsed else "" + public = [name for name in tops if not name.startswith("_")] + if dist_mod in tops: + chosen = dist_mod + elif public: + chosen = public[0] + else: + chosen = tops[0] + return (f"import {chosen}",) def resolve_native_import_statements( @@ -210,8 +234,10 @@ def resolve_native_import_statements( """Import statements for a wheel, or ``None`` to skip the probe. First matching YAML row (name + platform / python / version) wins: ``skip: true`` - skips; ``imports`` override the default. Unlisted platform wheels use - ``top_level.txt`` when ``probe_unlisted`` is true. + skips; ``imports`` override the default. A row that does not match its filters + is ignored (the package is treated as unlisted). Unlisted platform wheels use + ``top_level.txt`` when ``probe_unlisted`` is true. Use ``skip: true`` when a + probe should not run. """ path = Path(wheel_path) parsed = parse_wheel_name(path.name) @@ -237,7 +263,45 @@ def resolve_native_import_statements( if entry.imports: return entry.imports break + if any(parsed[0] == prefix or parsed[0].startswith(f"{prefix}-") for prefix in cfg.skip_name_prefixes): + return None if not cfg.probe_unlisted: return None stmts = default_native_import_statements(path, cfg.skip_top_level) return stmts or None + + +def package_import_statements( + package_name: str, + *, + config: NativeImportGuardConfig | None = None, + repo_root: Path | None = None, + current_platform: str | None = None, + python_version: str | None = None, + wheel_version: str | None = None, +) -> tuple[str, ...] | None: + """YAML ``imports`` for a distribution name after platform / python / version filters. + + Used by ``scripts/run_native_import_probe.py`` (no wheel path). ``skip`` and + unmatched filters return ``None``; unlisted names are not default-probed. + """ + cfg = config if config is not None else load_native_import_guard(repo_root) + plat = current_platform if current_platform is not None else get_current_platform() + py_ver = python_version if python_version is not None else f"{sys.version_info.major}.{sys.version_info.minor}" + canonical = canonicalize_name(package_name) + for entry in cfg.packages: + if entry.name != canonical: + continue + if not _native_import_entry_matches( + entry, + current_platform=plat, + wheel_version=wheel_version, + python_version=py_ver, + ): + continue + if entry.skip: + return None + if entry.imports: + return entry.imports + break + return None diff --git a/native_import_guard.yaml b/native_import_guard.yaml index b0fbb6e..0f3c8ee 100644 --- a/native_import_guard.yaml +++ b/native_import_guard.yaml @@ -5,9 +5,12 @@ # run imports — or skip — and fail CI if import crashes or errors." # # Top-level: -# probe_unlisted — probe platform wheels not listed below via import (default true) +# probe_unlisted — probe unlisted platform wheels via one import from top_level.txt +# (default true; skipped when top_level.txt is missing — installs use --no-deps) # skip_pure_any — skip *-none-any.whl (default true) -# skip_top_level — top_level.txt names ignored when generating default imports +# skip_top_level — top_level.txt names ignored when generating default imports +# skip_name_prefixes — skip unlisted packages whose canonical name is this token +# or starts with ``-`` (e.g. pyobjc → pyobjc-framework-*) # # Each packages: entry (same filter tokens as exclude_list.yaml / include_list.yaml): # package_name — PyPI distribution name (legacy key ``name`` is also accepted) @@ -17,6 +20,8 @@ # macos_x86_64 | macos_arm64 (string or list). Omitted = all platforms. # python — python_version specifier(s), e.g. '>=3.10' or ['==3.8', '==3.9'] # version — package version specifier(s), e.g. '>=49' or ['>=1.0', '<2'] +# Unmatched filters treat the package as unlisted (probe_unlisted default import). +# Use skip: true when a probe should not run. # # ``import cffi`` alone is not enough — a broken auditwheel manylinux tag can segfault only in _cffi_backend. @@ -26,8 +31,15 @@ skip_top_level: - test - tests - testing +# Framework wheels import sibling PyObjC packages; installs use --no-deps. +skip_name_prefixes: + - pyobjc packages: + # Needs libdbus; ARMv7 test images fail ``import dbus`` even when the wheel is valid. + - package_name: dbus-python + skip: true + platform: linux_armv7 - package_name: cffi imports: - import _cffi_backend diff --git a/scripts/run_native_import_probe.py b/scripts/run_native_import_probe.py index 7d51bdd..8a5881d 100644 --- a/scripts/run_native_import_probe.py +++ b/scripts/run_native_import_probe.py @@ -9,7 +9,6 @@ from __future__ import annotations import argparse -import subprocess import sys from pathlib import Path @@ -19,19 +18,17 @@ sys.path.insert(0, str(_ROOT)) -def _guard_import_blocks(package_names: list[str]) -> dict[str, str]: - """Return canonical name → import probe source for requested packages.""" +def _guard_import_blocks(package_names: list[str]) -> dict[str, tuple[str, ...]]: + """Return canonical name → import probe statements for requested packages.""" from packaging.utils import canonicalize_name - from native_import_guard import load_native_import_guard + from native_import_guard import package_import_statements - wanted = {canonicalize_name(name) for name in package_names} - blocks: dict[str, str] = {} - for entry in load_native_import_guard(_ROOT).packages: - if entry.name not in wanted or not entry.imports: - continue - if entry.name not in blocks: - blocks[entry.name] = "\n".join(entry.imports) + blocks: dict[str, tuple[str, ...]] = {} + for name in package_names: + stmts = package_import_statements(name, repo_root=_ROOT) + if stmts: + blocks[canonicalize_name(name)] = stmts return blocks @@ -49,15 +46,19 @@ def main() -> int: failed = 0 from packaging.utils import canonicalize_name + from native_import_probe import run_import_probes + for name in args.packages: - code = blocks.get(canonicalize_name(name)) - if code is None: + stmts = blocks.get(canonicalize_name(name)) + if stmts is None: print(f"ERROR: no guard entry for {name!r}", file=sys.stderr) failed += 1 continue print(f"--- probe {name} ---") - result = subprocess.run([sys.executable, "-c", code], check=False) - if result.returncode != 0: + ok, msg = run_import_probes(stmts) + if msg: + print(msg) + if not ok: print(f"FAILED: {name}", file=sys.stderr) failed += 1 else: diff --git a/scripts/validate_cryptography_macos_intel_wheel.sh b/scripts/validate_cryptography_macos_intel_wheel.sh index 3c61321..34ef383 100644 --- a/scripts/validate_cryptography_macos_intel_wheel.sh +++ b/scripts/validate_cryptography_macos_intel_wheel.sh @@ -153,7 +153,8 @@ while IFS=$'\t' read -r WHEEL WHEEL_PYTHON; do # shellcheck disable=SC1091 source "${WORKDIR}/venv/bin/activate" pip install --upgrade pip - pip install pyyaml packaging + # Probe loads native_import_guard -> _helper_functions (colorama, packaging, PyYAML). + pip install -r "${ROOT}/build_requirements.txt" pip install --no-index --find-links "${WHEELS_DIR}" "${WHEEL}" python "${ROOT}/scripts/run_native_import_probe.py" cryptography deactivate 2>/dev/null || true diff --git a/test_build_wheels.py b/test_build_wheels.py index c552a1b..9535248 100644 --- a/test_build_wheels.py +++ b/test_build_wheels.py @@ -42,8 +42,10 @@ from wheel_compat_policy import prune_ci_macos_newer_than_pypi_mirror from wheel_compat_policy import prune_ci_manylinux_newer_than_228 from wheel_compat_policy import prune_ci_manylinux_newer_than_228_when_228_mirror_present +from wheel_compat_policy import run_pip_wheel from wheel_compat_policy import should_skip_linux_auditwheel_for_pypi_mirror from wheel_compat_policy import should_skip_macos_delocate_for_pypi_mirror +from wheel_compat_policy import without_only_binary_all from yaml_list_adapter import YAMLListAdapter @@ -408,21 +410,21 @@ class TestPipWheelInvocationArgs(unittest.TestCase): def test_armv7_cffi_uses_build_isolation(self, _armv7: object) -> None: from wheel_compat_policy import pip_wheel_invocation_args - args = pip_wheel_invocation_args("cffi>=1.15.0") + args = pip_wheel_invocation_args() self.assertNotIn("--no-build-isolation", args) @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=True) def test_armv7_any_package_uses_build_isolation(self, _armv7: object) -> None: from wheel_compat_policy import pip_wheel_invocation_args - args = pip_wheel_invocation_args("esptool~=4.9") + args = pip_wheel_invocation_args() self.assertNotIn("--no-build-isolation", args) @patch("wheel_compat_policy.is_linux_armv7_runner", return_value=False) def test_non_armv7_keeps_no_build_isolation(self, _armv7: object) -> None: from wheel_compat_policy import pip_wheel_invocation_args - args = pip_wheel_invocation_args("cffi>=1.15.0") + args = pip_wheel_invocation_args() self.assertIn("--no-build-isolation", args) @patch("wheel_compat_policy.platform.system", return_value="Darwin") @@ -430,7 +432,7 @@ def test_non_armv7_keeps_no_build_isolation(self, _armv7: object) -> None: def test_macos_pip_wheel_only_binary_all(self, _armv7: object, _sys: object) -> None: from wheel_compat_policy import pip_wheel_invocation_args - args = pip_wheel_invocation_args("psutil") + args = pip_wheel_invocation_args() only_at = args.index("--only-binary") self.assertEqual(args[only_at + 1], ":all:") @@ -493,6 +495,105 @@ def test_prune_noop_when_only_one_tag(self) -> None: self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 0) self.assertTrue(only.exists()) + def test_prune_does_not_drop_other_interpreter_exclusive_abi(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep_311 = links / "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl" + keep_312 = links / "cffi-1.17.1-cp312-cp312-macosx_15_0_x86_64.whl" + keep_311.write_bytes(b"pypi") + keep_312.write_bytes(b"ci") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 0) + self.assertTrue(keep_311.exists()) + self.assertTrue(keep_312.exists()) + + @patch("wheel_compat_policy.get_current_platform", return_value="macos_x86_64") + def test_prune_keeps_intel_cryptography_rebuild(self, _plat: object) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + pypi = links / "cryptography-47.0.0-cp38-abi3-macosx_10_9_x86_64.whl" + rebuild = links / "cryptography-47.0.0-cp38-abi3-macosx_10_12_x86_64.whl" + pypi.write_bytes(b"pypi") + rebuild.write_bytes(b"openssl4") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 0) + self.assertTrue(pypi.exists()) + self.assertTrue(rebuild.exists()) + + @patch("wheel_compat_policy.get_current_platform", return_value="macos_arm64") + def test_prune_still_drops_cryptography_on_arm(self, _plat: object) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep = links / "cryptography-47.0.0-cp38-abi3-macosx_10_9_x86_64.whl" + drop = links / "cryptography-47.0.0-cp38-abi3-macosx_15_0_x86_64.whl" + keep.write_bytes(b"pypi") + drop.write_bytes(b"ci") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 1) + self.assertTrue(keep.exists()) + self.assertFalse(drop.exists()) + + def test_prune_abi3_still_drops_competing_exclusive_higher_tag(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep = links / "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl" + drop = links / "psutil-7.2.2-cp311-cp311-macosx_15_0_x86_64.whl" + keep.write_bytes(b"pypi") + drop.write_bytes(b"ci") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 1) + self.assertTrue(keep.exists()) + self.assertFalse(drop.exists()) + + def test_prune_x86_64_loses_to_lower_universal2_sibling(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep = links / "cffi-1.17.1-cp311-cp311-macosx_11_0_universal2.whl" + drop = links / "cffi-1.17.1-cp311-cp311-macosx_15_0_x86_64.whl" + keep.write_bytes(b"pypi") + drop.write_bytes(b"ci") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 1) + self.assertTrue(keep.exists()) + self.assertFalse(drop.exists()) + + def test_cpython_ranges_parse_three_digit_tags(self) -> None: + from wheel_compat_policy import _wheel_cpython_ranges + from wheel_compat_policy import _wheels_compete_for_same_cpython + + self.assertEqual( + _wheel_cpython_ranges("cryptography-49.0.0-cp311-abi3-macosx_10_12_x86_64.whl"), + [((3, 11), None)], + ) + self.assertEqual( + _wheel_cpython_ranges("cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl"), + [((3, 10), (3, 10))], + ) + self.assertTrue( + _wheels_compete_for_same_cpython( + "psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", + "psutil-7.2.2-cp311-cp311-macosx_15_0_x86_64.whl", + ) + ) + self.assertTrue( + _wheels_compete_for_same_cpython( + "cryptography-49.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", + "cryptography-49.0.0-cp312-abi3-manylinux_2_34_x86_64.whl", + ) + ) + self.assertFalse( + _wheels_compete_for_same_cpython( + "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", + "cffi-1.17.1-cp312-cp312-macosx_15_0_x86_64.whl", + ) + ) + + def test_prune_intel_and_x86_64_compete(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep = links / "psutil-7.2.2-cp36-abi3-macosx_10_9_intel.whl" + drop = links / "psutil-7.2.2-cp36-abi3-macosx_15_0_x86_64.whl" + keep.write_bytes(b"pypi") + drop.write_bytes(b"ci") + self.assertEqual(prune_ci_macos_newer_than_pypi_mirror(wheel_dir=links), 1) + self.assertTrue(keep.exists()) + self.assertFalse(drop.exists()) + class TestParseWheelName(unittest.TestCase): """Test the parse_wheel_name function from _helper_functions.py.""" @@ -881,6 +982,36 @@ def test_armv7_pip_wheel_env_leaves_global_for_other_packages(self, _mock_armv7) self.assertEqual(env["PIP_NO_BINARY"], "cffi,argon2-cffi-bindings") +class TestRunPipWheelPreferBinary(unittest.TestCase): + def test_without_only_binary_all_strips_flag(self) -> None: + stripped = without_only_binary_all(["pip", "wheel", "pkg", "--only-binary", ":all:", "--no-deps"]) + self.assertEqual(stripped, ["pip", "wheel", "pkg", "--no-deps"]) + self.assertIsNone(without_only_binary_all(["pip", "wheel", "pkg"])) + + @patch("wheel_compat_policy.pip_wheel_invocation_args", return_value=["--only-binary", ":all:"]) + @patch("wheel_compat_policy.subprocess.run") + def test_retries_without_only_binary_when_first_pip_fails(self, mock_run, _args) -> None: + fail = type("R", (), {"returncode": 1, "stdout": b"", "stderr": b"no binary"})() + ok = type("R", (), {"returncode": 0, "stdout": b"sdist ok", "stderr": b""})() + mock_run.side_effect = [fail, ok] + out = run_pip_wheel("sphinx_selective_exclude==1.0.3") + self.assertEqual(out.returncode, 0) + self.assertEqual(mock_run.call_count, 2) + first_cmd = mock_run.call_args_list[0][0][0] + retry_cmd = mock_run.call_args_list[1][0][0] + self.assertIn("--only-binary", first_cmd) + self.assertNotIn("--only-binary", retry_cmd) + + @patch("wheel_compat_policy.pip_wheel_invocation_args", return_value=["--find-links", "."]) + @patch("wheel_compat_policy.subprocess.run") + def test_no_retry_when_only_binary_absent(self, mock_run, _args) -> None: + fail = type("R", (), {"returncode": 1, "stdout": b"", "stderr": b"fail"})() + mock_run.return_value = fail + out = run_pip_wheel("cffi") + self.assertEqual(out.returncode, 1) + self.assertEqual(mock_run.call_count, 1) + + class TestPipWheelOrMirrorSuccess(unittest.TestCase): @patch("wheel_compat_policy.mirror_pypi_manylinux228_wheel", return_value=True) def test_pip_success_still_mirrors_and_returns_true(self, mock_mirror): @@ -987,6 +1118,30 @@ def test_prune_when_228_mirror_present_drops_newer_sibling(self): self.assertTrue(compound.exists()) self.assertFalse(ci_only.exists()) + def test_prune_when_228_mirror_keeps_other_interpreter_exclusive_abi(self): + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep_228 = links / "cffi-1.17.1-cp311-cp311-manylinux_2_28_x86_64.whl" + keep_234 = links / "cffi-1.17.1-cp312-cp312-manylinux_2_34_x86_64.whl" + keep_228.write_bytes(b"") + keep_234.write_bytes(b"") + removed = prune_ci_manylinux_newer_than_228_when_228_mirror_present(wheel_dir=links) + self.assertEqual(removed, 0) + self.assertTrue(keep_228.exists()) + self.assertTrue(keep_234.exists()) + + def test_prune_named_version_keeps_other_interpreter_exclusive_abi(self): + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + keep_228 = links / "cffi-1.17.1-cp311-cp311-manylinux_2_28_x86_64.whl" + keep_234 = links / "cffi-1.17.1-cp312-cp312-manylinux_2_34_x86_64.whl" + keep_228.write_bytes(b"") + keep_234.write_bytes(b"") + removed = prune_ci_manylinux_newer_than_228("cffi", "1.17.1", wheel_dir=links) + self.assertEqual(removed, 0) + self.assertTrue(keep_228.exists()) + self.assertTrue(keep_234.exists()) + class TestSkipAuditwheelForPypiMirror(unittest.TestCase): @patch("platform.machine", return_value="x86_64") @@ -1017,6 +1172,15 @@ def tearDown(self): def test_no_binary_for_cryptography_on_intel(self): args = get_cryptography_macos_intel_pip_wheel_args("cryptography>=49") self.assertEqual(args, ["--no-binary", "cryptography"]) + self.assertEqual(get_cryptography_macos_intel_pip_wheel_args("cryptography"), ["--no-binary", "cryptography"]) + self.assertEqual( + get_cryptography_macos_intel_pip_wheel_args("cryptography>=50"), ["--no-binary", "cryptography"] + ) + + def test_no_binary_skipped_for_pre_49_cryptography_pins(self): + self.assertEqual(get_cryptography_macos_intel_pip_wheel_args("cryptography<43,>=2.1.4"), []) + self.assertEqual(get_cryptography_macos_intel_pip_wheel_args("cryptography<45,>=2.1.4"), []) + self.assertEqual(get_cryptography_macos_intel_pip_wheel_args("cryptography<49,>=2.1.4"), []) def test_no_binary_only_for_cryptography(self): self.assertEqual(get_cryptography_macos_intel_pip_wheel_args("requests"), []) @@ -1080,6 +1244,32 @@ def test_still_skip_obsolete_cryptography_pin(self, _mock_plat): ) ) + @patch("_helper_functions.matching_release_version_strings", return_value=["42.0.8"]) + @patch("_helper_functions.get_current_platform", return_value="macos_x86_64") + def test_do_not_rebuild_pre_49_pin_behind_pypi(self, _mock_plat, _mock_pypi): + with tempfile.TemporaryDirectory() as tmp: + links = Path(tmp) + (links / "cryptography-41.0.7-cp38-abi3-macosx_10_12_x86_64.whl").write_bytes(b"") + req = Requirement("cryptography<43,>=2.1.4") + self.assertFalse( + macos_intel_cryptography_rebuild_instead_of_find_links_skip( + req, + "find-links already has cryptography 41.0.7 matching <43,>=2.1.4", + links, + ) + ) + + @patch("_helper_functions.matching_release_version_strings", return_value=["44.0.3"]) + @patch("_helper_functions.get_current_platform", return_value="macos_x86_64") + def test_do_not_rebuild_cryptography_lt_45_behind_pypi(self, _mock_plat, _mock_pypi): + req = Requirement("cryptography<45,>=2.1.4") + self.assertFalse( + macos_intel_cryptography_rebuild_instead_of_find_links_skip( + req, + "find-links already has cryptography 41.0.7 matching <45,>=2.1.4", + ) + ) + @patch("_helper_functions.matching_release_version_strings", return_value=["50.0.1"]) @patch("_helper_functions.get_current_platform", return_value="linux_x86_64") def test_linux_does_not_rebuild_for_older_find_links(self, _mock_plat, _mock_pypi): diff --git a/test_build_wheels_from_file.py b/test_build_wheels_from_file.py index a0763e1..41f6b28 100644 --- a/test_build_wheels_from_file.py +++ b/test_build_wheels_from_file.py @@ -34,6 +34,7 @@ def test_force_interpreter_bypasses_find_links_skip_for_normal_packages( bwf._dependent_requirement_skip_line("requests>=2.28.0", force_interpreter=True), ) + @patch.object(bwf, "prune_ci_macos_newer_than_pypi_mirror") @patch.object(bwf, "bounded_pin_without_find_links_skip", return_value=(False, "")) @patch.object(bwf, "_pypi_preflight_skip_line", return_value=False) @patch.object( @@ -48,11 +49,13 @@ def test_force_interpreter_still_applies_find_links_skip_for_cryptography( _mock_find_links, _mock_pypi, _mock_armv7_pin, + _mock_prune, ) -> None: self.assertTrue( bwf._dependent_requirement_skip_line("cryptography>=2.1.4", force_interpreter=True), ) + @patch.object(bwf, "prune_ci_macos_newer_than_pypi_mirror") @patch("_helper_functions.is_linux_armv7_runner", return_value=True) @patch.object(bwf, "bounded_pin_without_find_links_skip", return_value=(False, "")) @patch.object(bwf, "_pypi_preflight_skip_line", return_value=False) @@ -70,11 +73,13 @@ def test_armv7_skips_cryptography_find_links( _mock_pypi, _mock_armv7_pin, _mock_armv7, + _mock_prune, ) -> None: self.assertTrue( bwf._dependent_requirement_skip_line("cryptography>=2.1.4", force_interpreter=True), ) + @patch.object(bwf, "prune_ci_macos_newer_than_pypi_mirror") @patch("_helper_functions.is_linux_armv7_runner", return_value=True) @patch.object(bwf, "bounded_pin_without_find_links_skip", return_value=(False, "")) @patch.object(bwf, "_pypi_preflight_skip_line", return_value=False) @@ -92,6 +97,7 @@ def test_armv7_still_skips_obsolete_cryptography_pin( _mock_pypi, _mock_armv7_pin, _mock_armv7, + _mock_prune, ) -> None: self.assertTrue( bwf._dependent_requirement_skip_line("cryptography<43", force_interpreter=True), @@ -119,6 +125,7 @@ def test_macos_intel_rebuilds_stale_cryptography_find_links( bwf._dependent_requirement_skip_line("cryptography>=2.1.4", force_interpreter=True), ) + @patch.object(bwf, "prune_ci_macos_newer_than_pypi_mirror") @patch.object(bwf, "bounded_pin_without_find_links_skip", return_value=(False, "")) @patch.object(bwf, "_pypi_preflight_skip_line", return_value=False) @patch.object( @@ -131,10 +138,22 @@ def test_find_links_skip_when_not_forcing_interpreter( _mock_find_links, _mock_pypi, _mock_armv7, + mock_prune, ) -> None: self.assertTrue( bwf._dependent_requirement_skip_line("cryptography>=2.1.4", force_interpreter=False), ) + mock_prune.assert_called_once() + + @patch.object(bwf.platform, "system", return_value="Darwin") + def test_force_interpreter_ignored_on_macos(self, _sys) -> None: + self.assertFalse(bwf._apply_force_interpreter_binary(True)) + self.assertEqual(bwf._force_interpreter_no_binary_args("cffi>=1.15"), []) + + @patch.object(bwf.platform, "system", return_value="Linux") + def test_force_interpreter_no_binary_on_linux(self, _sys) -> None: + self.assertTrue(bwf._apply_force_interpreter_binary(True)) + self.assertEqual(bwf._force_interpreter_no_binary_args("cffi>=1.15"), ["--no-binary", "cffi"]) if __name__ == "__main__": diff --git a/test_native_import_guard.py b/test_native_import_guard.py index 760c4e3..2d73f60 100644 --- a/test_native_import_guard.py +++ b/test_native_import_guard.py @@ -19,6 +19,7 @@ from native_import_guard import is_pure_any_wheel_name from native_import_guard import load_native_import_guard from native_import_guard import native_import_guard_by_name +from native_import_guard import package_import_statements from native_import_guard import resolve_native_import_statements @@ -31,6 +32,7 @@ def _cfg( probe_unlisted=probe_unlisted, skip_pure_any=skip_pure_any, skip_top_level=DEFAULT_SKIP_TOP_LEVEL_IMPORTS, + skip_name_prefixes=frozenset(), packages=packages, ) @@ -69,7 +71,9 @@ def test_yaml_uses_package_name_and_probe_unlisted(self) -> None: self.assertTrue(cfg.probe_unlisted) self.assertTrue(cfg.skip_pure_any) self.assertEqual(cfg.skip_top_level, DEFAULT_SKIP_TOP_LEVEL_IMPORTS) + self.assertEqual(cfg.skip_name_prefixes, frozenset({"pyobjc"})) names = {entry.name for entry in cfg.packages} + self.assertIn("dbus-python", names) self.assertIn("cffi", names) self.assertIn("psutil", names) self.assertIn("pillow", names) @@ -95,14 +99,18 @@ def test_default_imports_from_top_level_txt(self) -> None: zf.writestr("greenlet-3.1.0.dist-info/top_level.txt", "greenlet\ntest\n") self.assertEqual(default_native_import_statements(wheel), ("import greenlet",)) - def test_default_imports_fallback_to_distribution_name(self) -> None: + def test_default_imports_skip_when_top_level_missing(self) -> None: with tempfile.TemporaryDirectory() as tmp: wheel = Path(tmp) / "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_10_9_x86_64.whl" wheel.write_bytes(b"not-a-zip") - self.assertEqual( - default_native_import_statements(wheel), - ("import ruamel_yaml_clib",), - ) + self.assertEqual(default_native_import_statements(wheel), ()) + + def test_default_imports_prefers_public_matching_distribution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "greenlet-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl" + with zipfile.ZipFile(wheel, "w") as zf: + zf.writestr("greenlet-3.1.0.dist-info/top_level.txt", "_greenlet\ngreenlet\n") + self.assertEqual(default_native_import_statements(wheel), ("import greenlet",)) def test_resolve_custom_imports(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -143,7 +151,11 @@ def test_resolve_skip_on_platform(self) -> None: def test_resolve_python_and_version_filters(self) -> None: with tempfile.TemporaryDirectory() as tmp: - wheel = _write_wheel(tmp, "bar-2.0.0-cp311-cp311-manylinux_2_28_x86_64.whl") + wheel = _write_wheel( + tmp, + "bar-2.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", + top_level="bar\n", + ) cfg = _cfg( NativeImportGuardEntry( name="bar", @@ -161,6 +173,7 @@ def test_resolve_python_and_version_filters(self) -> None: ), ("import bar_ext",), ) + # Unmatched filters treat the package as unlisted (default top_level import). self.assertEqual( resolve_native_import_statements( wheel, @@ -181,12 +194,73 @@ def test_resolve_probe_unlisted_false_skips_unknown(self) -> None: cfg = _cfg(probe_unlisted=False) self.assertIsNone(resolve_native_import_statements(wheel, config=cfg, current_platform="macos_x86_64")) + def test_resolve_skips_pyobjc_name_prefix(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel( + tmp, + "pyobjc_framework_CoreBluetooth-10.3.2-cp36-abi3-macosx_11_0_universal2.whl", + top_level="CoreBluetooth\n", + ) + cfg = NativeImportGuardConfig( + probe_unlisted=True, + skip_pure_any=True, + skip_top_level=DEFAULT_SKIP_TOP_LEVEL_IMPORTS, + skip_name_prefixes=frozenset({"pyobjc"}), + packages=(), + ) + self.assertIsNone(resolve_native_import_statements(wheel, config=cfg, current_platform="macos_x86_64")) + + def test_resolve_skips_dbus_python_on_armv7(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel( + tmp, + "dbus_python-1.2.18-cp313-cp313-manylinux_2_36_armv7l.whl", + top_level="dbus\n", + ) + cfg = load_native_import_guard() + self.assertIsNone(resolve_native_import_statements(wheel, config=cfg, current_platform="linux_armv7")) + self.assertEqual( + resolve_native_import_statements(wheel, config=cfg, current_platform="linux_x86_64"), + ("import dbus",), + ) + def test_resolve_skips_pure_any(self) -> None: with tempfile.TemporaryDirectory() as tmp: wheel = _write_wheel(tmp, "six-1.16.0-py2.py3-none-any.whl", top_level="six\n") cfg = _cfg() self.assertIsNone(resolve_native_import_statements(wheel, config=cfg)) + def test_invalid_version_specifier_raises_value_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + wheel = _write_wheel(tmp, "bar-2.0.0-cp311-cp311-manylinux_2_28_x86_64.whl") + cfg = _cfg( + NativeImportGuardEntry( + name="bar", + imports=("import bar_ext",), + skip=False, + raw={"package_name": "bar", "version": ">>>1"}, + ) + ) + with self.assertRaises(ValueError) as caught: + resolve_native_import_statements(wheel, config=cfg, current_platform="linux_x86_64") + self.assertIn(">>>1", str(caught.exception)) + self.assertIn("native_import_guard.yaml", str(caught.exception)) + + def test_package_import_statements_applies_platform_filter(self) -> None: + cfg = _cfg( + NativeImportGuardEntry( + name="foo", + imports=("import foo_ext",), + skip=False, + raw={"package_name": "foo", "platform": "linux_armv7"}, + ) + ) + self.assertEqual( + package_import_statements("foo", config=cfg, current_platform="linux_armv7"), + ("import foo_ext",), + ) + self.assertIsNone(package_import_statements("foo", config=cfg, current_platform="macos_x86_64")) + if __name__ == "__main__": unittest.main() diff --git a/wheel_compat_policy.py b/wheel_compat_policy.py index dfbf33c..2487b62 100644 --- a/wheel_compat_policy.py +++ b/wheel_compat_policy.py @@ -17,7 +17,9 @@ from colorama import Fore from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement +from packaging.utils import InvalidWheelFilename from packaging.utils import canonicalize_name +from packaging.utils import parse_wheel_filename from _helper_functions import _REPO_ROOT from _helper_functions import DEFAULT_WHEEL_DIR @@ -30,10 +32,78 @@ from _helper_functions import parse_wheel_name from _helper_functions import print_color -# macOS: ``pip wheel --only-binary :all:`` so CI does not sdist-compile against the -# runner SDK (higher ``macosx_*`` tags steal pip preference; local ``.so`` can SIGABRT). -# ``--no-binary cryptography`` on Intel still wins (OpenSSL 4 rebuild). +# macOS: ``pip wheel --only-binary :all:`` first so CI does not sdist-compile against +# the runner SDK (higher ``macosx_*`` tags steal pip preference; local ``.so`` can SIGABRT). +# Retry without that flag when a requirement or dependency has no binary wheel. +# ``--no-binary cryptography`` on Intel still wins (OpenSSL 4 rebuild) and is not pruned. _MACOSX_PLAT_TAG = re.compile(r"macosx_(\d+)_(\d+)_([a-z0-9_]+)", re.IGNORECASE) +# PEP 425: ``cp`` + one-digit major + remaining digits as minor (``cp38``, ``cp310``). +_CPYTHON_INTERP = re.compile(r"^cp(?P\d)(?P\d{1,})$") + + +def _wheel_python_abi_key(wheel_name: str) -> tuple[str, str] | None: + """Sorted interpreter / ABI tags from a wheel filename (compressed sets joined).""" + try: + _name, _version, _build, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return None + interpreters = tuple(sorted({tag.interpreter for tag in tags})) + abis = tuple(sorted({tag.abi for tag in tags})) + if not interpreters: + return None + return ",".join(interpreters), ",".join(abis) + + +def _wheel_cpython_ranges(wheel_name: str) -> list[tuple[tuple[int, int], tuple[int, int] | None]]: + """CPython versions a wheel can serve: ``(min, max)`` with ``max`` None meaning unbounded.""" + try: + _name, _version, _build, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return [] + ranges: list[tuple[tuple[int, int], tuple[int, int] | None]] = [] + seen: set[tuple[tuple[int, int], tuple[int, int] | None]] = set() + for tag in tags: + match = _CPYTHON_INTERP.fullmatch(tag.interpreter) + if not match: + continue + ver = (int(match.group("major")), int(match.group("minor"))) + if tag.abi == "abi3": + item: tuple[tuple[int, int], tuple[int, int] | None] = (ver, None) + elif tag.abi == tag.interpreter or tag.abi.startswith("cp"): + item = (ver, ver) + else: + continue + if item not in seen: + seen.add(item) + ranges.append(item) + return ranges + + +def _cpython_ranges_overlap( + left: list[tuple[tuple[int, int], tuple[int, int] | None]], + right: list[tuple[tuple[int, int], tuple[int, int] | None]], +) -> bool: + for a_min, a_max in left: + for b_min, b_max in right: + start = max(a_min, b_min) + ends = [bound for bound in (a_max, b_max) if bound is not None] + if not ends or start <= min(ends): + return True + return False + + +def _wheels_compete_for_same_cpython(name_a: str, name_b: str) -> bool: + """True when pip could consider both wheels for the same CPython interpreter. + + Exclusive ABIs (``cp311-cp311`` vs ``cp312-cp312``) do not compete. ``abi3`` + wheels compete with each other and with exclusive wheels they can serve, so a + ``cp36-abi3`` PyPI tag still wins over a CI ``cp311-cp311`` sibling. + """ + left = _wheel_cpython_ranges(name_a) + right = _wheel_cpython_ranges(name_b) + if left and right: + return _cpython_ranges_overlap(left, right) + return _wheel_python_abi_key(name_a) == _wheel_python_abi_key(name_b) def get_macos_only_binary_args() -> list[str]: @@ -43,6 +113,85 @@ def get_macos_only_binary_args() -> list[str]: return ["--only-binary", ":all:"] +def without_only_binary_all(args: list[str]) -> list[str] | None: + """Return ``args`` without ``--only-binary :all:``, or ``None`` if that flag is absent.""" + out: list[str] = [] + found = False + i = 0 + while i < len(args): + if args[i] == "--only-binary" and i + 1 < len(args) and args[i + 1] == ":all:": + found = True + i += 2 + continue + out.append(args[i]) + i += 1 + return out if found else None + + +def run_pip_wheel( + requirement: str, + extra_args: list[str] | None = None, + *, + find_links_dir: Path | str = DEFAULT_WHEEL_DIR, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + """Run ``pip wheel``. On macOS, retry without ``--only-binary :all:`` if binaries are missing.""" + cmd = [ + sys.executable, + "-m", + "pip", + "wheel", + requirement, + *pip_wheel_invocation_args(find_links_dir), + *(extra_args or []), + ] + out = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + if out.returncode == 0: + return out + fallback = without_only_binary_all(cmd) + if fallback is None: + return out + print_color( + f"-- retry {requirement} without --only-binary :all: (no binary wheel for this requirement or a dependency)", + Fore.YELLOW, + ) + return subprocess.run(fallback, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + + +def _iter_unique_wheels(dest: Path): + """Yield ``*.whl`` under ``dest`` once per inode (``rglob`` can list symlink twins).""" + seen: set[object] = set() + for path in dest.rglob("*.whl"): + try: + if not path.is_file(): + continue + st = path.stat() + # Windows FAT (and some NTFS views) report ``st_ino == 0`` for every file. + key: object = (st.st_dev, st.st_ino) if st.st_ino else path.resolve() + except OSError: + continue + if key in seen: + continue + seen.add(key) + yield path + + +def _macos_arch_families(arch: str) -> frozenset[str]: + """Machine families a ``macosx_*_`` tag can serve (``universal2`` is both).""" + token = arch.lower() + if token in {"universal2", "universal", "fat64"}: + return frozenset({"intel", "arm"}) + if token in {"x86_64", "amd64", "intel", "i386", "i686"}: + return frozenset({"intel"}) + if token in {"arm64", "aarch64", "arm64e"}: + return frozenset({"arm"}) + return frozenset({token}) + + +def _macos_archs_compete(arch_a: str, arch_b: str) -> bool: + return bool(_macos_arch_families(arch_a) & _macos_arch_families(arch_b)) + + def _macosx_deployment_and_arch(wheel_name: str) -> tuple[tuple[int, int], str] | None: """Return ``((major, minor), arch)`` from a ``macosx_M_N_arch`` wheel filename.""" match = _MACOSX_PLAT_TAG.search(wheel_name) @@ -78,13 +227,18 @@ def prune_ci_macos_newer_than_pypi_mirror( Pip prefers the highest compatible ``macosx_*`` tag. A CI sdist build on ``macos-15-intel`` becomes ``macosx_15_0_x86_64`` and wins over the official ``macosx_10_9_x86_64`` PyPI copy on the same extra-index. Applies to every - distribution (not only psutil). + distribution (not only psutil). Only compares wheels that compete for the + same CPython (python+ABI family) and the same machine family (``x86_64`` / + ``intel`` / ``universal2``, or ``arm64`` / ``universal2``), so ``cp311-cp311`` + is not dropped just because ``cp312-cp312`` has a lower tag. Intel + cryptography OpenSSL 4 rebuilds are kept even when a lower PyPI tag exists. """ dest = Path(wheel_dir) if not dest.is_dir(): return 0 - groups: dict[tuple[str, str, str], list[tuple[tuple[int, int], Path]]] = {} - for path in dest.rglob("*.whl"): + keep_intel_cryptography = get_current_platform() == "macos_x86_64" + groups: dict[tuple[str, str], list[tuple[tuple[int, int], str, Path]]] = {} + for path in _iter_unique_wheels(dest): parsed = parse_wheel_name(path.name) if not parsed: continue @@ -92,22 +246,31 @@ def prune_ci_macos_newer_than_pypi_mirror( if not tag: continue deployment, arch = tag - groups.setdefault((parsed[0], parsed[1], arch), []).append((deployment, path)) + groups.setdefault((parsed[0], parsed[1]), []).append((deployment, arch, path)) removed = 0 - for (dist, version, arch), items in groups.items(): + for (dist, version), items in groups.items(): + if keep_intel_cryptography and dist == canonicalize_name("cryptography"): + continue if len(items) < 2: continue - lowest = min(deployment for deployment, _path in items) - for deployment, path in items: - if deployment <= lowest: + for deployment, arch, path in items: + lower_siblings = [ + other_dep + for other_dep, other_arch, other in items + if other_dep < deployment + and _macos_archs_compete(arch, other_arch) + and _wheels_compete_for_same_cpython(path.name, other.name) + ] + if not lower_siblings: continue + keep_tag = min(lower_siblings) print_color( f"-- removed {path.name} (macosx_{deployment[0]}_{deployment[1]}_{arch}; " - f"keeping macosx_{lowest[0]}_{lowest[1]} for {dist}=={version})", + f"keeping macosx_{keep_tag[0]}_{keep_tag[1]} for {dist}=={version})", Fore.YELLOW, ) - path.unlink() + path.unlink(missing_ok=True) removed += 1 return removed @@ -129,16 +292,14 @@ def pip_wheel_standard_args(find_links_dir: Path | str = DEFAULT_WHEEL_DIR) -> l def pip_wheel_invocation_args( - requirement_name: str, find_links_dir: Path | str = DEFAULT_WHEEL_DIR, ) -> list[str]: - """``pip wheel`` flags for a single requirement (may drop ``--no-build-isolation`` on ARMv7).""" + """``pip wheel`` flags (may drop ``--no-build-isolation`` on ARMv7).""" args = list(pip_wheel_standard_args(find_links_dir)) if is_linux_armv7_runner(): # cffi 2.x sdists fail metadata prep under host setuptools + --no-build-isolation # (project.license validation), including transitive cffi pulls (e.g. esptool → cryptography). # PEP 517 isolation matches build_requirements install on ARMv7 Docker. - del requirement_name # per-requirement env still via armv7_pip_wheel_subprocess_env() args = [arg for arg in args if arg != "--no-build-isolation"] return args @@ -178,19 +339,26 @@ def prune_ci_manylinux_newer_than_228( dest = Path(wheel_dir) canonical = canonicalize_name(package_name) version = str(package_version) - removed = 0 + candidates: list[Path] = [] for path in dest.glob("*.whl"): parsed = parse_wheel_name(path.name) if not parsed or parsed[0] != canonical or parsed[1] != version: continue + candidates.append(path) + mirrors = [path for path in candidates if _wheel_has_manylinux_228_tag(path.name)] + removed = 0 + for path in candidates: glibc = _wheel_highest_manylinux_glibc(path.name) - if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): - print_color( - f"-- removed {path.name} (CI manylinux_{glibc[0]}_{glibc[1]}; PyPI 2_28 mirror kept)", - Fore.YELLOW, - ) - path.unlink() - removed += 1 + if glibc is None or glibc <= (2, 28) or _wheel_has_manylinux_228_tag(path.name): + continue + if mirrors and not any(_wheels_compete_for_same_cpython(path.name, mirror.name) for mirror in mirrors): + continue + print_color( + f"-- removed {path.name} (CI manylinux_{glibc[0]}_{glibc[1]}; PyPI 2_28 mirror kept)", + Fore.YELLOW, + ) + path.unlink(missing_ok=True) + removed += 1 return removed @@ -199,14 +367,16 @@ def prune_ci_manylinux_newer_than_228_when_228_mirror_present( wheel_dir: Path | str = DEFAULT_WHEEL_DIR, package_names: frozenset[str] | None = None, ) -> int: - """Drop manylinux > ``2_28`` when a ``2_28`` wheel exists for the same dist+version. + """Drop manylinux > ``2_28`` when a competing ``2_28`` wheel exists. Used after merging matrix artifacts and after ``repair_wheels`` so CI or auditwheel outputs cannot coexist with PyPI ``manylinux_2_28`` mirrors on the upload index. + A ``2_28`` wheel for one exclusive ABI does not prune ``>2_28`` wheels for + another interpreter; ``abi3`` mirrors still prune competing newer tags. """ dest = Path(wheel_dir) by_dist_version: dict[tuple[str, str], list[Path]] = {} - for path in dest.rglob("*.whl"): + for path in _iter_unique_wheels(dest): parsed = parse_wheel_name(path.name) if not parsed: continue @@ -218,18 +388,22 @@ def prune_ci_manylinux_newer_than_228_when_228_mirror_present( removed = 0 for (dist, version), paths in by_dist_version.items(): - if not any(_wheel_has_manylinux_228_tag(p.name) for p in paths): + mirrors = [path for path in paths if _wheel_has_manylinux_228_tag(path.name)] + if not mirrors: continue for path in paths: glibc = _wheel_highest_manylinux_glibc(path.name) - if glibc is not None and glibc > (2, 28) and not _wheel_has_manylinux_228_tag(path.name): - print_color( - f"-- removed {path.name} (manylinux_{glibc[0]}_{glibc[1]}; " - f"keeping manylinux_2_28 for {dist}=={version})", - Fore.YELLOW, - ) - path.unlink() - removed += 1 + if glibc is None or glibc <= (2, 28) or _wheel_has_manylinux_228_tag(path.name): + continue + if not any(_wheels_compete_for_same_cpython(path.name, mirror.name) for mirror in mirrors): + continue + print_color( + f"-- removed {path.name} (manylinux_{glibc[0]}_{glibc[1]}; " + f"keeping manylinux_2_28 for {dist}=={version})", + Fore.YELLOW, + ) + path.unlink(missing_ok=True) + removed += 1 return removed From c2c5155c9a354a3b6041815ac5bf3b4b48bfd4b6 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Tue, 8 Sep 2026 11:15:34 +0200 Subject: [PATCH 5/7] feat: Keep Debian 11 builds after EOL --- os_dependencies/debian_eol_apt.sh | 58 +++++++++++++++ os_dependencies/linux_armv7_docker_prepare.sh | 6 ++ test_debian_eol_apt.py | 72 +++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 os_dependencies/debian_eol_apt.sh create mode 100644 test_debian_eol_apt.py diff --git a/os_dependencies/debian_eol_apt.sh b/os_dependencies/debian_eol_apt.sh new file mode 100644 index 0000000..5dcc151 --- /dev/null +++ b/os_dependencies/debian_eol_apt.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# +# Debian 11 (bullseye) LTS ended 2026-08-31. The project no longer refreshes +# bullseye-security InRelease, so ``apt-get update`` exits 100: +# Release file ... is expired (invalid since ...) +# +# Do not rewrite security URLs to archive.debian.org yet — that suite is not +# copied there. Ignore Valid-Until so the last published snapshot still works. +# Call before the first ``apt-get update``. Bookworm and later are unchanged. + +debian_allow_expired_release_files() { + local root="${1:-/}" + mkdir -p "${root}/etc/apt/apt.conf.d" + cat > "${root}/etc/apt/apt.conf.d/99no-check-valid-until" <<'EOF' +Acquire::Check-Valid-Until "false"; +EOF +} + +debian_repoint_eol_archive() { + local root="${1:-/}" + # Archived releases: archive.debian.org has no separate "-security" suite — + # the archived main suite already reflects the final EOL state. Comment out + # any *-security source and repoint the main debian.org source at the archive. + local list="${root}/etc/apt/sources.list" + if [ -f "$list" ]; then + sed -i \ + -e '/-security/ s/^deb /# deb /' \ + -e 's|http://deb.debian.org/debian |http://archive.debian.org/debian |' \ + "$list" + fi + # Handle split sources.list.d/*.list if the bullseye image uses deb822 or split files + for f in "${root}"/etc/apt/sources.list.d/*.list; do + [ -e "$f" ] || continue + sed -i \ + -e '/-security/ s/^deb /# deb /' \ + -e 's|http://deb.debian.org/debian |http://archive.debian.org/debian |' \ + "$f" + done +} + +debian_prepare_eol_apt() { + local root="${1:-/}" + local codename="${2:-}" + if [ -z "${codename}" ] && [ -f "${root}/etc/os-release" ]; then + # shellcheck disable=SC1090 + . "${root}/etc/os-release" + codename="${VERSION_CODENAME:-}" + fi + case "${codename}" in + bullseye) + debian_allow_expired_release_files "${root}" + debian_repoint_eol_archive "${root}" + ;; + *) return 0 ;; + esac +} diff --git a/os_dependencies/linux_armv7_docker_prepare.sh b/os_dependencies/linux_armv7_docker_prepare.sh index e6337dd..c8bede2 100644 --- a/os_dependencies/linux_armv7_docker_prepare.sh +++ b/os_dependencies/linux_armv7_docker_prepare.sh @@ -23,6 +23,12 @@ case "${VERSION_CODENAME:-}" in *) LIBFFI_RUNTIME= ;; esac +_armv7_prepare_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=os_dependencies/debian_eol_apt.sh +. "${_armv7_prepare_dir}/debian_eol_apt.sh" +debian_prepare_eol_apt / "${VERSION_CODENAME:-}" +unset _armv7_prepare_dir + apt-get update -qq apt-get install -y --no-install-recommends \ ca-certificates \ diff --git a/test_debian_eol_apt.py b/test_debian_eol_apt.py new file mode 100644 index 0000000..a3b100e --- /dev/null +++ b/test_debian_eol_apt.py @@ -0,0 +1,72 @@ +# +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +import unittest + +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +HELPER_PATH = ROOT / "os_dependencies" / "debian_eol_apt.sh" +HELPER = HELPER_PATH.read_text(encoding="utf-8") +PREPARE = (ROOT / "os_dependencies" / "linux_armv7_docker_prepare.sh").read_text(encoding="utf-8") + + +class TestDebianEolApt(unittest.TestCase): + def test_helper_ignores_expired_inrelease_on_bullseye_only(self) -> None: + self.assertIn("bullseye", HELPER) + self.assertIn('Acquire::Check-Valid-Until "false"', HELPER) + self.assertIn("debian_prepare_eol_apt", HELPER) + self.assertNotIn("bookworm)", HELPER) + + def test_prepare_applies_helper_before_apt_update(self) -> None: + self.assertIn("debian_eol_apt.sh", PREPARE) + self.assertIn("debian_prepare_eol_apt /", PREPARE) + self.assertLess( + PREPARE.index("debian_prepare_eol_apt"), + PREPARE.index("apt-get update"), + ) + + @unittest.skipIf( + sys.platform == "win32" or not shutil.which("bash"), + "helper is sourced in Linux Docker, not on Windows", + ) + def test_bullseye_writes_apt_conf(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run( + [ + "bash", + "-c", + f'. "{HELPER_PATH}" && debian_prepare_eol_apt "{root}" bullseye', + ], + check=True, + ) + conf = root / "etc" / "apt" / "apt.conf.d" / "99no-check-valid-until" + self.assertTrue(conf.is_file()) + self.assertIn('Acquire::Check-Valid-Until "false"', conf.read_text(encoding="utf-8")) + + @unittest.skipIf( + sys.platform == "win32" or not shutil.which("bash"), + "helper is sourced in Linux Docker, not on Windows", + ) + def test_bookworm_is_noop(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run( + [ + "bash", + "-c", + f'. "{HELPER_PATH}" && debian_prepare_eol_apt "{root}" bookworm', + ], + check=True, + ) + self.assertFalse((root / "etc").exists()) From 0b89c59b9342871157525394e2f29cb95a6187e0 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Thu, 10 Sep 2026 14:58:47 +0200 Subject: [PATCH 6/7] fix: Unbreak ARMv7 Legacy wheel builds Sourcing linux_armv7_docker_prepare.sh turned errexit off in the workflow shell: bash clears errexit inside the command substitution of an assignment, so the "$(set +o)" snapshot restored "set +o errexit". Failures in linux_arm.sh then looked like success. That hid an apt failure on the bullseye armhf images, where python3-dev has no installable version (they carry libpython3.9-stdlib from bullseye-security, which is gone from archive.debian.org, while main only offers the matching -dev at the original point release). Setup aborted before installing Rust and the dbus dev packages, so bcrypt and dbus-python failed to build half an hour later. Wheels build against the interpreter on PATH, so the distro headers are now best-effort. macos_openssl4_intel.sh had the same errexit snapshot bug. Co-authored-by: Cursor --- os_dependencies/linux_arm.sh | 7 +- os_dependencies/linux_armv7_docker_prepare.sh | 13 ++- os_dependencies/macos_openssl4_intel.sh | 10 ++- test_os_dependencies_shell.py | 81 +++++++++++++++++++ 4 files changed, 105 insertions(+), 6 deletions(-) create mode 100644 test_os_dependencies_shell.py diff --git a/os_dependencies/linux_arm.sh b/os_dependencies/linux_arm.sh index 4a92fd8..daf629a 100755 --- a/os_dependencies/linux_arm.sh +++ b/os_dependencies/linux_arm.sh @@ -15,7 +15,12 @@ apt-get install -y cmake build-essential # PyGObject needs build dependencies https://pygobject.readthedocs.io/en/latest/getting_started.html # Install both dev packages (for building) and runtime libraries (for auditwheel repair) -apt-get install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config python3-dev libglib2.0-dev +apt-get install -y libgirepository1.0-dev gcc libcairo2-dev pkg-config libglib2.0-dev +# Wheels build against the interpreter on PATH (the Docker images ship CPython in /usr/local), +# so the distro headers only help builds that target the distro python. They are uninstallable +# on the bullseye armhf images: those carry libpython3.9-stdlib from bullseye-security, which no +# longer exists on archive.debian.org, while main only offers the matching -dev at 3.9.2-1. +apt-get install -y python3-dev || echo "python3-dev has no installable version here; skipping" # gir1.2-gtk-4.0 not available on all distros/architectures (e.g. Debian armhf) apt-get install -y gir1.2-gtk-4.0 || true # Try to install girepository-2.0 packages if available (may not exist in older distros) diff --git a/os_dependencies/linux_armv7_docker_prepare.sh b/os_dependencies/linux_armv7_docker_prepare.sh index c8bede2..1a044d7 100644 --- a/os_dependencies/linux_armv7_docker_prepare.sh +++ b/os_dependencies/linux_armv7_docker_prepare.sh @@ -2,9 +2,11 @@ # Minimal OS setup for ARMv7 Docker builds *before* pip installs (build_requirements / wheels). # Expects to run as root (official python:*-bookworm / *-bullseye images). # Must be sourced (not subprocess bash) so PIP_NO_BINARY persists for later pip / PEP 517 builds. -# Restore caller ``set`` options afterwards so ``errexit`` does not leak into that shell. +# Restore the caller's ``errexit`` afterwards so ``set -e`` does not leak into that shell. +# Read ``$-`` directly: bash clears errexit inside the command substitution of an assignment, +# so a ``$(set +o)`` snapshot reports it as off and disables it in the caller on restore. -_armv7_prepare_saved_opts="$(set +o)" +_armv7_prepare_saved_flags="$-" set -e export DEBIAN_FRONTEND=noninteractive @@ -53,5 +55,8 @@ if [ "$arch" = "armv7l" ]; then done fi -eval "${_armv7_prepare_saved_opts}" -unset _armv7_prepare_saved_opts +case "${_armv7_prepare_saved_flags}" in + *e*) ;; + *) set +e ;; +esac +unset _armv7_prepare_saved_flags diff --git a/os_dependencies/macos_openssl4_intel.sh b/os_dependencies/macos_openssl4_intel.sh index 4ae3bf1..ace39fd 100644 --- a/os_dependencies/macos_openssl4_intel.sh +++ b/os_dependencies/macos_openssl4_intel.sh @@ -8,11 +8,19 @@ # # Sourced from macos.sh / spike CI: restore caller ``set`` options so ``-u``/``pipefail`` # do not leak into the rest of that shell. +# +# errexit needs ``$-`` on top of the snapshot: bash clears errexit inside the command +# substitution of an assignment, so ``$(set +o)`` reports it as off and would disable a +# caller's ``set -e`` on restore. _openssl4_saved_opts="$(set +o)" +_openssl4_saved_flags="$-" _openssl4_restore_shell() { eval "${_openssl4_saved_opts}" - unset _openssl4_saved_opts prefix + case "${_openssl4_saved_flags}" in + *e*) set -e ;; + esac + unset _openssl4_saved_opts _openssl4_saved_flags prefix unset -f _append_github_env _brew_openssl4_prefix _install_openssl4_from_source \ _configure_openssl4_env _openssl4_restore_shell 2>/dev/null || true } diff --git a/test_os_dependencies_shell.py b/test_os_dependencies_shell.py new file mode 100644 index 0000000..737b083 --- /dev/null +++ b/test_os_dependencies_shell.py @@ -0,0 +1,81 @@ +# +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# +# SPDX-License-Identifier: Apache-2.0 +# + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +PREPARE_PATH = ROOT / "os_dependencies" / "linux_armv7_docker_prepare.sh" +LINUX_ARM = (ROOT / "os_dependencies" / "linux_arm.sh").read_text(encoding="utf-8") + +# The prepare script only needs apt-get stubbed out; everything else it touches +# (``/etc/os-release``, ``uname``) is read-only on a Linux runner. +APT_STUB = "#!/bin/sh\nexit 0\n" + + +def _prepare_errexit_after_sourcing(caller_errexit: bool) -> str: + """Return "on"/"off" for errexit in a shell that sourced the prepare script.""" + with tempfile.TemporaryDirectory() as tmp: + stub_dir = Path(tmp) / "bin" + stub_dir.mkdir() + apt_get = stub_dir / "apt-get" + apt_get.write_text(APT_STUB, encoding="utf-8") + apt_get.chmod(0o755) + + script = "\n".join( + [ + "set -e" if caller_errexit else "set +e", + f'. "{PREPARE_PATH}"', + 'case "$-" in *e*) echo errexit=on ;; *) echo errexit=off ;; esac', + ] + ) + env = dict(os.environ, PATH=f"{stub_dir}{os.pathsep}{os.environ['PATH']}") + result = subprocess.run( + ["bash", "-c", script], + env=env, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip().rpartition("errexit=")[2] + + +@unittest.skipIf( + sys.platform == "win32" or not shutil.which("bash") or not Path("/etc/os-release").exists(), + "prepare script is sourced in Linux Docker images", +) +class TestArmv7PrepareShellOptions(unittest.TestCase): + def test_keeps_caller_errexit(self) -> None: + # A ``$(set +o)`` snapshot reports errexit as off, because bash clears it inside the + # command substitution of an assignment. Restoring that snapshot disabled ``set -e`` in + # the workflow shell, so a failing ``linux_arm.sh`` (no Rust, no dbus headers installed) + # looked like success and only surfaced as unrelated wheel build errors an hour later. + self.assertEqual("on", _prepare_errexit_after_sourcing(caller_errexit=True)) + + def test_does_not_leak_errexit(self) -> None: + self.assertEqual("off", _prepare_errexit_after_sourcing(caller_errexit=False)) + + +class TestLinuxArmDependencies(unittest.TestCase): + def test_distro_python_headers_are_optional(self) -> None: + # The bullseye armhf images carry libpython3.9-stdlib from bullseye-security, which is + # gone from archive.debian.org, so python3-dev has no installable version there. Wheels + # build against the interpreter on PATH, so a missing python3-dev must not abort setup. + lines = [line for line in LINUX_ARM.splitlines() if "python3-dev" in line] + self.assertEqual(1, len(lines)) + self.assertRegex(lines[0], r"python3-dev\s*\|\|") + + +if __name__ == "__main__": + unittest.main() From 84946676370805d7c3d321b6e71c1c44e904cfa5 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Fri, 11 Sep 2026 09:41:30 +0200 Subject: [PATCH 7/7] fix: Added import guard for pyyaml macOS ARM --- native_import_guard.yaml | 7 +++++++ test_native_import_guard.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/native_import_guard.yaml b/native_import_guard.yaml index 0f3c8ee..d99b94d 100644 --- a/native_import_guard.yaml +++ b/native_import_guard.yaml @@ -72,6 +72,13 @@ packages: - package_name: pillow imports: - from PIL import Image + # PyPI has no macOS arm64 wheel for PyYAML on Python 3.8, so we build the sdist without + # libyaml and ship a working pure-Python wheel that has no CSafeLoader. + - package_name: pyyaml + platform: macos_arm64 + python: '==3.8' + imports: + - import yaml - package_name: pyyaml imports: - from yaml import CSafeLoader diff --git a/test_native_import_guard.py b/test_native_import_guard.py index 2d73f60..7b6cd3d 100644 --- a/test_native_import_guard.py +++ b/test_native_import_guard.py @@ -84,10 +84,23 @@ def test_yaml_uses_package_name_and_probe_unlisted(self) -> None: def test_loads_deeper_native_imports(self) -> None: guarded = native_import_guard_by_name() self.assertEqual(guarded["pillow"].imports, ("from PIL import Image",)) - self.assertEqual(guarded["pyyaml"].imports, ("from yaml import CSafeLoader",)) self.assertEqual(guarded["brotli"].imports, ("import brotli",)) self.assertEqual(guarded["pycryptodome"].imports, ("from Crypto.Cipher import AES",)) + def test_pyyaml_requires_csafeloader_except_on_macos_arm64_python38(self) -> None: + self.assertEqual( + package_import_statements("pyyaml", current_platform="linux_x86_64", python_version="3.8"), + ("from yaml import CSafeLoader",), + ) + self.assertEqual( + package_import_statements("pyyaml", current_platform="macos_arm64", python_version="3.9"), + ("from yaml import CSafeLoader",), + ) + self.assertEqual( + package_import_statements("pyyaml", current_platform="macos_arm64", python_version="3.8"), + ("import yaml",), + ) + def test_pure_any_wheel_name(self) -> None: self.assertTrue(is_pure_any_wheel_name("six-1.16.0-py2.py3-none-any.whl")) self.assertFalse(is_pure_any_wheel_name("psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl"))