diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..33aee19ec --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Ported kernels are generated: /port/port.kdl plus /port/overlay +# are the inputs, and /src is kernel-port's output, committed so that +# what ships stays reviewable and auditable. +# +# Collapse the generated tree in diffs so review lands on the inputs. The output +# is not taken on trust: .github/workflows/check-ports.yaml regenerates every +# port and fails if /src does not match byte for byte. +*/src/** linguist-generated=true diff --git a/.github/scripts/check_kernel_test_versions.py b/.github/scripts/check_kernel_test_versions.py index 433e5a7f1..f271084cc 100644 --- a/.github/scripts/check_kernel_test_versions.py +++ b/.github/scripts/check_kernel_test_versions.py @@ -40,7 +40,9 @@ def discover_kernels(repository_root: Path) -> list[Kernel]: if not manifest.is_file(): # Ported kernels keep generated sources and build.toml under src/. manifest = root / "src" / "build.toml" - if not manifest.is_file() or not (root / "tests").is_dir(): + # A ported kernel generates its tests next to build.toml in src/. + tests_root = manifest.parent if (manifest.parent / "tests").is_dir() else root + if not manifest.is_file() or not (tests_root / "tests").is_dir(): continue with manifest.open("rb") as handle: @@ -52,7 +54,7 @@ def discover_kernels(repository_root: Path) -> list[Kernel]: raise ValueError(f"{manifest}: [general.hub].repo-id must be a string") if type(version) is not int: raise ValueError(f"{manifest}: [general].version must be an integer") - kernels.append(Kernel(root=root, repo_id=repo_id, version=version)) + kernels.append(Kernel(root=tests_root, repo_id=repo_id, version=version)) return kernels diff --git a/.github/scripts/kernel_maintainers.py b/.github/scripts/kernel_maintainers.py index 5bd220db1..ee5ca0383 100755 --- a/.github/scripts/kernel_maintainers.py +++ b/.github/scripts/kernel_maintainers.py @@ -22,7 +22,9 @@ def load_registry() -> dict: def check_registry() -> None: registry = load_registry() + # Ported kernels keep build.toml in /src. kernels = {path.parent.name for path in Path(".").glob("*/build.toml")} + kernels |= {path.parent.parent.name for path in Path(".").glob("*/src/build.toml")} registered = set(registry) problems = [ diff --git a/.github/scripts/pr_kernel_build_guidance.py b/.github/scripts/pr_kernel_build_guidance.py index 1f100203d..4be60b4dd 100644 --- a/.github/scripts/pr_kernel_build_guidance.py +++ b/.github/scripts/pr_kernel_build_guidance.py @@ -43,7 +43,10 @@ def paginate(url: str, token: str) -> list[dict]: def kernel_directories(root: Path) -> set[str]: - return {manifest.parent.name for manifest in root.glob("*/build.toml")} + # Ported kernels keep build.toml in /src. + return {manifest.parent.name for manifest in root.glob("*/build.toml")} | { + manifest.parent.parent.name for manifest in root.glob("*/src/build.toml") + } def is_documentation(path: PurePosixPath) -> bool: @@ -55,7 +58,8 @@ def touched_source_kernels(files: list[dict], existing_kernels: set[str]) -> lis new_kernels = { path.parts[0] for path in paths - if len(path.parts) == 2 and path.name == "build.toml" + if path.name == "build.toml" + and (len(path.parts) == 2 or (len(path.parts) == 3 and path.parts[1] == "src")) } kernels = existing_kernels | new_kernels touched = set(new_kernels) diff --git a/.github/scripts/test_check_kernel_test_versions.py b/.github/scripts/test_check_kernel_test_versions.py index d60b8caaf..27eea2b86 100644 --- a/.github/scripts/test_check_kernel_test_versions.py +++ b/.github/scripts/test_check_kernel_test_versions.py @@ -46,6 +46,21 @@ def test_matching_import_with_nested_manifest(tmp_path): assert checker.check_repository(tmp_path) == ([], 1, 1) +def test_ported_kernel_tests_under_src(tmp_path): + kernel = tmp_path / "example" + (kernel / "src" / "tests").mkdir(parents=True) + (kernel / "src" / "build.toml").write_text( + '[general]\nversion = 2\n\n[general.hub]\nrepo-id = "kernels-community/example"\n' + ) + (kernel / "src" / "tests" / "test_example.py").write_text( + "from kernels import get_kernel\n" + 'example = get_kernel("kernels-community/example", version=1)\n' + ) + + problems, _, _ = checker.check_repository(tmp_path) + assert len(problems) == 1 + + def test_matching_call_is_formatting_agnostic(tmp_path): kernel = write_kernel(tmp_path) (kernel / "tests" / "test_example.py").write_text( diff --git a/.github/scripts/test_pr_kernel_build_guidance.py b/.github/scripts/test_pr_kernel_build_guidance.py index 8ad5fed83..b7218814c 100644 --- a/.github/scripts/test_pr_kernel_build_guidance.py +++ b/.github/scripts/test_pr_kernel_build_guidance.py @@ -39,6 +39,22 @@ def test_unknown_extensions_and_test_changes_are_build_relevant(tmp_path): ) == ["relu"] +def test_detects_ported_kernel_source(tmp_path): + (tmp_path / "einops" / "src").mkdir(parents=True) + (tmp_path / "einops" / "src" / "build.toml").touch() + files = [{"filename": "einops/port/port.kdl", "status": "modified"}] + + assert guidance.touched_source_kernels( + files, guidance.kernel_directories(tmp_path) + ) == ["einops"] + + +def test_detects_new_ported_kernel_from_added_manifest(): + files = [{"filename": "new-kernel/src/build.toml", "status": "added"}] + + assert guidance.touched_source_kernels(files, set()) == ["new-kernel"] + + def test_detects_new_kernel_from_added_manifest(): files = [ {"filename": "new-kernel/build.toml", "status": "added"}, diff --git a/.github/workflows/build-release-dispatch.yaml b/.github/workflows/build-release-dispatch.yaml index c67b72a4a..cc09b4faa 100644 --- a/.github/workflows/build-release-dispatch.yaml +++ b/.github/workflows/build-release-dispatch.yaml @@ -40,7 +40,11 @@ jobs: KERNEL: ${{ inputs.kernel_name }} run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - if [ -d "$KERNEL" ] && [ -f "$KERNEL/flake.nix" ] && [ -f "$KERNEL/build.toml" ]; then + # Ported kernels keep build.toml in /src; the flake stays at + # and points nix at it. + CONFIG="$KERNEL" + [ -f "$KERNEL/src/build.toml" ] && CONFIG="$KERNEL/src" + if [ -d "$KERNEL" ] && [ -f "$KERNEL/flake.nix" ] && [ -f "$CONFIG/build.toml" ]; then echo "kernel=$KERNEL" >> $GITHUB_OUTPUT echo "skip=false" >> $GITHUB_OUTPUT else diff --git a/.github/workflows/check-ports.yaml b/.github/workflows/check-ports.yaml new file mode 100644 index 000000000..b4e57ced7 --- /dev/null +++ b/.github/workflows/check-ports.yaml @@ -0,0 +1,50 @@ +name: Check Ports + +# Verify committed src trees match their port recipes. PRs check changed ports; +# checker changes, pushes to main, scheduled runs and manual runs check all ports. +on: + push: + branches: [main] + # Catch kernel-port regressions even when no port changes. + schedule: + - cron: "0 6 * * *" + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - "*/port/**" + - "*/src/**" + - "scripts/check_ports.py" + - ".github/workflows/check-ports.yaml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-ports: + name: Check ports reproduce + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + + - name: Regenerate selected ports and compare + env: + # Refresh the runner to catch upstream changes. + # TODO: switch back to main once huggingface/kernels#877 is merged. + KERNEL_PORT: "nix run --refresh github:huggingface/kernels/kernel-port-no-importlib#kernel-port --" + BASE_REF: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || '' }} + run: | + if [ -n "$BASE_REF" ]; then + python3 scripts/check_ports.py --changed-since "$BASE_REF" + else + python3 scripts/check_ports.py + fi diff --git a/.gitignore b/.gitignore index 4dcaf8670..c9216b6cf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ build # Editor files .* !.github/ +!/.gitattributes # Python stuff __pycache__/ diff --git a/einops/flake.nix b/einops/flake.nix index 0b2a92f1e..70f28b83d 100644 --- a/einops/flake.nix +++ b/einops/flake.nix @@ -1,5 +1,5 @@ { - description = "Flake for eigen kernel"; + description = "Flake for einops kernel"; inputs = { kernel-builder.url = "github:huggingface/kernels"; @@ -12,6 +12,6 @@ }: kernel-builder.lib.genKernelFlakeOutputs { inherit self; - path = ./.; + path = ./src; }; } diff --git a/einops/port/overlay/tests/conftest.py b/einops/port/overlay/tests/conftest.py new file mode 100644 index 000000000..3fe86a864 --- /dev/null +++ b/einops/port/overlay/tests/conftest.py @@ -0,0 +1,4 @@ +import os + +# Use the CI backends by default, while allowing local overrides. +os.environ.setdefault("EINOPS_TEST_BACKENDS", "torch,numpy") diff --git a/einops/port/port.kdl b/einops/port/port.kdl new file mode 100644 index 000000000..89ac6b50b --- /dev/null +++ b/einops/port/port.kdl @@ -0,0 +1,78 @@ +recipe version=1 + +// Pin the exact clean upstream tree this recipe transforms. +source \ + repo="https://github.com/arogozhnikov/einops.git" \ + commit="ad2c8d67d08cbebb45e1aff2be21dc88635fc695" + +// Retain only the Python package and its upstream tests. +prune keep="einops/**" + +// Lift upstream tests out of the package into the kernel test root. +move \ + from="einops/tests" \ + to="tests" + +// Place the distributable package in the torch extension tree. +move \ + from="einops" \ + to="torch-ext/einops" + +// Make package-internal imports work under the Hub-generated module name. +relativize_imports \ + in="torch-ext/einops/**" \ + package_root="torch-ext/einops" + +// Expose array_api because Hub consumers cannot import it by package name. +ensure_import \ + in="torch-ext/einops/__init__.py" \ + from="." \ + name="array_api" + +// Point imports of the former einops.tests package at the lifted tests. +remap_module \ + in="tests/{__init__,test_*}.py" \ + from="einops.tests" \ + to="tests" + +// Make imports between lifted test modules package-relative. +relativize_imports \ + in="tests/{__init__,test_*}.py" \ + package_root="tests" + +// Resolve product imports through the locally built Hub kernel. The package +// init binds it privately so `from . import *` cannot pick it up. +kernelize_imports \ + in="tests/__init__.py" \ + package="einops" \ + binding="_einops" \ + kernel="kernels-community/einops" \ + version=1 + +kernelize_imports \ + in="tests/test_*.py" \ + package="einops" \ + kernel="kernels-community/einops" \ + version=1 + +// Default to Torch and NumPy tests, with an EINOPS_TEST_BACKENDS override. +overlay from="overlay" + +// Use the supported replacement for NumPy 2's removed in1d API. +replace in="tests/test_layers.py" \ + find="numpy.in1d(grad, [0, 1])" \ + with="numpy.isin(grad, [0, 1])" + +// Select every upstream test for the kernel CI suite. +mark_tests in="tests/test_*.py" marker="kernels_ci" changes=63 + +// Emit the noarch kernel manifest and supported runtime backends. +manifest \ + name="einops" \ + version=1 \ + edition=5 \ + license="MIT" \ + upstream="https://github.com/arogozhnikov/einops.git" \ + backends="cpu,cuda,metal,rocm,xpu" \ + repo_id="kernels-community/einops" \ + noarch=#true diff --git a/einops/build.toml b/einops/src/build.toml similarity index 72% rename from einops/build.toml rename to einops/src/build.toml index fcf94d562..5fd31b09b 100644 --- a/einops/build.toml +++ b/einops/src/build.toml @@ -4,7 +4,13 @@ version = 1 edition = 5 license = "MIT" upstream = "https://github.com/arogozhnikov/einops.git" -backends = ["cpu", "cuda", "metal", "rocm", "xpu"] +backends = [ + "cpu", + "cuda", + "metal", + "rocm", + "xpu", +] [general.hub] repo-id = "kernels-community/einops" diff --git a/einops/src/port-provenance.json b/einops/src/port-provenance.json new file mode 100644 index 000000000..c683484df --- /dev/null +++ b/einops/src/port-provenance.json @@ -0,0 +1,12 @@ +{ + "format": 1, + "recipe": "sha256:238127312e85c9489a67de689d3b2e59c75c456f6839afb640f344ef0e7965d5", + "runner": "kernel-port 0.1.0", + "sources": { + "upstream": { + "repo": "https://github.com/arogozhnikov/einops.git", + "commit": "ad2c8d67d08cbebb45e1aff2be21dc88635fc695" + } + }, + "tree": "sha256:418031b9de86c73cadcf391017cb3f666fd816137449513b481f4af2caa6fe53" +} diff --git a/einops/tests/__init__.py b/einops/src/tests/__init__.py similarity index 82% rename from einops/tests/__init__.py rename to einops/src/tests/__init__.py index b8c629217..8972f0707 100644 --- a/einops/tests/__init__.py +++ b/einops/src/tests/__init__.py @@ -46,23 +46,15 @@ def unparse_backends(backend_names: List[str]) -> Tuple[str, str]: @lru_cache(maxsize=1) def parse_backends_to_test() -> List[str]: - if ENVVAR_NAME in os.environ: - parsed_backends = os.environ[ENVVAR_NAME].split(",") - _known_backends = find_names_of_all_frameworks() - for backend_name in parsed_backends: - if backend_name not in _known_backends: - raise RuntimeError(f"Unknown framework: {backend_name}") - return parsed_backends - - # Auto-detect available frameworks. - available = [] - for module, name in [("torch", "torch"), ("jax", "jax"), ("numpy", "numpy"), ("tensorflow", "tensorflow")]: - try: - __import__(module) - available.append(name) - except ImportError: - pass - return available + if ENVVAR_NAME not in os.environ: + raise RuntimeError(f"Testing frameworks were not specified, env var {ENVVAR_NAME} not set") + parsed_backends = os.environ[ENVVAR_NAME].split(",") + _known_backends = find_names_of_all_frameworks() + for backend_name in parsed_backends: + if backend_name not in _known_backends: + raise RuntimeError(f"Unknown framework: {backend_name}") + + return parsed_backends def is_backend_tested(backend: str) -> bool: diff --git a/einops/src/tests/conftest.py b/einops/src/tests/conftest.py new file mode 100644 index 000000000..3fe86a864 --- /dev/null +++ b/einops/src/tests/conftest.py @@ -0,0 +1,4 @@ +import os + +# Use the CI backends by default, while allowing local overrides. +os.environ.setdefault("EINOPS_TEST_BACKENDS", "torch,numpy") diff --git a/einops/tests/run_tests.py b/einops/src/tests/run_tests.py similarity index 100% rename from einops/tests/run_tests.py rename to einops/src/tests/run_tests.py diff --git a/einops/tests/test_einsum.py b/einops/src/tests/test_einsum.py similarity index 99% rename from einops/tests/test_einsum.py rename to einops/src/tests/test_einsum.py index 2e05d939d..addf117b3 100644 --- a/einops/tests/test_einsum.py +++ b/einops/src/tests/test_einsum.py @@ -1,10 +1,10 @@ from typing import Any, Callable +from . import collect_test_backends import kernels einops = kernels.get_kernel("kernels-community/einops", version=1) _compactify_pattern_for_einsum = einops.einops._compactify_pattern_for_einsum einsum = einops.einops.einsum -EinopsError = einops.EinopsError -from . import collect_test_backends +EinopsError = einops.einops.EinopsError import numpy as np import pytest import string diff --git a/einops/tests/test_examples.py b/einops/src/tests/test_examples.py similarity index 100% rename from einops/tests/test_examples.py rename to einops/src/tests/test_examples.py diff --git a/einops/tests/test_layers.py b/einops/src/tests/test_layers.py similarity index 99% rename from einops/tests/test_layers.py rename to einops/src/tests/test_layers.py index 538b3b8e1..eab53c908 100644 --- a/einops/tests/test_layers.py +++ b/einops/src/tests/test_layers.py @@ -9,8 +9,7 @@ rearrange = einops.rearrange reduce = einops.reduce EinopsError = einops.EinopsError -from . import collect_test_backends, is_backend_tested -from . import FLOAT_REDUCTIONS as REDUCTIONS +from . import collect_test_backends, is_backend_tested, FLOAT_REDUCTIONS as REDUCTIONS __author__ = "Alex Rogozhnikov" diff --git a/einops/tests/test_ops.py b/einops/src/tests/test_ops.py similarity index 98% rename from einops/tests/test_ops.py rename to einops/src/tests/test_ops.py index ff668bb50..1873524e1 100644 --- a/einops/tests/test_ops.py +++ b/einops/src/tests/test_ops.py @@ -11,8 +11,7 @@ reduce = einops.einops.reduce repeat = einops.einops.repeat _enumerate_directions = einops.einops._enumerate_directions -from . import collect_test_backends, is_backend_tested -from . import FLOAT_REDUCTIONS as REDUCTIONS +from . import collect_test_backends, is_backend_tested, FLOAT_REDUCTIONS as REDUCTIONS imp_op_backends = collect_test_backends(symbolic=False, layers=False) sym_op_backends = collect_test_backends(symbolic=True, layers=False) @@ -370,10 +369,7 @@ def logsumexp_numpy(x, tuple_of_axes): y = numpy.sum(y, axis=tuple_of_axes) return numpy.log(y) + minused - TorchBackend = einops._backends.TorchBackend - TensorflowBackend = einops._backends.TensorflowBackend - TFKerasBackend = einops._backends.TFKerasBackend - NumpyBackend = einops._backends.NumpyBackend + TorchBackend, TensorflowBackend, TFKerasBackend, NumpyBackend = einops._backends.TorchBackend, einops._backends.TensorflowBackend, einops._backends.TFKerasBackend, einops._backends.NumpyBackend backend2callback = { TorchBackend.framework_name: logsumexp_torch, diff --git a/einops/tests/test_other.py b/einops/src/tests/test_other.py similarity index 100% rename from einops/tests/test_other.py rename to einops/src/tests/test_other.py diff --git a/einops/tests/test_packing.py b/einops/src/tests/test_packing.py similarity index 100% rename from einops/tests/test_packing.py rename to einops/src/tests/test_packing.py diff --git a/einops/tests/test_parsing.py b/einops/src/tests/test_parsing.py similarity index 100% rename from einops/tests/test_parsing.py rename to einops/src/tests/test_parsing.py diff --git a/einops/torch-ext/einops/__init__.py b/einops/src/torch-ext/einops/__init__.py similarity index 100% rename from einops/torch-ext/einops/__init__.py rename to einops/src/torch-ext/einops/__init__.py diff --git a/einops/torch-ext/einops/_backends.py b/einops/src/torch-ext/einops/_backends.py similarity index 100% rename from einops/torch-ext/einops/_backends.py rename to einops/src/torch-ext/einops/_backends.py diff --git a/einops/torch-ext/einops/_torch_specific.py b/einops/src/torch-ext/einops/_torch_specific.py similarity index 100% rename from einops/torch-ext/einops/_torch_specific.py rename to einops/src/torch-ext/einops/_torch_specific.py diff --git a/einops/torch-ext/einops/array_api.py b/einops/src/torch-ext/einops/array_api.py similarity index 100% rename from einops/torch-ext/einops/array_api.py rename to einops/src/torch-ext/einops/array_api.py diff --git a/einops/torch-ext/einops/einops.py b/einops/src/torch-ext/einops/einops.py similarity index 100% rename from einops/torch-ext/einops/einops.py rename to einops/src/torch-ext/einops/einops.py diff --git a/einops/torch-ext/einops/experimental/__init__.py b/einops/src/torch-ext/einops/experimental/__init__.py similarity index 100% rename from einops/torch-ext/einops/experimental/__init__.py rename to einops/src/torch-ext/einops/experimental/__init__.py diff --git a/einops/torch-ext/einops/experimental/indexing.py b/einops/src/torch-ext/einops/experimental/indexing.py similarity index 100% rename from einops/torch-ext/einops/experimental/indexing.py rename to einops/src/torch-ext/einops/experimental/indexing.py diff --git a/einops/torch-ext/einops/layers/__init__.py b/einops/src/torch-ext/einops/layers/__init__.py similarity index 100% rename from einops/torch-ext/einops/layers/__init__.py rename to einops/src/torch-ext/einops/layers/__init__.py diff --git a/einops/torch-ext/einops/layers/_einmix.py b/einops/src/torch-ext/einops/layers/_einmix.py similarity index 100% rename from einops/torch-ext/einops/layers/_einmix.py rename to einops/src/torch-ext/einops/layers/_einmix.py diff --git a/einops/torch-ext/einops/layers/flax.py b/einops/src/torch-ext/einops/layers/flax.py similarity index 100% rename from einops/torch-ext/einops/layers/flax.py rename to einops/src/torch-ext/einops/layers/flax.py diff --git a/einops/torch-ext/einops/layers/keras.py b/einops/src/torch-ext/einops/layers/keras.py similarity index 100% rename from einops/torch-ext/einops/layers/keras.py rename to einops/src/torch-ext/einops/layers/keras.py diff --git a/einops/torch-ext/einops/layers/oneflow.py b/einops/src/torch-ext/einops/layers/oneflow.py similarity index 100% rename from einops/torch-ext/einops/layers/oneflow.py rename to einops/src/torch-ext/einops/layers/oneflow.py diff --git a/einops/torch-ext/einops/layers/paddle.py b/einops/src/torch-ext/einops/layers/paddle.py similarity index 100% rename from einops/torch-ext/einops/layers/paddle.py rename to einops/src/torch-ext/einops/layers/paddle.py diff --git a/einops/torch-ext/einops/layers/tensorflow.py b/einops/src/torch-ext/einops/layers/tensorflow.py similarity index 100% rename from einops/torch-ext/einops/layers/tensorflow.py rename to einops/src/torch-ext/einops/layers/tensorflow.py diff --git a/einops/torch-ext/einops/layers/torch.py b/einops/src/torch-ext/einops/layers/torch.py similarity index 100% rename from einops/torch-ext/einops/layers/torch.py rename to einops/src/torch-ext/einops/layers/torch.py diff --git a/einops/torch-ext/einops/packing.py b/einops/src/torch-ext/einops/packing.py similarity index 100% rename from einops/torch-ext/einops/packing.py rename to einops/src/torch-ext/einops/packing.py diff --git a/einops/torch-ext/einops/parsing.py b/einops/src/torch-ext/einops/parsing.py similarity index 100% rename from einops/torch-ext/einops/parsing.py rename to einops/src/torch-ext/einops/parsing.py diff --git a/einops/torch-ext/einops/py.typed b/einops/src/torch-ext/einops/py.typed similarity index 100% rename from einops/torch-ext/einops/py.typed rename to einops/src/torch-ext/einops/py.typed diff --git a/scripts/check_ports.py b/scripts/check_ports.py new file mode 100644 index 000000000..d6dd27644 --- /dev/null +++ b/scripts/check_ports.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +import filecmp +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# Extract source arguments; kernel-port validates them. +SOURCE_RE = re.compile( + r"^source\b(?P(?:[^\n\\]|\\\s*\n)*)", re.MULTILINE +) +ARG_RE = re.compile(r'(\w+)="([^"]*)"') + +DEFAULT_RUNNER = "kernel-port" + +CHECKER_FILES = {"scripts/check_ports.py", ".github/workflows/check-ports.yaml"} + + +def runner_cmd() -> list: + return shlex.split(os.environ.get("KERNEL_PORT", DEFAULT_RUNNER)) + + +def find_ports(root: Path) -> list: + return sorted(p.parent.parent.name for p in root.glob("*/port/port.kdl")) + + +def changed_ports(root: Path, ref: str) -> list: + """Select changed ports, or all ports when the checker or workflow changes.""" + out = subprocess.run( + ["git", "diff", "--name-only", f"{ref}...HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout + touched = set() + for line in out.splitlines(): + if line in CHECKER_FILES: + return find_ports(root) + parts = line.split("/") + if len(parts) >= 3 and parts[1] in ("port", "src"): + touched.add(parts[0]) + return sorted(k for k in find_ports(root) if k in touched) + + +def parse_source(recipe: Path) -> tuple: + text = recipe.read_text() + m = SOURCE_RE.search(text) + if not m: + raise SystemExit(f"{recipe}: no `source` op found") + args = dict(ARG_RE.findall(m.group("body").replace("\\\n", " "))) + try: + return args["repo"], args["commit"] + except KeyError as exc: + raise SystemExit(f"{recipe}: source is missing {exc}") from exc + + +def checkout(repo: str, commit: str, dest: Path) -> None: + subprocess.run( + ["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout", repo, str(dest)], + check=True, + ) + subprocess.run(["git", "-C", str(dest), "checkout", "--quiet", commit], check=True) + + +def tree_files(root: Path) -> set: + return { + p.relative_to(root).as_posix() + for p in root.rglob("*") + if p.is_file() and "__pycache__" not in p.parts + } + + +def compare(generated: Path, committed: Path) -> list: + gen, com = tree_files(generated), tree_files(committed) + problems = [] + for rel in sorted(com - gen): + problems.append(f"committed but not generated: {rel}") + for rel in sorted(gen - com): + problems.append(f"generated but not committed: {rel}") + for rel in sorted(gen & com): + if not filecmp.cmp(generated / rel, committed / rel, shallow=False): + problems.append(f"differs: {rel}") + return problems + + +def check(kernel: str, root: Path, workdir: Path) -> list: + recipe = root / kernel / "port" / "port.kdl" + repo, commit = parse_source(recipe) + print(f" {kernel}: {repo} @ {commit[:12]}") + + # Reuse checkouts for kernels sharing an upstream commit. + src = workdir / f"upstream-{commit[:12]}" + if not src.exists(): + checkout(repo, commit, src) + + out = workdir / f"out-{kernel}" + shutil.rmtree(out, ignore_errors=True) + proc = subprocess.run( + [*runner_cmd(), str(recipe), "--dir", str(src), "--out", str(out)], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + tail = (proc.stderr or proc.stdout).strip().splitlines()[-8:] + return [f"kernel-port failed:\n " + "\n ".join(tail)] + + return compare(out, root / kernel / "src") + + +def main() -> int: + root = Path.cwd() + args = sys.argv[1:] + if args[:1] == ["--changed-since"]: + if len(args) != 2: + raise SystemExit("usage: check_ports.py --changed-since ") + kernels = changed_ports(root, args[1]) + if not kernels: + print(f"No ported kernels changed since {args[1]}.") + return 0 + else: + kernels = args or find_ports(root) + if not kernels: + print("No ported kernels found (no */port/port.kdl).") + return 0 + + print(f"Checking {len(kernels)} port(s) reproduce their committed tree:\n") + failed = {} + with tempfile.TemporaryDirectory() as tmp: + workdir = Path(tmp) + for kernel in kernels: + if not (root / kernel / "port" / "port.kdl").is_file(): + failed[kernel] = [f"no port/port.kdl under {kernel}/"] + continue + problems = check(kernel, root, workdir) + if problems: + failed[kernel] = problems + else: + print(" ok - regenerates byte for byte") + + if failed: + print("\nERROR: committed tree does not match the recipe.\n", file=sys.stderr) + for kernel, problems in failed.items(): + print(f" {kernel}:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + print( + "\nRegenerate with:\n" + " kernel-port /port/port.kdl --dir --out /src\n" + "If an overlay file went stale, refresh it from /src first.\n" + "If only port-provenance.json differs, the recipe changed and the\n" + "port was not rerun.", + file=sys.stderr, + ) + return 1 + + print("\nAll ports reproduce their committed tree.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_kernels_checks.py b/scripts/run_kernels_checks.py index 39a75b2e1..ca854d43c 100644 --- a/scripts/run_kernels_checks.py +++ b/scripts/run_kernels_checks.py @@ -57,7 +57,10 @@ def discover_kernel_dirs(root: Path, excludes: list[str]) -> list[str]: name = entry.name if name in filtered: continue - elif entry.is_dir() and not (entry / "build.toml").exists(): + # Ported kernels keep build.toml in /src. + elif entry.is_dir() and not ( + (entry / "build.toml").exists() or (entry / "src" / "build.toml").exists() + ): logging.debug(f"Skipping {name} because it doesn't contain a `build.toml`.") continue if entry.is_dir():