feat: prefer recipe ported einops

.gitattributes

@@ -0,0 +1,8 @@
+# Ported kernels are generated: <kernel>/port/port.kdl plus <kernel>/port/overlay
+# are the inputs, and <kernel>/src is kernel-port's output, committed so that
+# what ships stays reviewable and auditable.
+#
+# Collapse the generated tree in diffs so review lands on the inputs. The output
+# is not taken on trust: .github/workflows/check-ports.yaml regenerates every
+# port and fails if <kernel>/src does not match byte for byte.
+*/src/** linguist-generated=true

.github/scripts/check_kernel_test_versions.py

@@ -40,7 +40,9 @@ def discover_kernels(repository_root: Path) -> list[Kernel]:
         if not manifest.is_file():
             # Ported kernels keep generated sources and build.toml under src/.
             manifest = root / "src" / "build.toml"
-        if not manifest.is_file() or not (root / "tests").is_dir():
+        # A ported kernel generates its tests next to build.toml in src/.
+        tests_root = manifest.parent if (manifest.parent / "tests").is_dir() else root
+        if not manifest.is_file() or not (tests_root / "tests").is_dir():
             continue
 
         with manifest.open("rb") as handle:
@@ -52,7 +54,7 @@ def discover_kernels(repository_root: Path) -> list[Kernel]:
             raise ValueError(f"{manifest}: [general.hub].repo-id must be a string")
         if type(version) is not int:
             raise ValueError(f"{manifest}: [general].version must be an integer")
-        kernels.append(Kernel(root=root, repo_id=repo_id, version=version))
+        kernels.append(Kernel(root=tests_root, repo_id=repo_id, version=version))
     return kernels
 
 

.github/scripts/kernel_maintainers.py

@@ -22,7 +22,9 @@ def load_registry() -> dict:
 
 def check_registry() -> None:
     registry = load_registry()
+    # Ported kernels keep build.toml in <kernel>/src.
     kernels = {path.parent.name for path in Path(".").glob("*/build.toml")}
+    kernels |= {path.parent.parent.name for path in Path(".").glob("*/src/build.toml")}
     registered = set(registry)
 
     problems = [

.github/scripts/pr_kernel_build_guidance.py

@@ -43,7 +43,10 @@ def paginate(url: str, token: str) -> list[dict]:
 
 
 def kernel_directories(root: Path) -> set[str]:
-    return {manifest.parent.name for manifest in root.glob("*/build.toml")}
+    # Ported kernels keep build.toml in <kernel>/src.
+    return {manifest.parent.name for manifest in root.glob("*/build.toml")} | {
+        manifest.parent.parent.name for manifest in root.glob("*/src/build.toml")
+    }
 
 
 def is_documentation(path: PurePosixPath) -> bool:
@@ -55,7 +58,8 @@ def touched_source_kernels(files: list[dict], existing_kernels: set[str]) -> lis
     new_kernels = {
         path.parts[0]
         for path in paths
-        if len(path.parts) == 2 and path.name == "build.toml"
+        if path.name == "build.toml"
+        and (len(path.parts) == 2 or (len(path.parts) == 3 and path.parts[1] == "src"))
     }
     kernels = existing_kernels | new_kernels
     touched = set(new_kernels)

.github/scripts/test_check_kernel_test_versions.py

@@ -46,6 +46,21 @@ def test_matching_import_with_nested_manifest(tmp_path):
     assert checker.check_repository(tmp_path) == ([], 1, 1)
 
 
+def test_ported_kernel_tests_under_src(tmp_path):
+    kernel = tmp_path / "example"
+    (kernel / "src" / "tests").mkdir(parents=True)
+    (kernel / "src" / "build.toml").write_text(
+        '[general]\nversion = 2\n\n[general.hub]\nrepo-id = "kernels-community/example"\n'
+    )
+    (kernel / "src" / "tests" / "test_example.py").write_text(
+        "from kernels import get_kernel\n"
+        'example = get_kernel("kernels-community/example", version=1)\n'
+    )
+
+    problems, _, _ = checker.check_repository(tmp_path)
+    assert len(problems) == 1
+
+
 def test_matching_call_is_formatting_agnostic(tmp_path):
     kernel = write_kernel(tmp_path)
     (kernel / "tests" / "test_example.py").write_text(

.github/scripts/test_pr_kernel_build_guidance.py

@@ -39,6 +39,22 @@ def test_unknown_extensions_and_test_changes_are_build_relevant(tmp_path):
     ) == ["relu"]
 
 
+def test_detects_ported_kernel_source(tmp_path):
+    (tmp_path / "einops" / "src").mkdir(parents=True)
+    (tmp_path / "einops" / "src" / "build.toml").touch()
+    files = [{"filename": "einops/port/port.kdl", "status": "modified"}]
+
+    assert guidance.touched_source_kernels(
+        files, guidance.kernel_directories(tmp_path)
+    ) == ["einops"]
+
+
+def test_detects_new_ported_kernel_from_added_manifest():
+    files = [{"filename": "new-kernel/src/build.toml", "status": "added"}]
+
+    assert guidance.touched_source_kernels(files, set()) == ["new-kernel"]
+
+
 def test_detects_new_kernel_from_added_manifest():
     files = [
         {"filename": "new-kernel/build.toml", "status": "added"},

.github/workflows/build-release-dispatch.yaml

@@ -40,7 +40,11 @@ jobs:
           KERNEL: ${{ inputs.kernel_name }}
         run: |
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            if [ -d "$KERNEL" ] && [ -f "$KERNEL/flake.nix" ] && [ -f "$KERNEL/build.toml" ]; then
+            # Ported kernels keep build.toml in <kernel>/src; the flake stays at
+            # <kernel> and points nix at it.
+            CONFIG="$KERNEL"
+            [ -f "$KERNEL/src/build.toml" ] && CONFIG="$KERNEL/src"
+            if [ -d "$KERNEL" ] && [ -f "$KERNEL/flake.nix" ] && [ -f "$CONFIG/build.toml" ]; then
               echo "kernel=$KERNEL" >> $GITHUB_OUTPUT
               echo "skip=false" >> $GITHUB_OUTPUT
             else

.github/workflows/check-ports.yaml

@@ -0,0 +1,50 @@
+name: Check Ports
+
+# Verify committed src trees match their port recipes. PRs check changed ports;
+# checker changes, pushes to main, scheduled runs and manual runs check all ports.
+on:
+  push:
+    branches: [main]
+  # Catch kernel-port regressions even when no port changes.
+  schedule:
+    - cron: "0 6 * * *"
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - "*/port/**"
+      - "*/src/**"
+      - "scripts/check_ports.py"
+      - ".github/workflows/check-ports.yaml"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  check-ports:
+    name: Check ports reproduce
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22
+
+      - name: Regenerate selected ports and compare
+        env:
+          # Refresh the runner to catch upstream changes.
+          # TODO: switch back to main once huggingface/kernels#877 is merged.
+          KERNEL_PORT: "nix run --refresh github:huggingface/kernels/kernel-port-no-importlib#kernel-port --"
+          BASE_REF: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || '' }}
+        run: |
+          if [ -n "$BASE_REF" ]; then
+            python3 scripts/check_ports.py --changed-since "$BASE_REF"
+          else
+            python3 scripts/check_ports.py
+          fi

.gitignore

@@ -15,6 +15,7 @@ build
 # Editor files
 .*
 !.github/
+!/.gitattributes
 
 # Python stuff
 __pycache__/

einops/flake.nix

@@ -1,5 +1,5 @@
 {
-  description = "Flake for eigen kernel";
+  description = "Flake for einops kernel";
 
   inputs = {
     kernel-builder.url = "github:huggingface/kernels";
@@ -12,6 +12,6 @@
     }:
     kernel-builder.lib.genKernelFlakeOutputs {
       inherit self;
-      path = ./.;
+      path = ./src;
     };
 }

einops/port/overlay/tests/conftest.py

@@ -0,0 +1,4 @@
+import os
+
+# Use the CI backends by default, while allowing local overrides.
+os.environ.setdefault("EINOPS_TEST_BACKENDS", "torch,numpy")

einops/port/port.kdl

@@ -0,0 +1,78 @@
+recipe version=1
+
+// Pin the exact clean upstream tree this recipe transforms.
+source \
+    repo="https://github.com/arogozhnikov/einops.git" \
+    commit="ad2c8d67d08cbebb45e1aff2be21dc88635fc695"
+
+// Retain only the Python package and its upstream tests.
+prune keep="einops/**"
+
+// Lift upstream tests out of the package into the kernel test root.
+move \
+    from="einops/tests" \
+    to="tests"
+
+// Place the distributable package in the torch extension tree.
+move \
+    from="einops" \
+    to="torch-ext/einops"
+
+// Make package-internal imports work under the Hub-generated module name.
+relativize_imports \
+    in="torch-ext/einops/**" \
+    package_root="torch-ext/einops"
+
+// Expose array_api because Hub consumers cannot import it by package name.
+ensure_import \
+    in="torch-ext/einops/__init__.py" \
+    from="." \
+    name="array_api"
+
+// Point imports of the former einops.tests package at the lifted tests.
+remap_module \
+    in="tests/{__init__,test_*}.py" \
+    from="einops.tests" \
+    to="tests"
+
+// Make imports between lifted test modules package-relative.
+relativize_imports \
+    in="tests/{__init__,test_*}.py" \
+    package_root="tests"
+
+// Resolve product imports through the locally built Hub kernel. The package
+// init binds it privately so `from . import *` cannot pick it up.
+kernelize_imports \
+    in="tests/__init__.py" \
+    package="einops" \
+    binding="_einops" \
+    kernel="kernels-community/einops" \
+    version=1
+
+kernelize_imports \
+    in="tests/test_*.py" \
+    package="einops" \
+    kernel="kernels-community/einops" \
+    version=1
+
+// Default to Torch and NumPy tests, with an EINOPS_TEST_BACKENDS override.
+overlay from="overlay"
+
+// Use the supported replacement for NumPy 2's removed in1d API.
+replace in="tests/test_layers.py" \
+    find="numpy.in1d(grad, [0, 1])" \
+    with="numpy.isin(grad, [0, 1])"
+
+// Select every upstream test for the kernel CI suite.
+mark_tests in="tests/test_*.py" marker="kernels_ci" changes=63
+
+// Emit the noarch kernel manifest and supported runtime backends.
+manifest \
+    name="einops" \
+    version=1 \
+    edition=5 \
+    license="MIT" \
+    upstream="https://github.com/arogozhnikov/einops.git" \
+    backends="cpu,cuda,metal,rocm,xpu" \
+    repo_id="kernels-community/einops" \
+    noarch=#true