From 74f7f06e6cc5fd2b9a08429df973b278f2469164 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 16:09:59 +0200 Subject: [PATCH 1/4] ci: drop X64 from runs-on of arch-neutral workspace-test-shard, mg-reuse, x86-main-advisories (Refs #5736) Operator 14:18Z via aprender-57. Audit: handoff/x64-pin-audit-1021.md. x86-main stays X64 (jq-amd64 pin, determinism[X64]). scripts/ci_mg_reuse.sh pool-row self-test updated to match. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 10 +++++----- scripts/ci_mg_reuse.sh | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bf828a5b4..dd6da6de22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: # its own. It runs on push only; on merge_group it is skipped at t=0, which the verdict blocks # below read as `skipped:::skipped` = "x86-main + determinism ran". if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 10 permissions: contents: read @@ -314,7 +314,7 @@ jobs: x86-main-advisories: needs: [mg-reuse] if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' }} - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 30 permissions: contents: read @@ -437,11 +437,11 @@ jobs: # failure, never a pass. # ========================================================================== # The workspace tests, one shard per job (nextest hash partition N/12 plus the - # explicit runner's round-robin N/12), in parallel on the X64 clean-room pool. + # explicit runner's round-robin N/12), in parallel on the clean-room pool (arch-neutral, PMAT-3138). # The section body is ci/sections.yml `workspace-test-shard`, unchanged. workspace-test-shard: - # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry - runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} + # clean-room for pull_request/push, mq for a merge_group entry; no X64 pin (operator 2026-10-21 14:18Z) + runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","mq"]') || fromJSON('["self-hosted","Linux","clean-room"]') }} timeout-minutes: 180 strategy: fail-fast: false diff --git a/scripts/ci_mg_reuse.sh b/scripts/ci_mg_reuse.sh index 86573f5aff..b87174df96 100755 --- a/scripts/ci_mg_reuse.sh +++ b/scripts/ci_mg_reuse.sh @@ -528,11 +528,12 @@ STUB # Each row is compared as the EXACT line (trailing comment stripped), so a third state cannot slip in. poolrows() { # poolrows WF -> one line per violated pool row local j l want - local plain="runs-on: [self-hosted, Linux, X64, clean-room]" + local plain="runs-on: [self-hosted, Linux, clean-room]" # arch-neutral job: cargo-deny, no X64 pin local lane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"clean-room\"]') }}" + local nlane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"clean-room\"]') }}" for j in x86-main x86-main-advisories workspace-test-shard; do l=$(awk -v j=" $j:" '$0 == j {f = 1; next} f && /^ [A-Za-z0-9_-]+:$/ {exit} f' "$1" 2> /dev/null | grep -m1 '^ runs-on:' | sed -E 's/^ +//; s/ +#.*$//') - want=$plain; case $j in x86-main | workspace-test-shard) want=$lane ;; esac + want=$plain; case $j in x86-main) want=$lane ;; workspace-test-shard) want=$nlane ;; esac if [ "$l" != "$want" ]; then echo "$j does not land on the clean-room pool (mq on merge_group for x86-main/workspace-test-shard) (runs-on: $l)"; fi done @@ -552,7 +553,7 @@ STUB poolmut pool-advisories-event-conditional "does not land on the clean-room pool" '/^ x86-main-advisories:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: ${{ github.event_name == '"'"'merge_group'"'"' \&\& fromJSON('"'"'["clean-room"]'"'"') || fromJSON('"'"'["clean-room"]'"'"') }}/}' poolmut mq-lane-lost-on-x86-main "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"mq"/"clean-room"/}' poolmut mq-lane-dropped-on-shard "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: [self-hosted, Linux, X64, clean-room]/}' - poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","X64","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","X64","mq"]'"'"')/}' + poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","mq"]'"'"')/}' poolmut mq-lane-third-label "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"X64","mq"/"X64","mq","pr-only"/}' # S1 (20X): mg-reuse runs on push only, so on merge_group it is SKIPPED and every dependent From 2d8f3103c49a36b94faa3eb543826dca80c39502 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 16:44:22 +0200 Subject: [PATCH 2/4] test(kreg): per-arch parity bounds -- receipts carry per_arch..tolerance_rel; q6_k aarch64 measured on gx10 (Refs #5736) The shard moved to the arch-neutral pool and cpu.matvec.q6_k read 1.65e-6 on aarch64 against the x86_64 receipt's 4e-7. The test now takes the bound from the receipt's own host_arch, else from per_arch., and otherwise fails listing every receipt that lacks one with the host's measured error. No skip, no borrowed bound. Co-Authored-By: Claude Sonnet 5.5 --- .../src/kernel_registry_parity.rs | 22 ++++++++++++++++++- evidence/kreg/parity/cpu.matvec.q6_k.json | 19 +++++++++++----- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/aprender-serve/src/kernel_registry_parity.rs b/crates/aprender-serve/src/kernel_registry_parity.rs index 0f47c86e85..bdb6e3a798 100644 --- a/crates/aprender-serve/src/kernel_registry_parity.rs +++ b/crates/aprender-serve/src/kernel_registry_parity.rs @@ -743,6 +743,7 @@ fn committed() -> Vec<(serde_json::Value, serde_json::Value)> { #[test] fn committed_parity_receipts_hold_on_this_host() { let r = registry().expect("registry"); + let mut no_arch_receipt = Vec::new(); for (entry, rc) in committed() { let id = rc["kernel_id"].as_str().expect("kernel_id"); let path = entry["receipt"].as_str().expect("receipt path"); @@ -789,8 +790,22 @@ fn committed_parity_receipts_hold_on_this_host() { seed: wl["seed"].as_u64().expect("seed"), trials: dim("trials"), }; - let bound = rc["tolerance_rel"].as_f64().expect("tolerance_rel"); let now = measure(kernel(id), w, false); + // A receipt's `tolerance_rel` is the bound measured on its own `host_arch`; libm and FMA + // contraction move the last ulps between architectures. Any other arch must carry its own + // measured bound under `per_arch..tolerance_rel` -- never a borrowed one, never a skip. + let bound = if rc["host_arch"] == host_arch() { + rc["tolerance_rel"].as_f64().expect("tolerance_rel") + } else if let Some(b) = rc["per_arch"][host_arch()]["tolerance_rel"].as_f64() { + b + } else { + no_arch_receipt.push(format!( + "{path}: no {} receipt (host measured max_rel_err {})", + host_arch(), + now.max_rel_err + )); + continue; + }; assert_eq!( rc["margin"]["verdict"], "pass", "{path}: KTEST-02 margin verdict (max ≤ 1 under EM-DOT)" @@ -842,6 +857,11 @@ fn committed_parity_receipts_hold_on_this_host() { p => panic!("{id}: no precision check for {p}"), } } + assert!( + no_arch_receipt.is_empty(), + "committed receipts lack a bound for this arch:\n{}", + no_arch_receipt.join("\n") + ); } /// `KREG_INPUT_SETS_OUT`: the input-set hashes recomputed from the tree at `KREG_GIT_SHA`, one diff --git a/evidence/kreg/parity/cpu.matvec.q6_k.json b/evidence/kreg/parity/cpu.matvec.q6_k.json index 220f53b4a7..96b5bede3b 100644 --- a/evidence/kreg/parity/cpu.matvec.q6_k.json +++ b/evidence/kreg/parity/cpu.matvec.q6_k.json @@ -1,8 +1,8 @@ { "build_identity": "5c9341007d7ea1ebeff29c0aad3133311dc4d8e2", "fp32_activations": { - "max_abs_err": 0.00006229749033082044, - "max_rel_err": 1.939624445869695e-7 + "max_abs_err": 6.229749033082044e-05, + "max_rel_err": 1.939624445869695e-07 }, "host": "mac-server", "host_arch": "x86_64", @@ -36,15 +36,24 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.6505971099429934e-06, + "source": "CI job 119280981403 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 4e-06, + "ts": "2026-10-21T14:37:43Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", "served": { - "max_abs_err": 0.00006229749033082044, - "max_rel_err": 1.939624445869695e-7 + "max_abs_err": 6.229749033082044e-05, + "max_rel_err": 1.939624445869695e-07 }, "source_fn": "fused_q6k_parallel_matvec", - "tolerance_rel": 4e-7, + "tolerance_rel": 4e-07, "ts": "2026-10-04T10:31:05Z", "workload": { "in_dim": 2048, From 74f033f4939d4fe627eb16db07db663543c73671 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 16:44:54 +0200 Subject: [PATCH 3/4] test(kreg): q6_k receipt -- add per_arch without reformatting existing numbers (Refs #5736) Co-Authored-By: Claude Sonnet 5.5 --- evidence/kreg/parity/cpu.matvec.q6_k.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/evidence/kreg/parity/cpu.matvec.q6_k.json b/evidence/kreg/parity/cpu.matvec.q6_k.json index 96b5bede3b..ce60279cc6 100644 --- a/evidence/kreg/parity/cpu.matvec.q6_k.json +++ b/evidence/kreg/parity/cpu.matvec.q6_k.json @@ -1,8 +1,8 @@ { "build_identity": "5c9341007d7ea1ebeff29c0aad3133311dc4d8e2", "fp32_activations": { - "max_abs_err": 6.229749033082044e-05, - "max_rel_err": 1.939624445869695e-07 + "max_abs_err": 0.00006229749033082044, + "max_rel_err": 1.939624445869695e-7 }, "host": "mac-server", "host_arch": "x86_64", @@ -39,9 +39,9 @@ "per_arch": { "aarch64": { "host": "gx10-pool1", - "max_rel_err": 1.6505971099429934e-06, + "max_rel_err": 1.6505971099429934e-6, "source": "CI job 119280981403 (PR #5871 shard 1), same seed and workload", - "tolerance_rel": 4e-06, + "tolerance_rel": 4e-6, "ts": "2026-10-21T14:37:43Z" } }, @@ -49,11 +49,11 @@ "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", "served": { - "max_abs_err": 6.229749033082044e-05, - "max_rel_err": 1.939624445869695e-07 + "max_abs_err": 0.00006229749033082044, + "max_rel_err": 1.939624445869695e-7 }, "source_fn": "fused_q6k_parallel_matvec", - "tolerance_rel": 4e-07, + "tolerance_rel": 4e-7, "ts": "2026-10-04T10:31:05Z", "workload": { "in_dim": 2048, From 006b0a4f6a83d6fc06f24861c0891432fc23894e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Wed, 21 Oct 2026 16:52:25 +0200 Subject: [PATCH 4/4] test(kreg): aarch64 per_arch bounds for the other nine cpu.matvec receipts, measured on gx10 CI (Refs #5736) Bound = tolerance_from(measured) (2x, rounded up one digit), the rule the x86_64 receipts used. Co-Authored-By: Claude Sonnet 5.5 --- evidence/kreg/parity/cpu.matvec.apr_q4.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.apr_q8.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.bf16.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.f16.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.f32.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.q4_0.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.q4_k.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.q5_k.json | 9 +++++++++ evidence/kreg/parity/cpu.matvec.q8_0.json | 9 +++++++++ 9 files changed, 81 insertions(+) diff --git a/evidence/kreg/parity/cpu.matvec.apr_q4.json b/evidence/kreg/parity/cpu.matvec.apr_q4.json index e7929dfd71..da9c1fc945 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q4.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q4.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.568608963671898e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.apr_q8.json b/evidence/kreg/parity/cpu.matvec.apr_q8.json index e95fb4c5c9..9f6f4ac2cf 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q8.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q8.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.982009671170112e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.bf16.json b/evidence/kreg/parity/cpu.matvec.bf16.json index e6aa4240c0..57309f51c6 100644 --- a/evidence/kreg/parity/cpu.matvec.bf16.json +++ b/evidence/kreg/parity/cpu.matvec.bf16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 4.068146011742945e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 9e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f16.json b/evidence/kreg/parity/cpu.matvec.f16.json index 17b3754d5a..af25b8ed9d 100644 --- a/evidence/kreg/parity/cpu.matvec.f16.json +++ b/evidence/kreg/parity/cpu.matvec.f16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 3.920968817889049e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 8e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f32.json b/evidence/kreg/parity/cpu.matvec.f32.json index 9926666b94..a41e599001 100644 --- a/evidence/kreg/parity/cpu.matvec.f32.json +++ b/evidence/kreg/parity/cpu.matvec.f32.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 7.356022484897627e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q4_0.json b/evidence/kreg/parity/cpu.matvec.q4_0.json index 38ab583a76..d1a94f3c46 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_0.json +++ b/evidence/kreg/parity/cpu.matvec.q4_0.json @@ -35,6 +35,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003859434297246209, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.69613194525447e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q4_k.json b/evidence/kreg/parity/cpu.matvec.q4_k.json index 501b8c768a..0e1ccc3ba7 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_k.json +++ b/evidence/kreg/parity/cpu.matvec.q4_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.005569836531516333, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.02, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.3720446505396093e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q5_k.json b/evidence/kreg/parity/cpu.matvec.q5_k.json index 01913cf231..d65c8a5ca9 100644 --- a/evidence/kreg/parity/cpu.matvec.q5_k.json +++ b/evidence/kreg/parity/cpu.matvec.q5_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.3110594897372703e-06, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 3e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q8_0.json b/evidence/kreg/parity/cpu.matvec.q8_0.json index b69febb636..0f6306509e 100644 --- a/evidence/kreg/parity/cpu.matvec.q8_0.json +++ b/evidence/kreg/parity/cpu.matvec.q8_0.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003738839019401381, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 2.0240994602880252e-7 },