diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bf828a5b4..dd6da6de22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: # its own. It runs on push only; on merge_group it is skipped at t=0, which the verdict blocks # below read as `skipped:::skipped` = "x86-main + determinism ran". if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 10 permissions: contents: read @@ -314,7 +314,7 @@ jobs: x86-main-advisories: needs: [mg-reuse] if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' }} - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 30 permissions: contents: read @@ -437,11 +437,11 @@ jobs: # failure, never a pass. # ========================================================================== # The workspace tests, one shard per job (nextest hash partition N/12 plus the - # explicit runner's round-robin N/12), in parallel on the X64 clean-room pool. + # explicit runner's round-robin N/12), in parallel on the clean-room pool (arch-neutral, PMAT-3138). # The section body is ci/sections.yml `workspace-test-shard`, unchanged. workspace-test-shard: - # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry - runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} + # clean-room for pull_request/push, mq for a merge_group entry; no X64 pin (operator 2026-10-21 14:18Z) + runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","mq"]') || fromJSON('["self-hosted","Linux","clean-room"]') }} timeout-minutes: 180 strategy: fail-fast: false diff --git a/crates/aprender-serve/src/kernel_registry_parity.rs b/crates/aprender-serve/src/kernel_registry_parity.rs index 0f47c86e85..bdb6e3a798 100644 --- a/crates/aprender-serve/src/kernel_registry_parity.rs +++ b/crates/aprender-serve/src/kernel_registry_parity.rs @@ -743,6 +743,7 @@ fn committed() -> Vec<(serde_json::Value, serde_json::Value)> { #[test] fn committed_parity_receipts_hold_on_this_host() { let r = registry().expect("registry"); + let mut no_arch_receipt = Vec::new(); for (entry, rc) in committed() { let id = rc["kernel_id"].as_str().expect("kernel_id"); let path = entry["receipt"].as_str().expect("receipt path"); @@ -789,8 +790,22 @@ fn committed_parity_receipts_hold_on_this_host() { seed: wl["seed"].as_u64().expect("seed"), trials: dim("trials"), }; - let bound = rc["tolerance_rel"].as_f64().expect("tolerance_rel"); let now = measure(kernel(id), w, false); + // A receipt's `tolerance_rel` is the bound measured on its own `host_arch`; libm and FMA + // contraction move the last ulps between architectures. Any other arch must carry its own + // measured bound under `per_arch..tolerance_rel` -- never a borrowed one, never a skip. + let bound = if rc["host_arch"] == host_arch() { + rc["tolerance_rel"].as_f64().expect("tolerance_rel") + } else if let Some(b) = rc["per_arch"][host_arch()]["tolerance_rel"].as_f64() { + b + } else { + no_arch_receipt.push(format!( + "{path}: no {} receipt (host measured max_rel_err {})", + host_arch(), + now.max_rel_err + )); + continue; + }; assert_eq!( rc["margin"]["verdict"], "pass", "{path}: KTEST-02 margin verdict (max ≤ 1 under EM-DOT)" @@ -842,6 +857,11 @@ fn committed_parity_receipts_hold_on_this_host() { p => panic!("{id}: no precision check for {p}"), } } + assert!( + no_arch_receipt.is_empty(), + "committed receipts lack a bound for this arch:\n{}", + no_arch_receipt.join("\n") + ); } /// `KREG_INPUT_SETS_OUT`: the input-set hashes recomputed from the tree at `KREG_GIT_SHA`, one diff --git a/evidence/kreg/parity/cpu.matvec.apr_q4.json b/evidence/kreg/parity/cpu.matvec.apr_q4.json index e7929dfd71..da9c1fc945 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q4.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q4.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.568608963671898e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.apr_q8.json b/evidence/kreg/parity/cpu.matvec.apr_q8.json index e95fb4c5c9..9f6f4ac2cf 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q8.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q8.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.982009671170112e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.bf16.json b/evidence/kreg/parity/cpu.matvec.bf16.json index e6aa4240c0..57309f51c6 100644 --- a/evidence/kreg/parity/cpu.matvec.bf16.json +++ b/evidence/kreg/parity/cpu.matvec.bf16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 4.068146011742945e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 9e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f16.json b/evidence/kreg/parity/cpu.matvec.f16.json index 17b3754d5a..af25b8ed9d 100644 --- a/evidence/kreg/parity/cpu.matvec.f16.json +++ b/evidence/kreg/parity/cpu.matvec.f16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 3.920968817889049e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 8e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f32.json b/evidence/kreg/parity/cpu.matvec.f32.json index 9926666b94..a41e599001 100644 --- a/evidence/kreg/parity/cpu.matvec.f32.json +++ b/evidence/kreg/parity/cpu.matvec.f32.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 7.356022484897627e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q4_0.json b/evidence/kreg/parity/cpu.matvec.q4_0.json index 38ab583a76..d1a94f3c46 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_0.json +++ b/evidence/kreg/parity/cpu.matvec.q4_0.json @@ -35,6 +35,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003859434297246209, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.69613194525447e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q4_k.json b/evidence/kreg/parity/cpu.matvec.q4_k.json index 501b8c768a..0e1ccc3ba7 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_k.json +++ b/evidence/kreg/parity/cpu.matvec.q4_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.005569836531516333, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.02, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.3720446505396093e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q5_k.json b/evidence/kreg/parity/cpu.matvec.q5_k.json index 01913cf231..d65c8a5ca9 100644 --- a/evidence/kreg/parity/cpu.matvec.q5_k.json +++ b/evidence/kreg/parity/cpu.matvec.q5_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.3110594897372703e-06, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 3e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q6_k.json b/evidence/kreg/parity/cpu.matvec.q6_k.json index 220f53b4a7..ce60279cc6 100644 --- a/evidence/kreg/parity/cpu.matvec.q6_k.json +++ b/evidence/kreg/parity/cpu.matvec.q6_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.6505971099429934e-6, + "source": "CI job 119280981403 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 4e-6, + "ts": "2026-10-21T14:37:43Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q8_0.json b/evidence/kreg/parity/cpu.matvec.q8_0.json index b69febb636..0f6306509e 100644 --- a/evidence/kreg/parity/cpu.matvec.q8_0.json +++ b/evidence/kreg/parity/cpu.matvec.q8_0.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003738839019401381, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 2.0240994602880252e-7 }, diff --git a/scripts/ci_mg_reuse.sh b/scripts/ci_mg_reuse.sh index 86573f5aff..b87174df96 100755 --- a/scripts/ci_mg_reuse.sh +++ b/scripts/ci_mg_reuse.sh @@ -528,11 +528,12 @@ STUB # Each row is compared as the EXACT line (trailing comment stripped), so a third state cannot slip in. poolrows() { # poolrows WF -> one line per violated pool row local j l want - local plain="runs-on: [self-hosted, Linux, X64, clean-room]" + local plain="runs-on: [self-hosted, Linux, clean-room]" # arch-neutral job: cargo-deny, no X64 pin local lane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"clean-room\"]') }}" + local nlane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"clean-room\"]') }}" for j in x86-main x86-main-advisories workspace-test-shard; do l=$(awk -v j=" $j:" '$0 == j {f = 1; next} f && /^ [A-Za-z0-9_-]+:$/ {exit} f' "$1" 2> /dev/null | grep -m1 '^ runs-on:' | sed -E 's/^ +//; s/ +#.*$//') - want=$plain; case $j in x86-main | workspace-test-shard) want=$lane ;; esac + want=$plain; case $j in x86-main) want=$lane ;; workspace-test-shard) want=$nlane ;; esac if [ "$l" != "$want" ]; then echo "$j does not land on the clean-room pool (mq on merge_group for x86-main/workspace-test-shard) (runs-on: $l)"; fi done @@ -552,7 +553,7 @@ STUB poolmut pool-advisories-event-conditional "does not land on the clean-room pool" '/^ x86-main-advisories:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: ${{ github.event_name == '"'"'merge_group'"'"' \&\& fromJSON('"'"'["clean-room"]'"'"') || fromJSON('"'"'["clean-room"]'"'"') }}/}' poolmut mq-lane-lost-on-x86-main "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"mq"/"clean-room"/}' poolmut mq-lane-dropped-on-shard "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: [self-hosted, Linux, X64, clean-room]/}' - poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","X64","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","X64","mq"]'"'"')/}' + poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","mq"]'"'"')/}' poolmut mq-lane-third-label "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"X64","mq"/"X64","mq","pr-only"/}' # S1 (20X): mg-reuse runs on push only, so on merge_group it is SKIPPED and every dependent