Skip to content

ci: drop X64 from runs-on of arch-neutral jobs (workspace-test-shard, mg-reuse, x86-main-advisories) #5871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
noahgift wants to merge 4 commits into from
Closed

ci: drop X64 from runs-on of arch-neutral jobs (workspace-test-shard, mg-reuse, x86-main-advisories) #5871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
74f7f06
ci: drop X64 from runs-on of arch-neutral workspace-test-shard, mg-re…
noahgift Oct 21, 2026
2d8f310
test(kreg): per-arch parity bounds -- receipts carry per_arch.<arch>.…
noahgift Oct 21, 2026
74f033f
test(kreg): q6_k receipt -- add per_arch without reformatting existin…
noahgift Oct 21, 2026
006b0a4
test(kreg): aarch64 per_arch bounds for the other nine cpu.matvec rec…
noahgift Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
# its own. It runs on push only; on merge_group it is skipped at t=0, which the verdict blocks
# below read as `skipped:::skipped` = "x86-main + determinism ran".
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
runs-on: [self-hosted, Linux, X64, clean-room]
runs-on: [self-hosted, Linux, clean-room]
timeout-minutes: 10
permissions:
contents: read
Expand Down Expand Up @@ -314,7 +314,7 @@ jobs:
x86-main-advisories:
needs: [mg-reuse]
if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' }}
runs-on: [self-hosted, Linux, X64, clean-room]
runs-on: [self-hosted, Linux, clean-room]
timeout-minutes: 30
permissions:
contents: read
Expand Down Expand Up @@ -437,11 +437,11 @@ jobs:
# failure, never a pass.
# ==========================================================================
# The workspace tests, one shard per job (nextest hash partition N/12 plus the
# explicit runner's round-robin N/12), in parallel on the X64 clean-room pool.
# explicit runner's round-robin N/12), in parallel on the clean-room pool (arch-neutral, PMAT-3138).
# The section body is ci/sections.yml `workspace-test-shard`, unchanged.
workspace-test-shard:
# same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry
runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }}
# clean-room for pull_request/push, mq for a merge_group entry; no X64 pin (operator 2026-10-21 14:18Z)
runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","mq"]') || fromJSON('["self-hosted","Linux","clean-room"]') }}
timeout-minutes: 180
strategy:
fail-fast: false
Expand Down
22 changes: 21 additions & 1 deletion crates/aprender-serve/src/kernel_registry_parity.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,7 @@ fn committed() -> Vec<(serde_json::Value, serde_json::Value)> {
#[test]
fn committed_parity_receipts_hold_on_this_host() {
let r = registry().expect("registry");
let mut no_arch_receipt = Vec::new();
for (entry, rc) in committed() {
let id = rc["kernel_id"].as_str().expect("kernel_id");
let path = entry["receipt"].as_str().expect("receipt path");
Expand Down Expand Up @@ -789,8 +790,22 @@ fn committed_parity_receipts_hold_on_this_host() {
seed: wl["seed"].as_u64().expect("seed"),
trials: dim("trials"),
};
let bound = rc["tolerance_rel"].as_f64().expect("tolerance_rel");
let now = measure(kernel(id), w, false);
// A receipt's `tolerance_rel` is the bound measured on its own `host_arch`; libm and FMA
// contraction move the last ulps between architectures. Any other arch must carry its own
// measured bound under `per_arch.<arch>.tolerance_rel` -- never a borrowed one, never a skip.
let bound = if rc["host_arch"] == host_arch() {
rc["tolerance_rel"].as_f64().expect("tolerance_rel")
} else if let Some(b) = rc["per_arch"][host_arch()]["tolerance_rel"].as_f64() {
b
} else {
no_arch_receipt.push(format!(
"{path}: no {} receipt (host measured max_rel_err {})",
host_arch(),
now.max_rel_err
));
continue;
};
assert_eq!(
rc["margin"]["verdict"], "pass",
"{path}: KTEST-02 margin verdict (max ≤ 1 under EM-DOT)"
Expand Down Expand Up @@ -842,6 +857,11 @@ fn committed_parity_receipts_hold_on_this_host() {
p => panic!("{id}: no precision check for {p}"),
}
}
assert!(
no_arch_receipt.is_empty(),
"committed receipts lack a bound for this arch:\n{}",
no_arch_receipt.join("\n")
);
}

/// `KREG_INPUT_SETS_OUT`: the input-set hashes recomputed from the tree at `KREG_GIT_SHA`, one
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.apr_q4.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@
},
"oracle": "apr_format_decode_f64",
"oracle_independent": true,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 5.568608963671898e-07,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 2e-6,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.apr_q8.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@
},
"oracle": "apr_format_decode_f64",
"oracle_independent": true,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 5.982009671170112e-07,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 2e-6,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.bf16.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,15 @@
},
"oracle": "le_decode_half_f64",
"oracle_independent": true,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 4.068146011742945e-07,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 9e-7,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.f16.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,15 @@
},
"oracle": "le_decode_half_f64",
"oracle_independent": true,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 3.920968817889049e-07,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 8e-7,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.f32.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,15 @@
},
"oracle": "le_decode_half_f64",
"oracle_independent": true,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 7.356022484897627e-07,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 2e-6,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.q4_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@
},
"oracle": "in_tree_dequant_f64",
"oracle_independent": false,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 0.003859434297246209,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 0.008,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": {
"max_rel_err": 1.69613194525447e-7
},
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.q4_k.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@
},
"oracle": "in_tree_dequant_f64",
"oracle_independent": false,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 0.005569836531516333,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 0.02,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": {
"max_rel_err": 1.3720446505396093e-7
},
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.q5_k.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@
},
"oracle": "in_tree_dequant_f64",
"oracle_independent": false,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 1.3110594897372703e-06,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 3e-6,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.q6_k.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@
},
"oracle": "in_tree_dequant_f64",
"oracle_independent": false,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 1.6505971099429934e-6,
"source": "CI job 119280981403 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 4e-6,
"ts": "2026-10-21T14:37:43Z"
}
},
"quantized_activation_oracle": null,
"registry_precision": "f32",
"schema": "kernel-parity-receipt/v1",
Expand Down
9 changes: 9 additions & 0 deletions evidence/kreg/parity/cpu.matvec.q8_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@
},
"oracle": "in_tree_dequant_f64",
"oracle_independent": false,
"per_arch": {
"aarch64": {
"host": "gx10-pool1",
"max_rel_err": 0.003738839019401381,
"source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload",
"tolerance_rel": 0.008,
"ts": "2026-10-21T14:51:19Z"
}
},
"quantized_activation_oracle": {
"max_rel_err": 2.0240994602880252e-7
},
Expand Down
7 changes: 4 additions & 3 deletions scripts/ci_mg_reuse.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,11 +528,12 @@ STUB
# Each row is compared as the EXACT line (trailing comment stripped), so a third state cannot slip in.
poolrows() { # poolrows WF -> one line per violated pool row
local j l want
local plain="runs-on: [self-hosted, Linux, X64, clean-room]"
local plain="runs-on: [self-hosted, Linux, clean-room]" # arch-neutral job: cargo-deny, no X64 pin
local lane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"clean-room\"]') }}"
local nlane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"clean-room\"]') }}"
for j in x86-main x86-main-advisories workspace-test-shard; do
l=$(awk -v j=" $j:" '$0 == j {f = 1; next} f && /^ [A-Za-z0-9_-]+:$/ {exit} f' "$1" 2> /dev/null | grep -m1 '^ runs-on:' | sed -E 's/^ +//; s/ +#.*$//')
want=$plain; case $j in x86-main | workspace-test-shard) want=$lane ;; esac
want=$plain; case $j in x86-main) want=$lane ;; workspace-test-shard) want=$nlane ;; esac
if [ "$l" != "$want" ]; then
echo "$j does not land on the clean-room pool (mq on merge_group for x86-main/workspace-test-shard) (runs-on: $l)"; fi
done
Expand All @@ -552,7 +553,7 @@ STUB
poolmut pool-advisories-event-conditional "does not land on the clean-room pool" '/^ x86-main-advisories:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: ${{ github.event_name == '"'"'merge_group'"'"' \&\& fromJSON('"'"'["clean-room"]'"'"') || fromJSON('"'"'["clean-room"]'"'"') }}/}'
poolmut mq-lane-lost-on-x86-main "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"mq"/"clean-room"/}'
poolmut mq-lane-dropped-on-shard "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: [self-hosted, Linux, X64, clean-room]/}'
poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","X64","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","X64","mq"]'"'"')/}'
poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","mq"]'"'"')/}'
poolmut mq-lane-third-label "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"X64","mq"/"X64","mq","pr-only"/}'

# S1 (20X): mg-reuse runs on push only, so on merge_group it is SKIPPED and every dependent
Expand Down
Loading