diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 905ae731c2..7d3fa6f001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: # its own. It runs on push only; on merge_group it is skipped at t=0, which the verdict blocks # below read as `skipped:::skipped` = "x86-main + determinism ran". if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 10 permissions: contents: read @@ -316,7 +316,7 @@ jobs: x86-main-advisories: needs: [mg-reuse] if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' }} - runs-on: [self-hosted, Linux, X64, clean-room] + runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 30 permissions: contents: read @@ -439,12 +439,35 @@ jobs: # failure, never a pass. # ========================================================================== # The workspace tests, one shard per job (nextest hash partition N/12 plus the - # explicit runner's round-robin N/12), in parallel on the X64 clean-room pool. + # explicit runner's round-robin N/12), in parallel on the clean-room pool (arch-neutral, PMAT-3138). # The section body is ci/sections.yml `workspace-test-shard`, unchanged. + # #5882: which tier this run owes decides WHERE the shards may run. The full tier's Σ-executed fan-in + # compares every shard with ONE universe list, and arch-gated tests (target_arch cfgs) differ per + # arch, so a full tier needs one architecture: X64. The quick/PR tier owes no Σ and stays + # arch-neutral. The tier is only knowable from the diff, so it is decided HERE, by the same script + # the shards run (scripts/ci/decide_tier.sh), and workspace-test-shard reads it in its runs-on. + # scripts/ci/sigma_arch_uniform.sh in the fan-in is the backstop: a mixed full tier is RED. + workspace-test-arch: + runs-on: [self-hosted, Linux, clean-room] + timeout-minutes: 15 + outputs: + tier: ${{ steps.tier.outputs.tier }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Decide the test tier (the shards' own step, one script) + id: tier + env: + GH_TOKEN: ${{ github.token }} + MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} + run: bash scripts/ci/decide_tier.sh workspace-test-shard: + needs: [workspace-test-arch] if: ${{ github.event.pull_request.draft != true }} - # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry - runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} + # arch-neutral (operator 2026-10-21 14:18Z) EXCEPT the full tier, which needs one arch for the Σ (#5882, workspace-test-arch) + # clean-room for pull_request/push, mq for a merge_group entry + runs-on: ${{ needs.workspace-test-arch.outputs.tier == 'full' && (github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]')) || (github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","mq"]') || fromJSON('["self-hosted","Linux","clean-room"]')) }} timeout-minutes: 180 strategy: fail-fast: false @@ -523,6 +546,7 @@ jobs: set -euo pipefail d="$RUNNER_TEMP/sigma" python3 scripts/ci/sigma_executed.py --self-test + bash scripts/ci/sigma_arch_uniform.sh --self-test # ONE shard count: the pins in ci/sections.yml (check_shard_count_single_source.sh) SHARDS="$(awk -v ev="$GITHUB_EVENT_NAME" '/^matrix-pins:/{p=1;next} p&&/^ workspace-test-shard:/{k="pr";next} p&&/^ workspace-test-shard-merge_group:/{k="mg";next} p&&/^ [a-z]/{k=""} k&&/^ - \{shard:/{c[k]++} END{print (ev=="merge_group"&&c["mg"]?c["mg"]:c["pr"])+0}' ci/sections.yml)" [ "${SHARDS:-0}" -ge 1 ] 2>/dev/null || { echo "::error::Σ: no workspace-test-shard pins"; exit 1; } @@ -539,6 +563,8 @@ jobs: [ "$#" -eq "$SHARDS" ] || { echo "::error::Σ: $# tier files for $SHARDS shards:$tiers"; exit 1; } for t in "$@"; do [ "$t" = "$1" ] || { echo "::error::Σ: the shards disagree on the tier:$tiers"; exit 1; }; done [ "$1" = full ] || { echo "every shard ran tier '$1': the lib/explicit Σ is not owed"; exit 0; } + # #5882: a full tier on two architectures is RED (arch-gated tests, one universe): placement is workspace-test-arch's, this is the backstop + bash scripts/ci/sigma_arch_uniform.sh "$d" "$SHARDS" j=() l=() for n in $(seq 1 "$SHARDS"); do j+=(--junit "$d/sigma-shard-$n/lib.junit.xml") diff --git a/ci/sections.yml b/ci/sections.yml index 4cc17744ef..4632770e37 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -378,66 +378,7 @@ jobs: GH_TOKEN: ${{ github.token }} MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} run: | - set -euo pipefail - args=(--event "${GITHUB_EVENT_NAME}") - # PMAT-1098 67-E2: the queue and a push to main re-derive their own - # diff from HEAD's FIRST PARENT (HEAD^1..HEAD is the PR's diff on the - # new base). actions/checkout is shallow, so that parent is not in the - # object store until we deepen -- and without it ci_test_tier.sh falls - # closed to `full` every single time, which would make the whole - # optimisation inert while looking like it worked. Best effort by - # design: if the deepen fails the decision still falls closed, and the - # warning below says so instead of the run silently paying the hour. - case "${GITHUB_EVENT_NAME}" in - merge_group) - # Operator 2026-10-17: the queue runs the quick tier over the entry's diff (base_sha = prior entry's head), - # MG_BASE_SHA..HEAD. Fetch the base; if that fails the tier falls closed to full. - bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${GITHUB_SHA}" "${MG_BASE_SHA}" 2>/dev/null \ - || bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${MG_BASE_SHA}" 2>/dev/null \ - || true - git cat-file -e "${MG_BASE_SHA}^{commit}" 2>/dev/null \ - || printf '::warning::merge_group base sha unavailable -- the tier falls closed to full\n' - ;; - push) - bash scripts/ci/fetch_p6.sh --no-tags --depth=2 origin "${GITHUB_SHA}" 2>/dev/null \ - || bash scripts/ci/fetch_p6.sh --no-tags --deepen=1 origin 2>/dev/null \ - || true - if git rev-parse -q --verify 'HEAD^1' >/dev/null 2>&1; then - printf 'first parent available: %s\n' "$(git rev-parse HEAD^1)" - else - printf '::warning::first parent unavailable after deepen -- the tier falls closed to full\n' - fi - ;; - esac - case "${GITHUB_EVENT_NAME}" in - pull_request) - bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" - # #5391: the PR's change is the diff from merge-base(base, HEAD), not the - # base tip -- main's later commits are not the PR's. --no-renames (#3664) - # lives in the helper. - bash scripts/ci/pr_touched_paths.sh "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" - args+=(--diff-from "$RUNNER_TEMP/touched.txt") - ;; - merge_group) - # Operator 2026-10-17 (cop window), verbatim: "only gating on tags/release". - # ci_test_tier.sh diffs MG_BASE_SHA..HEAD (env above) and selects like a PR. - ;; - push) - # T36: the merge queue already tested this very sha. Hand its - # workspace-test result to the tier script, which reuses it only - # on a success for HEAD itself. A failed lookup is the push's own - # tier, as before -- never a reuse, never a red step. - mg=$(bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}") || mg="" - if [ -n "$mg" ]; then - mapfile -t mgargs <<<"$mg" - args+=("${mgargs[@]}") - else - printf 'no merge_group workspace-test result for %s: the tier is decided from the push diff\n' "${GITHUB_SHA}" - fi - ;; - esac - bash scripts/ci_test_tier.sh "${args[@]}" | tee "$RUNNER_TEMP/tier.txt" - grep -E '^(tier|crates|targets|check_workspace|reason|cite)=' "$RUNNER_TEMP/tier.txt" >> "$GITHUB_OUTPUT" + bash scripts/ci/decide_tier.sh # #4502 row 2: lint_passes_on_real_contracts (aprender-contracts) requires the # refinement gate to RUN whenever CI is set, and that gate compares HEAD with # merge-base(HEAD, origin/main), else the origin/main tip. This section clones at @@ -774,6 +715,7 @@ jobs: sig="$RUNNER_TEMP/sigma" mkdir -p sigma-shard printf '%s\n' "$TIER" > sigma-shard/tier + uname -m > sigma-shard/arch if [ "$TIER" = full ]; then cp "$sig/lib.junit.xml" sigma-shard/lib.junit.xml cp "${FAT_SECTION_LOG:?the fat driver sets FAT_SECTION_LOG}" sigma-shard/section.log diff --git a/crates/aprender-serve/src/kernel_registry_parity.rs b/crates/aprender-serve/src/kernel_registry_parity.rs index 0f47c86e85..bdb6e3a798 100644 --- a/crates/aprender-serve/src/kernel_registry_parity.rs +++ b/crates/aprender-serve/src/kernel_registry_parity.rs @@ -743,6 +743,7 @@ fn committed() -> Vec<(serde_json::Value, serde_json::Value)> { #[test] fn committed_parity_receipts_hold_on_this_host() { let r = registry().expect("registry"); + let mut no_arch_receipt = Vec::new(); for (entry, rc) in committed() { let id = rc["kernel_id"].as_str().expect("kernel_id"); let path = entry["receipt"].as_str().expect("receipt path"); @@ -789,8 +790,22 @@ fn committed_parity_receipts_hold_on_this_host() { seed: wl["seed"].as_u64().expect("seed"), trials: dim("trials"), }; - let bound = rc["tolerance_rel"].as_f64().expect("tolerance_rel"); let now = measure(kernel(id), w, false); + // A receipt's `tolerance_rel` is the bound measured on its own `host_arch`; libm and FMA + // contraction move the last ulps between architectures. Any other arch must carry its own + // measured bound under `per_arch..tolerance_rel` -- never a borrowed one, never a skip. + let bound = if rc["host_arch"] == host_arch() { + rc["tolerance_rel"].as_f64().expect("tolerance_rel") + } else if let Some(b) = rc["per_arch"][host_arch()]["tolerance_rel"].as_f64() { + b + } else { + no_arch_receipt.push(format!( + "{path}: no {} receipt (host measured max_rel_err {})", + host_arch(), + now.max_rel_err + )); + continue; + }; assert_eq!( rc["margin"]["verdict"], "pass", "{path}: KTEST-02 margin verdict (max ≤ 1 under EM-DOT)" @@ -842,6 +857,11 @@ fn committed_parity_receipts_hold_on_this_host() { p => panic!("{id}: no precision check for {p}"), } } + assert!( + no_arch_receipt.is_empty(), + "committed receipts lack a bound for this arch:\n{}", + no_arch_receipt.join("\n") + ); } /// `KREG_INPUT_SETS_OUT`: the input-set hashes recomputed from the tree at `KREG_GIT_SHA`, one diff --git a/evidence/kreg/parity/cpu.matvec.apr_q4.json b/evidence/kreg/parity/cpu.matvec.apr_q4.json index e7929dfd71..da9c1fc945 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q4.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q4.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.568608963671898e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.apr_q8.json b/evidence/kreg/parity/cpu.matvec.apr_q8.json index e95fb4c5c9..9f6f4ac2cf 100644 --- a/evidence/kreg/parity/cpu.matvec.apr_q8.json +++ b/evidence/kreg/parity/cpu.matvec.apr_q8.json @@ -35,6 +35,15 @@ }, "oracle": "apr_format_decode_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 5.982009671170112e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.bf16.json b/evidence/kreg/parity/cpu.matvec.bf16.json index e6aa4240c0..57309f51c6 100644 --- a/evidence/kreg/parity/cpu.matvec.bf16.json +++ b/evidence/kreg/parity/cpu.matvec.bf16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 4.068146011742945e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 9e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f16.json b/evidence/kreg/parity/cpu.matvec.f16.json index 17b3754d5a..af25b8ed9d 100644 --- a/evidence/kreg/parity/cpu.matvec.f16.json +++ b/evidence/kreg/parity/cpu.matvec.f16.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 3.920968817889049e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 8e-7, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.f32.json b/evidence/kreg/parity/cpu.matvec.f32.json index 9926666b94..a41e599001 100644 --- a/evidence/kreg/parity/cpu.matvec.f32.json +++ b/evidence/kreg/parity/cpu.matvec.f32.json @@ -33,6 +33,15 @@ }, "oracle": "le_decode_half_f64", "oracle_independent": true, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 7.356022484897627e-07, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 2e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q4_0.json b/evidence/kreg/parity/cpu.matvec.q4_0.json index 38ab583a76..d1a94f3c46 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_0.json +++ b/evidence/kreg/parity/cpu.matvec.q4_0.json @@ -35,6 +35,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003859434297246209, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.69613194525447e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q4_k.json b/evidence/kreg/parity/cpu.matvec.q4_k.json index 501b8c768a..0e1ccc3ba7 100644 --- a/evidence/kreg/parity/cpu.matvec.q4_k.json +++ b/evidence/kreg/parity/cpu.matvec.q4_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.005569836531516333, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.02, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 1.3720446505396093e-7 }, diff --git a/evidence/kreg/parity/cpu.matvec.q5_k.json b/evidence/kreg/parity/cpu.matvec.q5_k.json index 01913cf231..d65c8a5ca9 100644 --- a/evidence/kreg/parity/cpu.matvec.q5_k.json +++ b/evidence/kreg/parity/cpu.matvec.q5_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.3110594897372703e-06, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 3e-6, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q6_k.json b/evidence/kreg/parity/cpu.matvec.q6_k.json index 220f53b4a7..ce60279cc6 100644 --- a/evidence/kreg/parity/cpu.matvec.q6_k.json +++ b/evidence/kreg/parity/cpu.matvec.q6_k.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 1.6505971099429934e-6, + "source": "CI job 119280981403 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 4e-6, + "ts": "2026-10-21T14:37:43Z" + } + }, "quantized_activation_oracle": null, "registry_precision": "f32", "schema": "kernel-parity-receipt/v1", diff --git a/evidence/kreg/parity/cpu.matvec.q8_0.json b/evidence/kreg/parity/cpu.matvec.q8_0.json index b69febb636..0f6306509e 100644 --- a/evidence/kreg/parity/cpu.matvec.q8_0.json +++ b/evidence/kreg/parity/cpu.matvec.q8_0.json @@ -36,6 +36,15 @@ }, "oracle": "in_tree_dequant_f64", "oracle_independent": false, + "per_arch": { + "aarch64": { + "host": "gx10-pool1", + "max_rel_err": 0.003738839019401381, + "source": "CI job 119289824035 (PR #5871 shard 1), same seed and workload", + "tolerance_rel": 0.008, + "ts": "2026-10-21T14:51:19Z" + } + }, "quantized_activation_oracle": { "max_rel_err": 2.0240994602880252e-7 }, diff --git a/scripts/check_ci_push_reuse.sh b/scripts/check_ci_push_reuse.sh index 0532c5d4e3..a62bb9021d 100755 --- a/scripts/check_ci_push_reuse.sh +++ b/scripts/check_ci_push_reuse.sh @@ -285,10 +285,13 @@ hasnt "D11 pull_request ignores --mg-*: never reuse" "tier=reuse" "$OUT" # -- W: the CI wiring ------------------------------------------------------------------------------ SEC="$ROOT/ci/sections.yml" +DT="$ROOT/scripts/ci/decide_tier.sh" # the tier step body lives in one script (#5882): the section step and workspace-test-arch both run it +want "W0 the section step runs the shared decide_tier.sh" 1 \ + "$(grep -cF "bash scripts/ci/decide_tier.sh" "$SEC")" want "W1 the tier step's push arm calls the lookup for GITHUB_SHA" 1 \ - "$(grep -cF 'bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}"' "$SEC")" + "$(grep -cF 'bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}"' "$DT")" want "W2 a failed lookup is not fatal to the step (it falls to today's tier)" 1 \ - "$(grep -cF '"${GITHUB_SHA}") || mg=""' "$SEC")" + "$(grep -cF '"${GITHUB_SHA}") || mg=""' "$DT")" W3PAT="push) printf 'cited merge_group run: %s" if grep -qF "$W3PAT" "$SEC"; then w3=1; else w3=0; fi want "W3 the reuse step labels a push citation as a merge_group run, not a PR head" 1 "$w3" diff --git a/scripts/ci/decide_tier.sh b/scripts/ci/decide_tier.sh new file mode 100755 index 0000000000..5cfa8a6d5d --- /dev/null +++ b/scripts/ci/decide_tier.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# decide_tier.sh -- which test tier this run owes (BSE-17, PMAT-1077); the one body of the +# "Decide the test tier" step, shared by the workspace-test-shard section and by the +# workspace-test-arch job that places the shards (a full tier needs ONE architecture: the +# cross-shard Σ compares every shard with one universe list). Env: GITHUB_EVENT_NAME, +# GITHUB_SHA, GITHUB_REPOSITORY, GITHUB_BASE_REF, MG_BASE_SHA, GH_TOKEN, RUNNER_TEMP, GITHUB_OUTPUT. +set -euo pipefail +args=(--event "${GITHUB_EVENT_NAME}") +# PMAT-1098 67-E2: the queue and a push to main re-derive their own +# diff from HEAD's FIRST PARENT (HEAD^1..HEAD is the PR's diff on the +# new base). actions/checkout is shallow, so that parent is not in the +# object store until we deepen -- and without it ci_test_tier.sh falls +# closed to `full` every single time, which would make the whole +# optimisation inert while looking like it worked. Best effort by +# design: if the deepen fails the decision still falls closed, and the +# warning below says so instead of the run silently paying the hour. +case "${GITHUB_EVENT_NAME}" in + merge_group) + # Operator 2026-10-17: the queue runs the quick tier over the entry's diff (base_sha = prior entry's head), + # MG_BASE_SHA..HEAD. Fetch the base; if that fails the tier falls closed to full. + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${GITHUB_SHA}" "${MG_BASE_SHA}" 2>/dev/null \ + || bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "${MG_BASE_SHA}" 2>/dev/null \ + || true + git cat-file -e "${MG_BASE_SHA}^{commit}" 2>/dev/null \ + || printf '::warning::merge_group base sha unavailable -- the tier falls closed to full\n' + ;; + push) + bash scripts/ci/fetch_p6.sh --no-tags --depth=2 origin "${GITHUB_SHA}" 2>/dev/null \ + || bash scripts/ci/fetch_p6.sh --no-tags --deepen=1 origin 2>/dev/null \ + || true + if git rev-parse -q --verify 'HEAD^1' >/dev/null 2>&1; then + printf 'first parent available: %s\n' "$(git rev-parse HEAD^1)" + else + printf '::warning::first parent unavailable after deepen -- the tier falls closed to full\n' + fi + ;; +esac +case "${GITHUB_EVENT_NAME}" in + pull_request) + bash scripts/ci/fetch_p6.sh --no-tags --depth=1 origin "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" + # #5391: the PR's change is the diff from merge-base(base, HEAD), not the + # base tip -- main's later commits are not the PR's. --no-renames (#3664) + # lives in the helper. + bash scripts/ci/pr_touched_paths.sh "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" + args+=(--diff-from "$RUNNER_TEMP/touched.txt") + ;; + merge_group) + # Operator 2026-10-17 (cop window), verbatim: "only gating on tags/release". + # ci_test_tier.sh diffs MG_BASE_SHA..HEAD (env above) and selects like a PR. + ;; + push) + # T36: the merge queue already tested this very sha. Hand its + # workspace-test result to the tier script, which reuses it only + # on a success for HEAD itself. A failed lookup is the push's own + # tier, as before -- never a reuse, never a red step. + mg=$(bash scripts/ci_mg_workspace_result.sh "${GITHUB_REPOSITORY}" "${GITHUB_SHA}") || mg="" + if [ -n "$mg" ]; then + mapfile -t mgargs <<<"$mg" + args+=("${mgargs[@]}") + else + printf 'no merge_group workspace-test result for %s: the tier is decided from the push diff\n' "${GITHUB_SHA}" + fi + ;; +esac +bash scripts/ci_test_tier.sh "${args[@]}" | tee "$RUNNER_TEMP/tier.txt" +grep -E '^(tier|crates|targets|check_workspace|reason|cite)=' "$RUNNER_TEMP/tier.txt" >> "$GITHUB_OUTPUT" diff --git a/scripts/ci/sigma_arch_uniform.sh b/scripts/ci/sigma_arch_uniform.sh new file mode 100755 index 0000000000..29b0b7bf26 --- /dev/null +++ b/scripts/ci/sigma_arch_uniform.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# sigma_arch_uniform.sh -- the FULL-tier Σ needs every shard on ONE architecture (#5882). +# +# The fan-in compares the union of every shard's executed ids with ONE universe, the list shard 1 +# made on ITS machine. Tests gated on target_arch (crates/aprender-zram-core/src/zstd/avx2.rs has 18 +# x86_64-only ones, 141 files carry a cfg) exist in one arch's list and not the other's, so a mixed +# x86 + aarch64 shard set is red or green by PLACEMENT, and a test owed on x86 whose hash slice drew +# an aarch64 shard runs nowhere. A per-arch compare cannot fix that: each arch's shards run only +# their own slices, so each arch would be owed tests it was never given. The sound rule is +# placement: ci.yml job workspace-test-arch decides the tier first and pins the full tier's shards to +# X64. This script is the backstop that keeps the rule true after the fact: at tier full, every shard +# must have staged x86_64; a missing, blank or other arch, or a blank tier, is RED. Any other agreed +# tier owes no Σ: not checked. +# +# sigma_arch_uniform.sh DIR SHARDS DIR/sigma-shard-N/{tier,arch}, N = 1..SHARDS +# sigma_arch_uniform.sh --self-test the case table (incl. the mixed placement that went red) +set -euo pipefail + +check() { # check DIR SHARDS + local d=$1 n=$2 i t tier arch first="" tiers="" arches="" + for i in $(seq 1 "$n"); do + [ -f "$d/sigma-shard-$i/tier" ] || { echo "::error::Σ arch: shard $i staged no tier file"; return 1; } + tier=$(tr -d '[:space:]' < "$d/sigma-shard-$i/tier") + [ -n "$tier" ] || { echo "::error::Σ arch: shard $i staged a blank tier (fail closed, never read as 'not full')"; return 1; } + tiers="$tiers $tier" + done + for t in $tiers; do [ "$t" = full ] || { echo "Σ arch: tier '$t' in this run, the lib/explicit Σ is not owed, arch not checked"; return 0; }; done + # Agreement alone is not enough: 12 aarch64 shards agree and avx2.rs's x86-only tests run nowhere. + # The full tier is pinned to X64, so every shard must say x86_64. + for i in $(seq 1 "$n"); do + [ -f "$d/sigma-shard-$i/arch" ] || { echo "::error::Σ arch: shard $i staged no arch at the full tier (a missing arch is RED, never uniform)"; return 1; } + arch=$(tr -d '[:space:]' < "$d/sigma-shard-$i/arch") + [ -n "$arch" ] || { echo "::error::Σ arch: shard $i staged a blank arch at the full tier (fail closed)"; return 1; } + arches="$arches $i=$arch" + if [ "$arch" != x86_64 ]; then + echo "::error::Σ arch: full tier shard $i ran on $arch, not x86_64:$arches" + echo "::error::Σ arch: x86_64-gated tests are owed and would run nowhere; workspace-test-arch must pin the full tier to X64" + return 1 + fi + done + echo "Σ arch: every shard of the full tier ran on x86_64" +} + +self_test() { + local td rows=0 bad=0 want got + td=$(mktemp -d) + mk() { # mk TIER ARCH... -> one shard dir per arch argument ("-" = no arch file) + local t=$1 i=0 a; shift; rm -rf "${td:?}/s"; mkdir -p "$td/s" + for a in "$@"; do + i=$((i + 1)); mkdir -p "$td/s/sigma-shard-$i"; printf '%s\n' "$t" > "$td/s/sigma-shard-$i/tier" + [ "$a" = - ] || printf '%s\n' "$a" > "$td/s/sigma-shard-$i/arch" + done + } + row() { # row WANT LABEL TIER ARCH... + want=$1; local label=$2; shift 2; mk "$@" + set +e; check "$td/s" "$(($# - 1))" > /dev/null 2>&1; got=$?; set -e + rows=$((rows + 1)); [ "$got" = "$want" ] || bad=$((bad + 1)) + printf '%s want_rc=%s got_rc=%s %s\n' "$([ "$got" = "$want" ] && echo 'ok ' || echo 'BAD')" "$want" "$got" "$label" + } + row 0 "full, 12 shards all x86_64" full x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 x86_64 + row 1 "full, 12 shards all aarch64: they agree, but x86-only tests run nowhere" full aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 + row 1 "full, one shard staged a BLANK arch" full x86_64 "" x86_64 + row 1 "full, every shard staged a blank arch" full "" "" + row 1 "full, one shard staged a whitespace-only arch" full x86_64 " " x86_64 + row 1 "a blank tier refuses (never read as 'not full')" "" x86_64 x86_64 + row 1 "full, MIXED: shard 3 x86_64, the rest aarch64 (run 39620701544's placement)" full aarch64 aarch64 x86_64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 aarch64 + row 1 "full, MIXED: shard 1 x86_64, the rest aarch64" full x86_64 aarch64 aarch64 aarch64 + row 1 "full, one shard staged no arch" full x86_64 - x86_64 + row 0 "quick, MIXED: no Σ owed, not checked (the PR tier stays arch-neutral)" quick aarch64 x86_64 aarch64 + mk full x86_64; rm -f "${td:?}/s/sigma-shard-1/tier" + set +e; check "$td/s" 1 > /dev/null 2>&1; got=$?; set -e + rows=$((rows + 1)); [ "$got" = 1 ] || bad=$((bad + 1)) + printf '%s want_rc=1 got_rc=%s a missing tier file refuses\n' "$([ "$got" = 1 ] && echo 'ok ' || echo 'BAD')" "$got" + # Why a per-arch compare is not the fix, on the real comparator: shard 1's list is aarch64 (no + # avx2 test), the x86 shard executed one -> "executed but not in the universe". + local j="$td/j.xml" l="$td/l.json" + printf '' > "$j" + printf '{"rust-suites":{"z":{"testcases":{"a::neon":{"filter-match":{"status":"matches"}}}}}}' > "$l" + set +e; python3 "$(dirname "$0")/sigma_executed.py" nextest --junit "$j" --list-json "$l" > /dev/null 2>&1; got=$?; set -e + rows=$((rows + 1)); [ "$got" = 1 ] || bad=$((bad + 1)) + printf '%s want_rc=1 got_rc=%s sigma_executed over a mixed-arch pair (x86 test, aarch64 universe) is RED\n' "$([ "$got" = 1 ] && echo 'ok ' || echo 'BAD')" "$got" + rm -rf "${td:?}" + echo "Σ arch self-test: $((rows - bad))/$rows rows as expected" + [ "$bad" = 0 ] +} + +case "${1:-}" in + --self-test) self_test ;; + "" | -h | --help) sed -n 2,16p "$0" ;; + *) check "$1" "${2:?SHARDS}" ;; +esac diff --git a/scripts/ci_mg_reuse.sh b/scripts/ci_mg_reuse.sh index 03aa3c1e37..5f133ce19e 100755 --- a/scripts/ci_mg_reuse.sh +++ b/scripts/ci_mg_reuse.sh @@ -528,11 +528,13 @@ STUB # Each row is compared as the EXACT line (trailing comment stripped), so a third state cannot slip in. poolrows() { # poolrows WF -> one line per violated pool row local j l want - local plain="runs-on: [self-hosted, Linux, X64, clean-room]" + local plain="runs-on: [self-hosted, Linux, clean-room]" # arch-neutral job: cargo-deny, no X64 pin local lane="runs-on: \${{ github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"clean-room\"]') }}" - for j in x86-main x86-main-advisories workspace-test-shard; do + # #5882: the full tier is pinned X64 (one universe for the Σ), every other tier stays arch-neutral + local nlane="runs-on: \${{ needs.workspace-test-arch.outputs.tier == 'full' && (github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"X64\",\"clean-room\"]')) || (github.event_name == 'merge_group' && fromJSON('[\"self-hosted\",\"Linux\",\"mq\"]') || fromJSON('[\"self-hosted\",\"Linux\",\"clean-room\"]')) }}" + for j in x86-main x86-main-advisories workspace-test-arch workspace-test-shard; do l=$(awk -v j=" $j:" '$0 == j {f = 1; next} f && /^ [A-Za-z0-9_-]+:$/ {exit} f' "$1" 2> /dev/null | grep -m1 '^ runs-on:' | sed -E 's/^ +//; s/ +#.*$//') - want=$plain; case $j in x86-main | workspace-test-shard) want=$lane ;; esac + want=$plain; case $j in x86-main) want=$lane ;; workspace-test-shard) want=$nlane ;; esac if [ "$l" != "$want" ]; then echo "$j does not land on the clean-room pool (mq on merge_group for x86-main/workspace-test-shard) (runs-on: $l)"; fi done @@ -552,8 +554,11 @@ STUB poolmut pool-advisories-event-conditional "does not land on the clean-room pool" '/^ x86-main-advisories:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: ${{ github.event_name == '"'"'merge_group'"'"' \&\& fromJSON('"'"'["clean-room"]'"'"') || fromJSON('"'"'["clean-room"]'"'"') }}/}' poolmut mq-lane-lost-on-x86-main "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"mq"/"clean-room"/}' poolmut mq-lane-dropped-on-shard "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: [self-hosted, Linux, X64, clean-room]/}' - poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","X64","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","X64","mq"]'"'"')/}' + poolmut pr-path-takes-mq "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/\|\| fromJSON\(.\["self-hosted","Linux","clean-room"\].\)/|| fromJSON('"'"'["self-hosted","Linux","mq"]'"'"')/}' poolmut mq-lane-third-label "does not land on the clean-room pool" '/^ x86-main:$/,/^ steps:/{s/"X64","mq"/"X64","mq","pr-only"/}' + poolmut full-tier-loses-x64-pin "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/"Linux","X64",/"Linux",/g}' + poolmut every-tier-pinned-x64 "does not land on the clean-room pool" '/^ workspace-test-shard:$/,/^ steps:/{s/^ runs-on: .*/ runs-on: ${{ github.event_name == '"'"'merge_group'"'"' \&\& fromJSON('"'"'["self-hosted","Linux","X64","mq"]'"'"') || fromJSON('"'"'["self-hosted","Linux","X64","clean-room"]'"'"') }}/}' + poolmut tier-job-off-the-pool "does not land on the clean-room pool" '/^ workspace-test-arch:$/,/^ steps:/{s/clean-room/build/}' # S1 (20X): mg-reuse runs on push only, so on merge_group it is SKIPPED and every dependent # runs only because its own `if:` carries !cancelled()/always() (a skipped need otherwise skips