From 72c1fd51acb45d2091d42a4605f40d38446dd333 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:49:51 +0200 Subject: [PATCH 01/10] ci(gate): EMG-001 C1 -- workspace-test-shard runs as one shard, threads 4 -> 16 (#5924) Shard pins/matrices -> [1] for PR and merge queue; steps gated on shard 2/3 also run when shards == 1; sigma-shard-1 upload no longer skipped at one shard; check_shard_count_single_source self-test updated. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 4 +- ci/sections.yml | 58 ++++++++-------------- docs/roadmaps/entries/PMAT-5924.yaml | 19 +++++++ scripts/check_shard_count_single_source.sh | 4 +- 4 files changed, 44 insertions(+), 41 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-5924.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 408fa9a820..4e8610109d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,8 +534,8 @@ jobs: strategy: fail-fast: false matrix: - shard: ${{ fromJSON(github.event_name == 'merge_group' && '[1,2,3,4]' || '[1,2,3,4,5,6,7,8,9,10,11,12]') }} - shards: ${{ fromJSON(github.event_name == 'merge_group' && '[4]' || '[12]') }} + shard: ${{ fromJSON(github.event_name == 'merge_group' && '[1]' || '[1]') }} + shards: ${{ fromJSON(github.event_name == 'merge_group' && '[1]' || '[1]') }} outputs: results: ${{ steps.run.outputs.results }} env: diff --git a/ci/sections.yml b/ci/sections.yml index 538998bd19..22c282ac94 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -28,7 +28,7 @@ sovereign-ci: advisories_diff_only: true # A matrix job becomes one section per pinned combination. workspace-test-shard -# runs as TWELVE shards (ONE count: the `shards:` pins below; ci.yml's matrix lists must match, check_shard_count_single_source.sh), each its own ci.yml job on its own runner: as one shard +# runs as ONE shard (EMG-001 C1; was twelve) (ONE count: the `shards:` pins below; ci.yml's matrix lists must match, check_shard_count_single_source.sh), each its own ci.yml job on its own runner: as one shard # it took 5358 s of x86-main and was the release critical path. nextest's hash # partition N/12 and the explicit runner's round-robin N/12 each cover their list # exactly once between the shards; the cross-shard Σ fan-in in the workspace-test @@ -36,23 +36,9 @@ sovereign-ci: # [X64], the determinism fat job runs [ARM64] + the compare. matrix-pins: workspace-test-shard: - - {shard: 1, shards: 12} - - {shard: 2, shards: 12} - - {shard: 3, shards: 12} - - {shard: 4, shards: 12} - - {shard: 5, shards: 12} - - {shard: 6, shards: 12} - - {shard: 7, shards: 12} - - {shard: 8, shards: 12} - - {shard: 9, shards: 12} - - {shard: 10, shards: 12} - - {shard: 11, shards: 12} - - {shard: 12, shards: 12} + - {shard: 1, shards: 1} workspace-test-shard-merge_group: - - {shard: 1, shards: 4} - - {shard: 2, shards: 4} - - {shard: 3, shards: 4} - - {shard: 4, shards: 4} + - {shard: 1, shards: 1} determinism: - {arch: X64} - {arch: ARM64} @@ -94,8 +80,8 @@ jobs: fail-fast: false matrix: # NOTE: the matrix-pins above override this matrix (fat_driver.py); the guard checks the pins and ci.yml only - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - shards: [12] + shard: [1] + shards: [1] # PMAT-3138 (#3138): arch-neutral -- intel, yoga OR gx10. Measured 2026-09-12 on # gx10 with this image and env, --no-fail-fast: step 1 (59k tests) 10.6 min vs # 34.5 on intel, step 2 (102 tree readers) 6.7 vs 27.4, the over-the-cap @@ -549,7 +535,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e SHARD -e SHARDS \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ @@ -565,7 +551,7 @@ jobs: mkdir -p "$RUNNER_TEMP/sigma" cp "/mnt/nvme-raid0/targets/aprender-ci/${PR_OR_REF}/run-${GITHUB_RUN_ID}/nextest/ci/junit.xml" "$RUNNER_TEMP/sigma/lib.junit.xml" - name: "GPU crates on their default features (per-package resolve, no device needed)" - if: steps.tier.outputs.tier == 'full' && matrix.shard == 2 + if: steps.tier.outputs.tier == 'full' && (matrix.shard == 2 || matrix.shards == 1) # aprender-gpu + aprender-cuda-edge, un-darked from the required check (T0 of the # NVIDIA CUDA Rust spec, #3062). A per-package resolve keeps `cuda` OFF (see the # comment on the workspace line above); the `driver`/`kernels`/`memory`/`ptx` tests @@ -583,13 +569,13 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ "$IMAGE" \ cargo nextest run --profile ci -p aprender-gpu -p aprender-cuda-edge --lib - name: "Σ-executed: keep the GPU crates junit (the next nextest run overwrites it)" - if: steps.tier.outputs.tier == 'full' && matrix.shard == 2 + if: steps.tier.outputs.tier == 'full' && (matrix.shard == 2 || matrix.shards == 1) # #4433: nextest writes /nextest/ci/junit.xml and every invocation # overwrites it, so each run's report is copied out before the next one. shell: bash @@ -601,7 +587,7 @@ jobs: # their Σ check runs here, on the shard that built and ran them. The universe is # listed here with the filters written out again, as in the shard-1 Σ step below. - name: "Σ-executed: the GPU crates ran exactly the tests they owe (#4433)" - if: steps.tier.outputs.tier == 'full' && matrix.shard == 2 + if: steps.tier.outputs.tier == 'full' && (matrix.shard == 2 || matrix.shards == 1) shell: bash run: | set -euo pipefail @@ -641,7 +627,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ "$IMAGE" \ bash -c 'cargo test -p aprender-compute --lib 2>&1 | tee /tmp/compute-test.log; grep -Eq "test result: ok\. [1-9][0-9]* passed" /tmp/compute-test.log && ! grep -q "test result: FAILED" /tmp/compute-test.log' - name: Integration tests @@ -713,7 +699,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e SHARD -e SHARDS \ "$IMAGE" \ bash scripts/ci_run_explicit_test_commands.sh --run ci/explicit-test-commands.d --shard "${SHARD}/${SHARDS}" @@ -747,7 +733,7 @@ jobs: -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ -e SCCACHE_DIR=/sccache \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ "$IMAGE" \ @@ -767,7 +753,6 @@ jobs: # on every tier, so the verdict job can tell "not a full run" from "a # shard's ids never arrived" -- a missing file there is RED, not a skip. - name: "Σ-executed: stage this shard's executed ids for the fan-in (#4433)" - if: matrix.shards != 1 shell: bash env: TIER: ${{ steps.tier.outputs.tier }} @@ -784,7 +769,6 @@ jobs: fi fi - uses: actions/upload-artifact@v7 - if: matrix.shards != 1 with: name: sigma-shard-${{ matrix.shard }} path: sigma-shard/ @@ -798,7 +782,7 @@ jobs: # together. Not a fragment because it is a build, not a test target, and # no wiring reader (check_model_tests_wired, check_beats_gated) names it. - name: Build every example (workspace, keep-going) - if: steps.tier.outputs.tier == 'full' && matrix.shard == 3 + if: steps.tier.outputs.tier == 'full' && (matrix.shard == 3 || matrix.shards == 1) timeout-minutes: 40 run: | set -euo pipefail @@ -814,7 +798,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e SHARD -e SHARDS \ "$IMAGE" \ cargo build --examples --workspace --keep-going @@ -883,7 +867,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ @@ -981,7 +965,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ @@ -1054,7 +1038,7 @@ jobs: -e RUSTC_WRAPPER=/usr/local/cargo/bin/sccache \ -e SCCACHE_DIR=/sccache \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e TREE="$tree" -e EXPR="$EXPR" \ @@ -1094,7 +1078,7 @@ jobs: -e SCCACHE_DIR=/sccache \ -e SCCACHE_CACHE_SIZE=50G \ -e CARGO_INCREMENTAL=0 \ - -e CARGO_BUILD_JOBS=4 -e RAYON_NUM_THREADS=4 -e NEXTEST_TEST_THREADS=4 -e RUST_TEST_THREADS=4 \ + -e CARGO_BUILD_JOBS=16 -e RAYON_NUM_THREADS=16 -e NEXTEST_TEST_THREADS=16 -e RUST_TEST_THREADS=16 \ -e CARGO_PROFILE_TEST_DEBUG=0 \ -e CARGO_PROFILE_DEV_DEBUG=0 \ -e CARGO_TERM_COLOR=never \ @@ -1126,7 +1110,7 @@ jobs: run: | printf 'workspace-test NOT RUN: %s\n' "$REASON" - name: Build.rs crate-root escape check (v0.31.1 yank guard) - if: matrix.shard == 2 + if: (matrix.shard == 2 || matrix.shards == 1) # Static Poka-Yoke: flags build.rs files that panic on files outside # CARGO_MANIFEST_DIR, which break `cargo install` from crates.io. # See scripts/check_build_rs_paths.sh for the full rationale. @@ -1138,7 +1122,7 @@ jobs: "$IMAGE" \ setsid --wait bash scripts/check_build_rs_paths.sh - name: apr-format leaf sovereignty guard (#2231) - if: matrix.shard == 2 + if: (matrix.shard == 2 || matrix.shards == 1) # Poka-Yoke: prove the extracted `apr-format` leaf pulls no ML/GPU/ # tokenizer/framework crate (so consumers `cargo add apr-format` without # aprender-core + trueno/wgpu). Discriminating: PASSES on apr-format + diff --git a/docs/roadmaps/entries/PMAT-5924.yaml b/docs/roadmaps/entries/PMAT-5924.yaml new file mode 100644 index 0000000000..024b480d2b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-5924.yaml @@ -0,0 +1,19 @@ +- id: PMAT-5924 + github_issue: 5924 + item_type: task + title: 'EMG-001 C1: workspace-test-shard single shard, threads 4 to 16' + status: planned + priority: high + assigned_to: null + created: 2026-10-22T10:55:00Z + updated: 2026-10-22T10:55:00Z + spec: null + acceptance_criteria: + - handoff/emg-001/EMG-001.md C1+C2; EMG-9 proof in PR body + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 diff --git a/scripts/check_shard_count_single_source.sh b/scripts/check_shard_count_single_source.sh index 9a63816326..a68c71c7be 100755 --- a/scripts/check_shard_count_single_source.sh +++ b/scripts/check_shard_count_single_source.sh @@ -79,11 +79,11 @@ if [ "${1:-}" = --self-test ]; then cp ci/sections.yml "$t/s.yml"; sed -E 's/^( +SHARDS=)"\$\(awk.*ci\/sections\.yml\)"/\112/' .github/workflows/ci.yml > "$t/c.yml" cmp -s .github/workflows/ci.yml "$t/c.yml" && { echo "self-test: SHARDS=12 mutant did not apply"; exit 1; } check "$t/s.yml" "$t/c.yml" >/dev/null && { echo "self-test: typed aggregator SHARDS=12 must be RED"; exit 1; } - sed -E '0,/^ shards: \[12\]/s// shards: [11]/' ci/sections.yml > "$t/s.yml" + sed -E '0,/^ shards: \[1\]/s// shards: [2]/' ci/sections.yml > "$t/s.yml" cmp -s ci/sections.yml "$t/s.yml" && { echo "self-test: inert-matrix mutant did not apply"; exit 1; } check "$t/s.yml" .github/workflows/ci.yml >/dev/null && { echo "self-test: sections.yml jobs matrix drift must be RED"; exit 1; } cp .github/workflows/ci.yml "$t/c.yml" - sed -E '/^ workspace-test-shard-merge_group:/,/^ determinism:/s/\{shard: 4, shards: 4\}/{shard: 4, shards: 3}/' ci/sections.yml > "$t/s.yml" + sed -E '/^ workspace-test-shard-merge_group:/,/^ determinism:/s/\{shard: 1, shards: 1\}/{shard: 1, shards: 2}/' ci/sections.yml > "$t/s.yml" cmp -s ci/sections.yml "$t/s.yml" && { echo "self-test: merge_group pin mutant did not apply"; exit 1; } check "$t/s.yml" "$t/c.yml" >/dev/null && { echo "self-test: a merge_group pin with the wrong shards must be RED"; exit 1; } awk '/^ workspace-test-shard-merge_group:/{print; for(i=1;i<=13;i++) printf " - {shard: %d, shards: 13}\n", i; skip=1; next} skip&&/^ - /{next} {skip=0; print}' ci/sections.yml > "$t/s.yml" From 411f58f8b5de280b917713594d9176562ad9ead4 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:58:39 +0200 Subject: [PATCH 02/10] chore(roadmap): aggregate PMAT-5924 fragment (#5924) Co-Authored-By: Claude Sonnet 5.5 --- docs/roadmaps/roadmap.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 8f5dbcf0ee..3cf166d272 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -25493,3 +25493,22 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 5510 +- id: PMAT-5924 + github_issue: 5924 + item_type: task + title: 'EMG-001 C1: workspace-test-shard single shard, threads 4 to 16' + status: planned + priority: high + assigned_to: null + created: 2026-10-22T10:55:00Z + updated: 2026-10-22T10:55:00Z + spec: null + acceptance_criteria: + - handoff/emg-001/EMG-001.md C1+C2; EMG-9 proof in PR body + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 From 19ce2e8e25b0be24f391a36821728361ab36128d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:54:20 +0200 Subject: [PATCH 03/10] ci(lab): checker uses the longest needs: chain + daytime starts; fix nightly/dogfood/mutants/fleet-toolset schedules (PMAT-5847) (squashed from #5848) Refs #5847 PMAT-Ticket: PMAT-5847 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- .../workflows/assets-rehearsal-nightly.yml | 4 ++ .github/workflows/book-contracts.yml | 6 ++ .github/workflows/book.yml | 2 + .github/workflows/dogfood-nightly.yml | 4 +- .github/workflows/edge-fast-verbs-nightly.yml | 2 +- .github/workflows/fleet-toolset.yml | 4 +- .github/workflows/install-script.yml | 1 + .github/workflows/mutants-nightly.yml | 2 +- .github/workflows/nightly-train.yml | 2 +- .github/workflows/nightly.yml | 6 +- .github/workflows/silicon-nightly.yml | 1 + docs/roadmaps/entries/PMAT-5847.yaml | 20 ++++++ scripts/check_lab_timeouts.sh | 61 +++++++++++++++++++ 13 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-5847.yaml create mode 100755 scripts/check_lab_timeouts.sh diff --git a/.github/workflows/assets-rehearsal-nightly.yml b/.github/workflows/assets-rehearsal-nightly.yml index 76740a1b3f..d06c13b696 100644 --- a/.github/workflows/assets-rehearsal-nightly.yml +++ b/.github/workflows/assets-rehearsal-nightly.yml @@ -65,6 +65,7 @@ jobs: name: pv ${{ matrix.target }} on ${{ matrix.host }} needs: assets runs-on: ${{ fromJSON(matrix.labels) }} + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -393,6 +394,7 @@ jobs: name: apr (cuda) ${{ matrix.target }} on ${{ matrix.host }} needs: assets runs-on: ${{ fromJSON(matrix.labels) }} + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -545,6 +547,7 @@ jobs: name: apr (cpu) ${{ matrix.target }} on ${{ matrix.host }} needs: assets runs-on: ${{ fromJSON(matrix.labels) }} + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -710,6 +713,7 @@ jobs: name: apr (cpu) aarch64-apple-darwin on mini needs: assets runs-on: [self-hosted, macOS, ARM64, mini-builder] + timeout-minutes: 120 env: MACOSX_DEPLOYMENT_TARGET: "11.0" steps: diff --git a/.github/workflows/book-contracts.yml b/.github/workflows/book-contracts.yml index b28d4404fe..8912ac96c4 100644 --- a/.github/workflows/book-contracts.yml +++ b/.github/workflows/book-contracts.yml @@ -60,6 +60,7 @@ jobs: chapter-examples-compile: name: Chapter Examples Compile runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -103,6 +104,7 @@ jobs: chapter-examples-run: name: Chapter Examples Run runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 needs: chapter-examples-compile steps: - uses: actions/checkout@v7 @@ -155,6 +157,7 @@ jobs: book-integration-tests: name: Book Integration Tests runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable @@ -187,6 +190,7 @@ jobs: contract-enforcement: name: Contract Enforcement runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 - name: Every chapter has a contract @@ -247,6 +251,7 @@ jobs: namespace-discipline: name: Namespace Discipline runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 - name: No legacy imports in chapter examples @@ -288,6 +293,7 @@ jobs: summary-integrity: name: SUMMARY.md Integrity runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 - name: No dead links in SUMMARY.md diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index f695c6ed4e..eebb8f914a 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -73,6 +73,7 @@ jobs: build: name: Build Book runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 120 steps: - uses: actions/checkout@v7 @@ -206,6 +207,7 @@ jobs: deploy: name: Deploy to GitHub Pages runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 needs: build if: github.event_name == 'push' && github.ref == 'refs/heads/main' # GitHub Pages deploys never overlap and are never cancelled (was workflow-level). diff --git a/.github/workflows/dogfood-nightly.yml b/.github/workflows/dogfood-nightly.yml index 0ba1a1dbab..0f75434201 100644 --- a/.github/workflows/dogfood-nightly.yml +++ b/.github/workflows/dogfood-nightly.yml @@ -24,11 +24,11 @@ name: dogfood-nightly on: schedule: - # 23:05 UTC: five minutes after nightly.yml's 23:00, so both land in the same late dispatch. + # 20:05 UTC: five minutes after nightly.yml's 20:00 (night 75 + dogfood 420 ends 04:20Z), so both land in the same late dispatch. # The train judges the head of THAT night's scheduled nightly.yml run (C); the `night` job # below resolves C and the dogfood job checks it out, so a merge after 21:00 no longer # leaves the lane measuring some other commit (#5079). - - cron: '5 23 * * *' + - cron: '5 20 * * *' workflow_dispatch: inputs: plant: diff --git a/.github/workflows/edge-fast-verbs-nightly.yml b/.github/workflows/edge-fast-verbs-nightly.yml index 9982ce23c2..c0ca914091 100644 --- a/.github/workflows/edge-fast-verbs-nightly.yml +++ b/.github/workflows/edge-fast-verbs-nightly.yml @@ -10,7 +10,7 @@ name: Edge fast-verbs LAB nightly on: schedule: - - cron: '30 2 * * *' + - cron: '30 0 * * *' workflow_dispatch: inputs: model: diff --git a/.github/workflows/fleet-toolset.yml b/.github/workflows/fleet-toolset.yml index a8f02d4ac4..ad9286db80 100644 --- a/.github/workflows/fleet-toolset.yml +++ b/.github/workflows/fleet-toolset.yml @@ -41,10 +41,10 @@ name: Fleet Toolset on: schedule: - # 04:47 UTC — an off-minute, and clear of cuda-nightly (01:30) and + # 04:35 UTC — an off-minute, and clear of cuda-nightly (01:30) and # silicon-nightly (03:30) so the probe never queues behind a 90-minute # measurement it would otherwise appear to have slowed down. - - cron: "47 4 * * *" + - cron: "35 4 * * *" workflow_dispatch: permissions: diff --git a/.github/workflows/install-script.yml b/.github/workflows/install-script.yml index f8daec95c5..b2a6ee87b0 100644 --- a/.github/workflows/install-script.yml +++ b/.github/workflows/install-script.yml @@ -50,6 +50,7 @@ jobs: test: name: install_test.sh runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index 6da1f6d287..01a9453bbd 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -12,7 +12,7 @@ name: Mutants Nightly on: schedule: - - cron: '0 3 * * 0' # Sunday 03:00 UTC: the quietest PR hours (03-05Z, 10 of 300 PR CI runs 2026-09-29..10-06) on the quietest day + - cron: '0 1 * * 0' # Sunday 01:00 UTC (mutants 180 + stop 5 ends 04:05Z): the quietest PR hours (03-05Z, 10 of 300 PR CI runs 2026-09-29..10-06) on the quietest day workflow_dispatch: {} permissions: diff --git a/.github/workflows/nightly-train.yml b/.github/workflows/nightly-train.yml index cd63822b01..4c347007ff 100644 --- a/.github/workflows/nightly-train.yml +++ b/.github/workflows/nightly-train.yml @@ -2,7 +2,7 @@ name: Nightly Train on: schedule: - - cron: '45 4 * * *' # 04:45Z, after the last producer (mutants 01:00Z) + - cron: '30 4 * * *' # 04:30Z, after the last producer (mutants 01:00Z) workflow_dispatch: {} # drills only; a dispatched run never counts toward the streak permissions: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6843812910..13790c1e96 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -48,7 +48,7 @@ on: # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by # ~3h and is growing. The durable fix is a fleet-side timer calling # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 23 * * *' # 23:00 UTC -> lands ~04:00 UTC + - cron: '0 20 * * *' # 20:00 UTC: gate+build+publish chain is 420 min, so it ends by 03:00Z (PMAT-5847: end by 05:00Z) workflow_dispatch: permissions: @@ -68,6 +68,7 @@ jobs: # ── Is there work, and is HEAD green? ──────────────────── gate: runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 60 outputs: decision: ${{ steps.gate.outputs.decision }} sha: ${{ steps.gate.outputs.sha }} @@ -123,6 +124,7 @@ jobs: host: gx10 labels: '["self-hosted", "Linux", "ARM64", "cuda", "gx10"]' runs-on: ${{ fromJSON(matrix.labels) }} + timeout-minutes: 120 env: NIGHTLY_SHA: ${{ needs.gate.outputs.sha }} steps: @@ -338,6 +340,7 @@ jobs: needs: [gate, build] if: always() && needs.gate.result == 'success' runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 120 env: NIGHTLY_SHA: ${{ needs.gate.outputs.sha }} steps: @@ -400,6 +403,7 @@ jobs: needs: [gate, build-darwin, publish] if: always() && needs.gate.outputs.decision == 'build' && needs.build-darwin.result == 'success' runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 120 steps: - name: Download darwin artifact uses: actions/download-artifact@v4 diff --git a/.github/workflows/silicon-nightly.yml b/.github/workflows/silicon-nightly.yml index ae5ed579b9..2f28f3891f 100644 --- a/.github/workflows/silicon-nightly.yml +++ b/.github/workflows/silicon-nightly.yml @@ -219,6 +219,7 @@ jobs: needs: [coverage, x86_64-cpu, aarch64-cuda-sm121] if: always() runs-on: [self-hosted, clean-room, intel] + timeout-minutes: 60 steps: - name: Report per silicon run: | diff --git a/docs/roadmaps/entries/PMAT-5847.yaml b/docs/roadmaps/entries/PMAT-5847.yaml new file mode 100644 index 0000000000..6adefd6fe2 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-5847.yaml @@ -0,0 +1,20 @@ +- id: PMAT-5847 + github_issue: 5847 + item_type: task + title: 'LAB nightlies: timeout-minutes on every job and end by 05:00Z (falsifier check_lab_timeouts.sh)' + status: in_progress + priority: high + assigned_to: null + created: 2026-10-21T11:30:00Z + updated: 2026-10-21T11:30:00Z + spec: null + acceptance_criteria: + - 'scripts/check_lab_timeouts.sh is RED on the pre-fix tree (22 findings on the merge-base) and GREEN after; --self-test 6 rows' + - 'reaper catching queued + schedule runs is a separate change to the deployed pr-run-reaper.sh' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 diff --git a/scripts/check_lab_timeouts.sh b/scripts/check_lab_timeouts.sh new file mode 100755 index 0000000000..42d5990ec4 --- /dev/null +++ b/scripts/check_lab_timeouts.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# check_lab_timeouts.sh -- PMAT-5847. LAB/nightly (schedule:) workflows are bounded: every job has timeout-minutes, +# and the cron start plus the longest job timeout ends by 05:00Z (a start before 05:00Z ends by 05:00Z that day, +# a later start by 05:00Z the next day). Operator, verbatim, 11:04Z: "Lab/nightly workflows get timeout-minutes +# and must end by 05:00Z". A LAB check: run on demand or at night, never inside a PR or release job (C324). +# A job that only calls a reusable workflow (uses:) cannot carry timeout-minutes; the callee is scanned itself. +# Usage: check_lab_timeouts.sh [workflows-dir] | --self-test Exit: 0 green, 1 RED +set -euo pipefail +LIMIT=300 DAY=1440 +DAY_FROM=300 DAY_TO=1080 # a start in 05:00-18:00Z runs in the working day +# Named exception (reason in the ticket): rehearsal starts after nightly-train has read the night's producers; its 300 min cannot end by 05:00Z. Decision owed by the operator/cop. +EXCEPT="${LAB_TIMEOUT_EXCEPT-release-rehearsal-nightly.yml}" +PATH_AWK=$(cat <<'AWK' +function rec(j, k, best, v, n, a, i) { + if (j in memo) return memo[j] + best = 0; n = split(need[j], a, " ") + for (i = 1; i <= n; i++) if (a[i] in tm) { v = rec(a[i]); if (v > best) best = v } + return memo[j] = best + tm[j] +} +/^jobs:/ { in_jobs = 1; next } +in_jobs && /^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ { name = $1; sub(/:$/, "", name); tm[name] = 0; need[name] = ""; innd = 0; next } +in_jobs && name && /^ timeout-minutes:/ { tm[name] = $2 + 0; innd = 0; next } +in_jobs && name && /^ needs:/ { s = $0; sub(/^ needs:[[:space:]]*/, "", s); gsub(/[\[\],]/, " ", s); need[name] = s; innd = (s ~ /^[[:space:]]*$/); next } +in_jobs && name && innd && /^ - / { s = $0; sub(/^ - /, "", s); need[name] = need[name] " " s; next } +in_jobs && name && /^ [A-Za-z]/ { innd = 0 } +END { m = 0; for (j in tm) { v = rec(j); if (v > m) m = v } print m } +AWK +) +scan() { # one workflow file -> RED lines + local f="$1" j longest m h s end + grep -qE '^[[:space:]]+schedule:' "$f" || return 0 + case " $EXCEPT " in *" $(basename "$f") "*) return 0 ;; esac + for j in $(awk '/^jobs:/{in_jobs=1;next} in_jobs&&/^ [A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/{if(name&&runs&&!seen)print name; name=$1; runs=0; seen=0; next} in_jobs&&/^ runs-on:/{runs=1} in_jobs&&/^ timeout-minutes:/{seen=1} END{if(name&&runs&&!seen)print name}' "$f"); do + echo "RED $f job ${j%:}: no timeout-minutes" + done + longest=$(awk "$PATH_AWK" "$f") # longest needs: chain, not the longest single job + grep -E '^[[:space:]]+- cron:' "$f" | sed -E "s/.*cron:[[:space:]]*['\"]?([^'\"#]*).*/\1/" | while read -r m h _; do + case "$m$h" in *[!0-9]*) continue ;; esac # */10 or a range is not one nightly start + s=$((h * 60 + m)); end=$((s + longest)) + if [ "$s" -ge "$DAY_FROM" ] && [ "$s" -lt "$DAY_TO" ]; then echo "RED $f: cron $m $h starts in the working day (05:00-18:00Z)"; continue; fi + if [ "$s" -lt "$LIMIT" ]; then [ "$end" -le "$LIMIT" ] || echo "RED $f: cron $m $h + ${longest}m ends after 05:00Z" + else [ "$end" -le $((DAY + LIMIT)) ] || echo "RED $f: cron $m $h + ${longest}m ends after 05:00Z"; fi + done +} +check() { local bad=0 out f; for f in "$1"/*.yml; do out=$(scan "$f"); [ -z "$out" ] || { echo "$out"; bad=1; }; done; return "$bad"; } +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf -- "${t:?}"' EXIT; fail=0 + w() { rm -f "${t:?}"/*.yml; printf 'on:\n schedule:\n - cron: "%s"\njobs:\n a:\n runs-on: x\n%s steps: []\n' "$1" "$2" > "$t/w.yml"; } + w '5 23 * * *' $' timeout-minutes: 60\n'; check "$t" > /dev/null || { echo "FAIL bounded row is RED"; fail=1; } + w '5 23 * * *' ''; check "$t" > /dev/null && { echo "FAIL no-timeout row not RED"; fail=1; } + w '5 23 * * *' $' timeout-minutes: 420\n'; check "$t" > /dev/null && { echo "FAIL ends-after-05Z row not RED"; fail=1; } + w '5 23 * * *' $' timeout-minutes: 355\n'; check "$t" > /dev/null || { echo "FAIL ends-exactly-05:00 row is RED"; fail=1; } + w '20 3 * * *' $' timeout-minutes: 120\n'; check "$t" > /dev/null && { echo "FAIL 03:20+120m row not RED"; fail=1; } + rm -f "${t:?}"/*.yml; printf 'on:\n schedule:\n - cron: "5 22 * * *"\njobs:\n a:\n runs-on: x\n timeout-minutes: 200\n b:\n needs: a\n runs-on: x\n timeout-minutes: 200\n c:\n needs: [a, b]\n runs-on: x\n timeout-minutes: 100\n' > "$t/chain.yml" + check "$t" > /dev/null && { echo "FAIL chain overrun (a+b+c=500) not RED"; fail=1; } + w '0 12 * * *' $' timeout-minutes: 10\n'; check "$t" > /dev/null && { echo "FAIL daytime start not RED"; fail=1; } + rm -f "${t:?}"/*.yml; printf 'on:\n pull_request:\njobs:\n a:\n runs-on: x\n steps: []\n' > "$t/pr.yml" + check "$t" > /dev/null || { echo "FAIL non-schedule workflow flagged"; fail=1; } + [ "$fail" -eq 0 ] && echo "ok: self-test (8 rows)"; exit "$fail" +fi +check "${1:-.github/workflows}" From a4cbe70e86ace073437d951e3b1ef21bf5e269d7 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:54:50 +0200 Subject: [PATCH 04/10] fix(resolve_base): name merge_group.base_sha and pull_request.base.sha so a REBASE-replayed PR is judged whole (Refs #5855, #5859) (squashed from #5870) Refs #5859 PMAT-Ticket: PMAT-5859 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- scripts/check_roadmap_diff_additive.sh | 31 +++++++++++++++++++++++++- scripts/lib/resolve_base.sh | 24 ++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/check_roadmap_diff_additive.sh b/scripts/check_roadmap_diff_additive.sh index 07b1614084..38a0658346 100755 --- a/scripts/check_roadmap_diff_additive.sh +++ b/scripts/check_roadmap_diff_additive.sh @@ -92,7 +92,7 @@ if [ "${1:-}" = "--self-test" ]; then not_measured=0 # The case table has exactly this many rows; a run that reaches fewer, or more, is RED, never "n/n" # (#5047). Raise it with every row added. The two summary rows at the end are not counted. - SELFTEST_MIN_ROWS=27 + SELFTEST_MIN_ROWS=30 # selftest_verdict ROWS FAILS NOT_MEASURED MIN -> prints the summary; rc 0 all rows ran and # passed, 1 a row failed or other than MIN ran, 2 every row that ran passed but one was not measured. @@ -443,7 +443,36 @@ EOF *"is not a merge commit nor a commit on the origin/main tip"*) printf 'ok row %-2s stacked entry with deepening disabled (mutation): refused by name\n' "$row" ;; *) printf 'FAIL row %-2s stacked entry with deepening disabled: refused for the wrong reason (rc=%s): %s\n' "$row" "$rc7" "$err7"; fails=1 ;; esac + # Rows: a REBASE-replayed 3-commit pull request (#5855). The queue replays the PR as c2,c3,c4 on the + # main tip c1; the head's single parent is c3, so every parent rule above names c3 and judges ONLY the + # last commit. The event names the real base (merge_group.base_sha = c1); with the event base ignored + # (the mutation) the stacked rule answers c3, which is the defect. + row=$((row + 1)) + QR="$TD/rebase-src"; rm -rf "${QR:?}"; ( git init -q -b main "$QR" && cd "$QR" && git config user.email t@t && git config user.name t \ + && cp "$TD/base.yaml" r.yaml && git add r.yaml && git commit -qm c1 && git branch -q queue-base \ + && printf 'a\n' > a.txt && git add a.txt && git commit -qm c2 \ + && printf 'b\n' > b.txt && git add b.txt && git commit -qm c3 \ + && printf 'c\n' > c.txt && git add c.txt && git commit -qm c4 ) 2>/dev/null + rm -rf "${QR:?}.clone"; git clone -q --depth=1 -b main "file://$QR" "$QR.clone" 2>/dev/null; git -C "$QR.clone" fetch -q --depth=1 origin '+queue-base:refs/remotes/origin/main' 2>/dev/null + git -C "$QR" config uploadpack.allowReachableSHA1InWant true + wantr=$( git -C "$QR" rev-parse queue-base ) + printf '{"merge_group":{"base_sha":"%s"}}\n' "$wantr" > "$TD/rebase-event.json" + gotr=$( GITHUB_EVENT_NAME=merge_group GITHUB_EVENT_PATH="$TD/rebase-event.json" bash -c 'cd "$2" || exit 2; . "$0" --lib-only; REPO_ROOT="$1"; resolve_base HEAD && printf "%s|%s" "$BASE_REF" "$BASE_HOW"' "$SELF" "$QR.clone" "$QR.clone" 2>/dev/null ) || true + case "$gotr" in "$wantr|merge_group.base_sha"*) printf 'ok row %-2s REBASE-replayed 3-commit entry -> base = merge_group.base_sha, not the previous commit\n' "$row" ;; + *) printf 'FAIL row %-2s REBASE-replayed entry: wanted %s|merge_group.base_sha..., got %s\n' "$row" "$wantr" "$gotr"; fails=1 ;; esac + row=$((row + 1)) + gotr=$( GITHUB_EVENT_NAME=merge_group GITHUB_EVENT_PATH="$TD/rebase-event.json" ROADMAP_DIFF_NO_EVENT_BASE=1 bash -c 'cd "$2" || exit 2; . "$0" --lib-only; REPO_ROOT="$1"; resolve_base HEAD && printf "%s|%s" "$BASE_REF" "$BASE_HOW"' "$SELF" "$QR.clone" "$QR.clone" 2>/dev/null ) || true + case "$gotr" in "$wantr|"*) printf 'FAIL row %-2s REBASE-replayed entry with the event base ignored: still named the real base — the mutation did not bite\n' "$row"; fails=1 ;; + *) printf 'ok row %-2s REBASE-replayed entry with the event base ignored (mutation): base is the previous commit, not %s\n' "$row" "${wantr:0:9}" ;; esac # #4936 (P6): the same stacked entry with the deepen's first read dead (curl 92). fetch_p6.sh reads the + # The pull_request fallback on a REAL payload shape (#5859): base is {label, ref, repo{...}, sha, user{...}}, + # so a sed that stops at the first `}` returned nothing. Head's sha comes first and must not be taken. + row=$((row + 1)) + rm -rf "${QR:?}.clone"; git clone -q --depth=1 -b main "file://$QR" "$QR.clone" 2>/dev/null; git -C "$QR.clone" fetch -q --depth=1 origin '+queue-base:refs/remotes/origin/main' 2>/dev/null + printf '{"action":"synchronize","pull_request":{"head":{"label":"o:b","ref":"b","sha":"%s","user":{"login":"a"},"repo":{"id":3,"owner":{"login":"o"}}},"base":{"label":"o:main","ref":"main","repo":{"id":3,"owner":{"login":"o"},"license":{"key":"mit"}},"sha":"%s","user":{"login":"o"}}}}\n' "$(git -C "$QR" rev-parse HEAD)" "$wantr" > "$TD/pr-event.json" + gotr=$( GITHUB_EVENT_NAME=pull_request GITHUB_EVENT_PATH="$TD/pr-event.json" bash -c 'cd "$2" || exit 2; . "$0" --lib-only; REPO_ROOT="$1"; resolve_base HEAD && printf "%s|%s" "$BASE_REF" "$BASE_HOW"' "$SELF" "$QR.clone" "$QR.clone" 2>/dev/null ) || true + case "$gotr" in "$wantr|pull_request.base.sha"*) printf 'ok row %-2s pull_request fallback on a real nested payload -> base = base.sha, not head.sha\n' "$row" ;; + *) printf 'FAIL row %-2s pull_request fallback: wanted %s|pull_request.base.sha..., got %s\n' "$row" "$wantr" "$gotr"; fails=1 ;; esac # deepen again and the base still resolves. A planted git logs every fetch read; both must be the # deepen, so a deepen that is one bare read (it dies, and the by-sha fetch after it answers) is RED. # It runs from a cwd outside REPO_ROOT (the shim's own dir), as check_ont_ratchet.sh may (it never cds), diff --git a/scripts/lib/resolve_base.sh b/scripts/lib/resolve_base.sh index d955594dfe..0ffc76eddb 100644 --- a/scripts/lib/resolve_base.sh +++ b/scripts/lib/resolve_base.sh @@ -24,6 +24,25 @@ # stacked entry is refused below, as it was before #4936 when the fetch failed. _RESOLVE_BASE_P6="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/../ci/fetch_p6.sh" || _RESOLVE_BASE_P6="" +# _resolve_event_base: the base sha the CI event itself names (merge_group.base_sha, else +# pull_request.base.sha), or nothing. Under a REBASE merge queue a multi-commit pull request is +# replayed as N commits, so the head's parent is the previous replayed commit, not the base: +# the parent-based rules below would judge only the LAST commit (#5855). Parsed with sed, no jq. +# ROADMAP_DIFF_NO_EVENT_BASE=1 is the case table's mutation. +_resolve_event_base() { + local ev=${GITHUB_EVENT_PATH:-} sha="" + [ "${ROADMAP_DIFF_NO_EVENT_BASE:-0}" != 1 ] && [ -n "$ev" ] && [ -r "$ev" ] || return 0 + case "${GITHUB_EVENT_NAME:-}" in + merge_group) sha=$(sed -n 's/.*"base_sha"[[:space:]]*:[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' "$ev" | head -n 1) ;; + pull_request|pull_request_target) sha=$(awk '{ s = s $0 } END { if (!match(s, /"base"[ \t]*:[ \t]*\{/)) exit; s = substr(s, RSTART + RLENGTH); if (!match(s, /"sha"[ \t]*:[ \t]*"[0-9a-f]+"/)) exit; s = substr(s, RSTART, RLENGTH); sub(/^"sha"[ \t]*:[ \t]*"/, "", s); sub(/"$/, "", s); if (length(s) == 40) print s }' "$ev") ;; + esac + [ -n "$sha" ] || return 0 + git -C "$REPO_ROOT" cat-file -e "$sha^{commit}" 2>/dev/null \ + || git -C "$REPO_ROOT" fetch -q --depth 1 origin "$sha" >/dev/null 2>&1 || true + git -C "$REPO_ROOT" cat-file -e "$sha^{commit}" 2>/dev/null && printf '%s\n' "$sha" + return 0 +} + resolve_base() { local head=$1 mb parents headid main_tip local PROG="${PROG:-resolve_base}" @@ -50,6 +69,10 @@ resolve_base() { printf '%s: %s is the origin/main tip and its first parent is not fetched, so the only base would be %s itself; refused (never the tree against itself). Deepen the checkout: git fetch --deepen=1 origin +refs/heads/main:refs/remotes/origin/main\n' "$PROG" "$head" "$head" >&2 return 1 fi + local evb; evb=$(_resolve_event_base) + if [ -n "$evb" ] && [ "${GITHUB_EVENT_NAME:-}" = merge_group ]; then + BASE_REF="$evb"; BASE_HOW="merge_group.base_sha (event base; a REBASE-replayed head has no parent that names it)"; return 0 + fi mb="" if [ "${ROADMAP_DIFF_FORCE_SHALLOW:-0}" != 1 ]; then mb=$(git -C "$REPO_ROOT" merge-base origin/main "$head" 2>/dev/null || true) @@ -86,6 +109,7 @@ resolve_base() { BASE_REF="$p1"; BASE_HOW="single parent (stacked merge_group entry: the previous entry's squash, fetched by deepening the shallow checkout)"; return 0 fi fi + if [ -n "$evb" ]; then BASE_REF="$evb"; BASE_HOW="pull_request.base.sha (event base; shallow checkout)"; return 0; fi printf '%s: merge-base(origin/main, %s) is unresolvable (shallow checkout) and %s is not a merge commit nor a commit on the origin/main tip,\n' "$PROG" "$head" "$head" >&2 printf ' so no base can be named. A pull_request job checks out refs/pull/N/merge, a merge_group job the queue head; run with an explicit otherwise.\n' >&2 return 1 From a1b0a699cb4bf2044337b3c07fbb63404964ae5f Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:54:56 +0200 Subject: [PATCH 05/10] fix(guards): check_ladder_cells_producer reaps its fake serves when SIGKILLed (detached watcher) (squashed from #5868) Refs #5868 PMAT-Ticket: PMAT-5868 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- scripts/check_ladder_cells_producer.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/check_ladder_cells_producer.sh b/scripts/check_ladder_cells_producer.sh index 3ad6c300b1..4db96dca67 100755 --- a/scripts/check_ladder_cells_producer.sh +++ b/scripts/check_ladder_cells_producer.sh @@ -30,6 +30,11 @@ reap_work() { rm -rf "${WORK:?}" } trap reap_work EXIT +# EXIT does not run on SIGKILL (a cancelled or timed-out job): a detached watcher repeats the sweep once this shell is gone +# (8 fake serves leaked on lambda, 2026-10-21). The dir is removed by the same name, never by a pattern. +setsid bash -c 'while kill -0 "$1" 2>/dev/null; do sleep 2; done + for p in /proc/[0-9]*; do c=$(tr "\0" " " 2>/dev/null < "$p/cmdline") || continue; case "$c" in *"$2/"*) kill -TERM "${p#/proc/}" 2>/dev/null ;; esac; done + rm -rf -- "${2:?}"' _ "$$" "$WORK" < /dev/null > /dev/null 2>&1 & FAKE="$ROOT/scripts/lib/model_ladder_cells_produce_cases/fake_apr.py" cat > "$WORK/ladder.yaml" <<'EOF' From 934a1586f0451e8493f35d6eb98e4da5a1c80b6d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:55:28 +0200 Subject: [PATCH 06/10] fix(ladder): hung health-wait case asserts the stall event, not stall+3 ticks (K1, #5798 flake) (squashed from #5832) Refs #5798 PMAT-Ticket: PMAT-5798 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- scripts/check_ladder_serve_teardown.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/check_ladder_serve_teardown.sh b/scripts/check_ladder_serve_teardown.sh index a5350653f2..197be6bbaf 100755 --- a/scripts/check_ladder_serve_teardown.sh +++ b/scripts/check_ladder_serve_teardown.sh @@ -222,15 +222,20 @@ extract_wait() { printf '%s\n' "$body" } -wait_case() { # - local name="$1" mode="$2" secs="$3" stall="$4" ceil="$5" want="$6" maxs="$7" port pid out verdict took +# is the health wait's own tick count (one tick per loop turn), not wall time. For `hung` it is NOT +# a fixed number: the quiet counter restarts on any CPU-jiffy the tree burns, and a loaded host burns a few +# (python start-up, scheduler wake-ups), so 3 quiet ticks took 3..7 turns (9 of 40 runs read 4 under load, #5798). +# The contract is the EVENT: verdict `stalled` (not ceiling, not ready, not died) after at least quiet +# ticks (never early) and before the ceiling. The bound for `hung` is therefore the ceiling, not stall+3. +wait_case() { # [] + local name="$1" mode="$2" secs="$3" stall="$4" ceil="$5" want="$6" maxs="$7" mins="${8:-0}" port pid out verdict took port=$(( 20000 + (RANDOM % 20000) )) flock -w 5 "$LOCK" python3 "$TMP/fake_loader.py" "$port" "$mode" "$secs" "$TMP/$name.loader.pid" > "$TMP/$name.log" 2>&1 & pid=$! out=$(ladder_serve_wait_health "$pid" "$port" "$TMP/$name.log" "$stall" "$ceil") || true verdict=${out%% *}; took=${out##* } reap_pidfile "$TMP/$name.loader.pid"; kill -9 "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true - if [ "$verdict" != "$want" ] || [ "$took" -gt "$maxs" ]; then - echo " FAIL $name: '$out' — want '$want' within ${maxs}s"; return 1 + if [ "$verdict" != "$want" ] || [ "$took" -gt "$maxs" ] || [ "$took" -lt "$mins" ]; then + echo " FAIL $name: '$out' — want '$want' in ${mins}..${maxs} ticks"; return 1 fi echo " ok $name: '$out'" } @@ -242,7 +247,7 @@ run_wait_cases() { # -> 0 if every case lands eval "$body" wait_case slow-log slow-log 6 3 30 ready 10 || fails=1 wait_case slow-cpu slow-cpu 6 3 30 ready 10 || fails=1 - wait_case hung hung 60 3 30 stalled 6 || fails=1 + wait_case hung hung 60 3 30 stalled 29 3 || fails=1 wait_case dies dies 2 5 30 died 4 || fails=1 wait_case forever forever 0 3 6 ceiling 9 || fails=1 return "$fails" From 26383aede3f55634fe31f88bd1fd3b6c081bbf4e Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:53:42 +0200 Subject: [PATCH 07/10] Merge remote-tracking branch 'origin/main' into work/5738c (squashed from #5738) Refs #5394 PMAT-Ticket: PMAT-5394 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- .github/workflows/ci.yml | 23 ++++++++++++++++-- ci/step-budgets.tsv | 6 +++++ scripts/check_ci_failfast.sh | 41 +++++++++++++++++++++++++++++++++ scripts/ci/step_budget_check.sh | 31 +++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 ci/step-budgets.tsv create mode 100755 scripts/check_ci_failfast.sh create mode 100755 scripts/ci/step_budget_check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e8610109d..903e844daf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,7 +198,7 @@ jobs: if: ${{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' && github.event.pull_request.draft != true }} # pull_request/push stay on the shared clean-room pool; a merge_group entry takes the mq lane (custom label exactly mq) runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 240 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 240 }} # The sections carry their own permissions blocks (the sov provenance section # needs id-token + attestations); one job token must cover all of them. permissions: write-all @@ -304,6 +304,14 @@ jobs: # the teardown below keeps always() and still reaps the containers. if: ${{ !cancelled() }} run: python3 scripts/ci/fat_driver.py wait --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver @@ -530,7 +538,10 @@ jobs: if: ${{ !cancelled() && needs.mg-reuse-wt.outputs.wt != '1' && github.event.pull_request.draft != true }} # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} - timeout-minutes: 180 + timeout-minutes: ${{ github.event_name == 'merge_group' && 75 || 180 }} + permissions: + contents: read + actions: write # the fail-fast step cancels this run strategy: fail-fast: false matrix: @@ -555,6 +566,14 @@ jobs: python3 scripts/ci/fat_driver.py run --sections 'workspace-test-shard?${{ matrix.shard }}?${{ matrix.shards }}?' --results "$RUNNER_TEMP/fat/results.json" + # Operator 2026-10-19: root cause is no gate time budget. A merge_group entry that is already RED must + # not keep the other jobs and shards holding mq runners for another ~45 min: the first failed job cancels + # the run. merge_group only (a PR run keeps every verdict); the teardown below is always() and still reaps. + - name: Fail fast, cancel this merge-group run + if: ${{ failure() && github.event_name == 'merge_group' }} + env: + GH_TOKEN: ${{ github.token }} + run: gh run cancel "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" || echo "::warning::fail-fast cancel did not take" # #4948: a cancelled run never reaches a section's own container removal, # and its fat--
container then sits on `tail -f /dev/null` # for days. always() runs on cancel too; on the passing path the driver diff --git a/ci/step-budgets.tsv b/ci/step-budgets.tsv new file mode 100644 index 0000000000..75120c1ac2 --- /dev/null +++ b/ci/step-budgets.tsv @@ -0,0 +1,6 @@ +# jobstepbudget_seconds. A step over 1.5x its budget is flagged by scripts/ci/step_budget_check.sh. +# Seeded 2026-10-19 from MG run 39205769238 (shard norm ~24 min) and the guard-cargo green time after #5712 (~45 min x86-main). The owner tunes them from data. +x86-main Sections (wait for every one to finish) 2700 +workspace-test-shard (1) Sections 1440 +workspace-test-shard (2) Sections 1440 +workspace-test-shard (3) Sections 1440 diff --git a/scripts/check_ci_failfast.sh b/scripts/check_ci_failfast.sh new file mode 100755 index 0000000000..a185000244 --- /dev/null +++ b/scripts/check_ci_failfast.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# check_ci_failfast.sh -- a RED merge_group job must cancel its run (operator 2026-10-19, refs #5394). +# x86-main and workspace-test-shard each carry a `gh run cancel` step guarded by +# `failure() && github.event_name == 'merge_group'`, and the shard job holds `actions: write`. +# Also runs scripts/ci/step_budget_check.sh --self-test and checks every budget row has 3 fields. +# bash scripts/check_ci_failfast.sh [workflow] | --self-test +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +# job_has_failfast : 0 when the job block holds the guarded cancel step +job_has_failfast() { + awk -v job="$2" ' + $0 ~ "^ " job ":" { on = 1; next } + on && /^ [a-z0-9-]+:/ { on = 0 } + on && /if: .*failure\(\).*github\.event_name == .merge_group./ { g = 1 } + on && g && /gh run cancel "\$GITHUB_RUN_ID"/ { ok = 1 } + END { exit ok ? 0 : 1 }' "$1" +} + +check() { + local wf=$1 rc=0 j + for j in x86-main workspace-test-shard; do + job_has_failfast "$wf" "$j" || { echo "FAIL: job $j has no merge_group failure() -> gh run cancel step"; rc=1; } + done + awk '/^ workspace-test-shard:/{on=1;next} on&&/^ [a-z0-9-]+:/{on=0} on&&/actions: write/{f=1} END{exit f?0:1}' "$wf" \ + || { echo "FAIL: workspace-test-shard lacks 'actions: write' (gh run cancel would 403)"; rc=1; } + return $rc +} + +case "${1:-}" in -h|--help) echo "usage: check_ci_failfast.sh [workflow] | --self-test"; exit 0 ;; esac + +if [ "${1:-}" = "--self-test" ]; then + check .github/workflows/ci.yml || { echo "FAIL: the real workflow is not fail-fast"; exit 1; } + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + grep -v 'gh run cancel' .github/workflows/ci.yml > "$t/mut.yml" + check "$t/mut.yml" > /dev/null && { echo "FAIL: a workflow with the cancel step removed still passed"; exit 1; } + bash scripts/ci/step_budget_check.sh --self-test || exit 1 + awk -F'\t' '!/^#/ && NF != 3 { bad = 1 } END { exit bad }' ci/step-budgets.tsv || { echo "FAIL: ci/step-budgets.tsv row without 3 fields"; exit 1; } + echo "ok: check_ci_failfast case table"; exit 0 +fi +check "${1:-.github/workflows/ci.yml}" && echo "ok: merge_group failures cancel the run" diff --git a/scripts/ci/step_budget_check.sh b/scripts/ci/step_budget_check.sh new file mode 100755 index 0000000000..547032b300 --- /dev/null +++ b/scripts/ci/step_budget_check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# step_budget_check.sh -- flag any CI step that ran longer than 1.5x its budget (ci/step-budgets.tsv). +# Operator 2026-10-19: "per-step budgets, with CI flagging any step >1.5x its budget". Reads the run's jobs +# API (GraphQL-free REST, one call). Prints one line per breach; exit 0 always on a measured run, 2 when the +# run cannot be read (not_measured, never a pass). A LAB signal: run on demand or by the nightly LAB lane, never in a PR or merge job (C324 rule 5/6). +# bash scripts/ci/step_budget_check.sh bash scripts/ci/step_budget_check.sh --self-test +set -uo pipefail +BUD=${STEP_BUDGETS:-ci/step-budgets.tsv} + +# breaches: stdin = "jobstepseconds" rows; $BUD rows; prints "BREACH job | step | Ns > 1.5 x Bs" +breaches() { + awk -F'\t' 'NR==FNR { if ($0 !~ /^#/ && NF >= 3) b[$1 FS $2] = $3; next } + ($1 FS $2) in b { lim = b[$1 FS $2] * 1.5; if ($3 > lim) printf "BREACH %s | %s | %ds > 1.5 x %ds\n", $1, $2, $3, b[$1 FS $2] }' "$BUD" - +} + +if [ "${1:-}" = "--self-test" ]; then + t=$(mktemp -d); trap 'rm -rf "${t:?}"' EXIT + printf 'j\ts\t100\n' > "$t/b.tsv" + r1=$(printf 'j\ts\t151\n' | BUD="$t/b.tsv" breaches) + r2=$(printf 'j\ts\t150\n' | BUD="$t/b.tsv" breaches) + r3=$(printf 'other\ts\t999\n' | BUD="$t/b.tsv" breaches) + [ -n "$r1" ] && [ -z "$r2" ] && [ -z "$r3" ] || { echo "FAIL: 151>1.5x100 must breach; 150 and an unbudgeted step must not"; exit 1; } + echo "ok: step_budget_check case table"; exit 0 +fi + +run=${1:?usage: step_budget_check.sh |--self-test} +rows=$(gh api "repos/${GITHUB_REPOSITORY:-paiml/aprender}/actions/runs/$run/jobs?per_page=100" --jq \ + '.jobs[] | .name as $j | .steps[]? | select(.started_at and .completed_at) | [$j, .name, ((.completed_at|fromdate) - (.started_at|fromdate))] | @tsv') \ + || { echo "NOT_MEASURED: jobs API read failed for run $run"; exit 2; } +out=$(printf '%s\n' "$rows" | breaches) +if [ -n "$out" ]; then printf '%s\n' "$out" | while read -r l; do echo "::error::$l"; done; else echo "no step over 1.5x budget (run $run)"; fi From 36e44f1908cfe200f53b031fd6a55aa068f7c438 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:57:50 +0200 Subject: [PATCH 08/10] lab: apr stub race (squashed from #5874) Refs #5873 PMAT-Ticket: PMAT-5873 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- docs/roadmaps/roadmap.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 3cf166d272..5ae38ad180 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -25416,6 +25416,26 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 5510 +- id: PMAT-5847 + github_issue: 5847 + item_type: task + title: 'LAB nightlies: timeout-minutes on every job and end by 05:00Z (falsifier check_lab_timeouts.sh)' + status: in_progress + priority: high + assigned_to: null + created: 2026-10-21T11:30:00Z + updated: 2026-10-21T11:30:00Z + spec: null + acceptance_criteria: + - 'scripts/check_lab_timeouts.sh is RED on the pre-fix tree (22 findings on the merge-base) and GREEN after; --self-test 6 rows' + - 'reaper catching queued + schedule runs is a separate change to the deployed pr-run-reaper.sh' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 - id: PMAT-5873 github_issue: 5873 item_type: task From b75222add3dd0db2b7821baf087501ad2991ed60 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 12:58:01 +0200 Subject: [PATCH 09/10] ci(lab): kfocus shadow lab workflow + runner-label check (squashed from #5887) Refs #5889 PMAT-Ticket: PMAT-5889 Co-Authored-By: Claude Sonnet 5.5 PMAT-Batch: BATCH-aprender-build-speed --- .github/actionlint.yaml | 2 + .github/workflows/kfocus-shadow-lab.yml | 179 ++++++++++++++++++++++++ docs/roadmaps/entries/PMAT-5889.yaml | 19 +++ docs/roadmaps/roadmap.yaml | 19 +++ scripts/check_runner_labels.sh | 3 +- 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/kfocus-shadow-lab.yml create mode 100644 docs/roadmaps/entries/PMAT-5889.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 868efd8883..a33d55a44d 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -30,3 +30,5 @@ self-hosted-runner: - review-light # #5254: the 8 GB ARM64 CUDA edge device's LAB nightly (edge-fast-verbs-nightly) - jetson + # #5889: the Kubuntu Focus host, kfocus-shadow-lab (dispatch-only LAB) + - kfocus diff --git a/.github/workflows/kfocus-shadow-lab.yml b/.github/workflows/kfocus-shadow-lab.yml new file mode 100644 index 0000000000..9aeec189ba --- /dev/null +++ b/.github/workflows/kfocus-shadow-lab.yml @@ -0,0 +1,179 @@ +# kfocus shadow parity vs intel -- LAB (C324), dispatch only. +# +# Operator ruling 2026-10-21 17:15Z: run the SAME x86-main job as ci.yml on the kfocus +# host and compare the verdict with intel's. This is a LAB check: it never runs inside +# a PR, merge-queue or release job, so the only trigger is workflow_dispatch. Do not add +# pull_request, push, merge_group or schedule here. +# +# FULL TIER, never reuse (the 13:35Z parity run was "all tier=reuse, so no tests ran"): +# the tier is decided inside the guard-cargo/sov sections by ci/sections.yml's +# "Decide the test tier" step -> scripts/ci_test_tier.sh, which answers tier=full for +# event workflow_dispatch (scripts/ci_test_tier.sh, the `schedule|workflow_dispatch` arm). +# Reuse needs a `push` event plus a merge_group result and the mg-reuse job, which this +# workflow does not have. The "Pin the tier" step below asserts that arm still says full, +# so a future edit that gives dispatch a quick/reuse tier turns this lane red instead of +# silently running nothing. +name: kfocus-shadow-lab + +on: + workflow_dispatch: + inputs: + sha: + description: Commit to test (full 40-char sha) + required: true + type: string + +permissions: + contents: read + +concurrency: + group: kfocus-shadow-${{ inputs.sha }} + cancel-in-progress: false + +jobs: + x86-main-shadow: + runs-on: [self-hosted, Linux, X64, kfocus] + timeout-minutes: 240 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FAT_VARS_JSON: ${{ toJSON(vars) }} + FAT_SECRET_PR_REVIEW_SIGNING_KEY_B64: ${{ secrets.PR_REVIEW_SIGNING_KEY_B64 }} + steps: + # The fleet pins jq 1.8.2. Runners in this clean-room pool whose image ships jq 1.6 + # print a number literal such as 90.00 as 90, so the guard-tree section turns RED + # (scripts/release/tag_coverage_gate.sh --self-test, run by check_tag_coverage_gated.sh). + # This step puts the pinned jq first on PATH for every later step, and through it for + # every section fat_driver runs on the host (step_env copies the driver's PATH; a + # section with a job container gets the image's PATH, and none of those calls jq). + # Fail closed: a checksum, version or behaviour mismatch stops the job. The sha256 is + # the one in the upstream release's sha256sum.txt for jq-linux-amd64. + - name: Pin jq 1.8.2 (sha256-verified, fail closed) + env: + JQ_URL: https://github.com/jqlang/jq/releases/download/jq-1.8.2/jq-linux-amd64 + JQ_SHA256: b1c22172dd303f3be49e935aa56aa48a8b7a46e0bc838b4997d3bb451495870f + run: | + set -euo pipefail + d="$RUNNER_TEMP/jq-1.8.2" + mkdir -p "$d" + curl -fsSL --retry 3 --retry-delay 5 -o "$d/jq" "$JQ_URL" + printf '%s %s\n' "$JQ_SHA256" "$d/jq" | sha256sum -c - + chmod +x "$d/jq" + v=$("$d/jq" --version) + [ "$v" = "jq-1.8.2" ] || { echo "::error::the pinned jq reports '$v', not jq-1.8.2"; exit 1; } + rc=0; "$d/jq" -e . < /dev/null > /dev/null || rc=$? + [ "$rc" = 4 ] || { echo "::error::the pinned jq exits $rc on empty input with -e, not 4"; exit 1; } + echo "$d" >> "$GITHUB_PATH" + - name: jq on PATH is the pinned one + run: | + set -euo pipefail + command -v jq; jq --version + [ "$(command -v jq)" = "$RUNNER_TEMP/jq-1.8.2/jq" ] || { echo "::error::jq on PATH is $(command -v jq), not the pinned one"; exit 1; } + [ "$(jq --version)" = "jq-1.8.2" ] || { echo "::error::jq on PATH reports $(jq --version)"; exit 1; } + # Full history: a section whose own checkout asked for depth 1 (the default) + # is cut back to exactly that by the driver; one asking for 0 needs it all. + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + # Fail fast: the reds car#4429 took four fat runs to see, in ~2 min, before + # any section starts. rustfmt runs in the image sov.lint uses (same digest, + # so the same verdict); the ID-order rule is contract_data_integrity's, + # without cargo; the case tables are guard-tree's cheapest. + - name: Cheap checks (fmt, falsification-ID order, guard case tables) + run: | + set -euo pipefail + python3 scripts/ci/fail_fast.py self-test + python3 scripts/ci/fail_fast.py ids --base origin/main + docker run --rm -v "$GITHUB_WORKSPACE:/w:ro" -w /w \ + localhost:5000/sovereign-ci:stable@sha256:70f2cdc9e537fe4447306d33c317fea33017f78f1ffa3b7af99095391419ef36 \ + cargo fmt --all -- --check + bash scripts/tests/guard_tree_test.sh + bash scripts/bump-version.sh --self-test + setsid --wait bash scripts/check_no_claim_literals.sh --selftest + setsid --wait bash scripts/check_hardcoded_paths.sh --self-test + setsid --wait bash scripts/check_pr_review_wiring.sh --self-test + setsid --wait bash scripts/ci/fixture_quiet.sh --self-test + setsid --wait bash scripts/ci/fetch_p6.sh --self-test + setsid --wait bash scripts/ci/fetch_p6.sh --check-wiring + # The driver decides what every section's if:/matrix/needs means and whether + # the vendored workflow is the upstream one; its case table must go RED on + # each planted defect before it is trusted to report a section green. + # #4596: its fixture rows print real ::error:: lines; fixture_quiet.sh shows + # them as labelled text, never as job annotations, and keeps the exit status. + - name: fat_driver case table must still turn RED + run: setsid --wait bash scripts/ci/fixture_quiet.sh "fat_driver self-test" -- python3 scripts/ci/fat_driver.py self-test + # PMAT-4510: every secrets. a section reads must be plumbed above as + # FAT_SECRET_ under its full name; #4441 cut one short and the + # receipt signer ran with an empty key on every PR. Text-only, no build. + - name: FAT_SECRET plumbing guard case table + run: setsid --wait bash scripts/check_ci_fat_secrets_plumbed.sh --self-test + - name: every section secret is plumbed under its full name + run: setsid --wait bash scripts/check_ci_fat_secrets_plumbed.sh + # #5403: the toolchain step installs `stable`; guard-cargo#m60 runs cargo without + # RUSTUP_TOOLCHAIN, so cargo read rust-toolchain.toml and downloaded the pinned + # release at test time (one network blip failed a merge group). Install the pin first. + - name: install the toolchain rust-toolchain.toml pins (case table) + run: setsid --wait bash scripts/ci/install_pinned_toolchain.sh --self-test + - name: install the toolchain rust-toolchain.toml pins + run: setsid --wait bash scripts/ci/install_pinned_toolchain.sh + # LAB: pin the FULL tier. The dispatch arm of ci_test_tier.sh must say full (never reuse, never quick). + - name: Pin the tier to full (dispatch must never reuse or skip) + run: | + set -euo pipefail + out=$(bash scripts/ci_test_tier.sh --event workflow_dispatch) + printf "%s\n" "$out" + printf "%s\n" "$out" | grep -qx "tier=full" || { echo "::error::workflow_dispatch no longer yields tier=full; this shadow run would test nothing"; exit 1; } + - name: Sections (the determinism raster returns first; the rest keep running) + run: >- + python3 scripts/ci/fat_driver.py run + --sections 'sov.*,guard-tree,guard-cargo,vendored-schemas,pr-review-shadow,pr-review-sign,determinism[X64]' + --results "$RUNNER_TEMP/fat/results.json" + --background-until 'determinism[X64]' + # Published mid-run: the determinism job's compare section is waiting for it. + - uses: actions/upload-artifact@v7 + with: + name: determinism-X64 + path: ${{ runner.temp }}/fat/artifacts/determinism-X64/ + if-no-files-found: error + - name: Sections (wait for every one to finish) + id: wait + # A cancel must stop this poll (orphan queue runs held runners, 2026-10-16); + # the teardown below keeps always() and still reaps the containers. + if: ${{ !cancelled() }} + run: python3 scripts/ci/fat_driver.py wait --results "$RUNNER_TEMP/fat/results.json" + # #4948: a cancelled run never reaches a section's own container removal, + # and its fat--
container then sits on `tail -f /dev/null` + # for days. always() runs on cancel too; on the passing path the driver + # has already removed every container and this finds none. Cancelled + # before the checkout, no driver ran and there is no script to call. + - name: Remove this job's section containers (runs on cancel too) + if: always() + run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi + - uses: actions/upload-artifact@v7 + if: always() + with: + name: pr-review-shadow-sample + path: ${{ runner.temp }}/fat/artifacts/pr-review-shadow-sample/ + retention-days: 90 + if-no-files-found: ignore + - uses: actions/upload-artifact@v7 + if: always() + with: + name: bench-${{ github.event.repository.name }} + path: ${{ runner.temp }}/fat/artifacts/bench-${{ github.event.repository.name }}/ + retention-days: 90 + if-no-files-found: ignore + # #5397: free this job's own target dir (`/run-`), never the PR parent. + # Last step, so nothing after it reads the dir; always() so a failed or + # cancelled run frees it too. + - name: Free this job's own cargo target dir + if: always() + run: if [ -f scripts/ci/free_own_target_dir.sh ]; then bash scripts/ci/free_own_target_dir.sh "${{ github.event.pull_request.number || github.ref_name }}" "$GITHUB_RUN_ID"; fi + - name: Shadow verdict + if: always() + env: + SHADOW_SHA: ${{ inputs.sha }} + JOB_STATUS: ${{ job.status }} + run: | + result=failure; [ "$JOB_STATUS" = success ] && result=success + echo "SHADOW-VERDICT sha=${SHADOW_SHA} runner=${RUNNER_NAME} result=${result}" diff --git a/docs/roadmaps/entries/PMAT-5889.yaml b/docs/roadmaps/entries/PMAT-5889.yaml new file mode 100644 index 0000000000..05b3a7ea9a --- /dev/null +++ b/docs/roadmaps/entries/PMAT-5889.yaml @@ -0,0 +1,19 @@ +- id: PMAT-5889 + github_issue: 5889 + item_type: task + title: 'LAB kfocus-shadow-lab.yml: dispatch-only x86-main shadow on kfocus (OP 17:15Z parity)' + status: planned + priority: high + assigned_to: null + created: 2026-10-21T17:44:17Z + updated: 2026-10-21T17:44:17Z + spec: null + acceptance_criteria: + - 'Dispatch-only LAB workflow (C324) running x86-main at a given sha on kfocus, to compare verdicts with intel. PR #5887.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:measurement + notes: null + epic: 5510 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 5ae38ad180..403a41bfc2 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -25494,6 +25494,25 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 5510 +- id: PMAT-5889 + github_issue: 5889 + item_type: task + title: 'LAB kfocus-shadow-lab.yml: dispatch-only x86-main shadow on kfocus (OP 17:15Z parity)' + status: planned + priority: high + assigned_to: null + created: 2026-10-21T17:44:17Z + updated: 2026-10-21T17:44:17Z + spec: null + acceptance_criteria: + - 'Dispatch-only LAB workflow (C324) running x86-main at a given sha on kfocus, to compare verdicts with intel. PR #5887.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:measurement + notes: null + epic: 5510 - id: PMAT-5914 github_issue: 5914 item_type: task diff --git a/scripts/check_runner_labels.sh b/scripts/check_runner_labels.sh index f765cfd06d..67fa1cd144 100755 --- a/scripts/check_runner_labels.sh +++ b/scripts/check_runner_labels.sh @@ -24,7 +24,8 @@ cd "$(dirname "$0")/.." # review-light (#4729): one runner on intel-clean-room-15, same unit and isolation as the # clean-room pool, deliberately WITHOUT clean-room so build jobs cannot queue in front of # pr-review-quorum `present` (a seconds-long check). A one-box label, like perf-solo. -DISCRIM='clean-room|cuda|gpu|rtx4090|ada|blackwell|gb10|apple-silicon|m4|perf-solo|mini-builder|review-light' +# kfocus (#5889): the Kubuntu Focus host's own label; one box, so it discriminates like perf-solo. +DISCRIM='clean-room|cuda|gpu|rtx4090|ada|blackwell|gb10|apple-silicon|m4|perf-solo|mini-builder|review-light|kfocus' fail=0 while IFS=: read -r file line sel; do From d63346dbfddf92e9bc59836ccbcd05f6f0586595 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 22 Oct 2026 13:57:55 +0200 Subject: [PATCH 10/10] fix(ci): ladder-cells watcher in a subshell so the bare wait cannot deadlock; step-budgets names the shard job (1, 1) Review fixes for #5927 (w5032 REQUEST-CHANGES): the setsid watcher was a child of the top-level shell, so wait at the end blocked on it (rc=124); ( setsid ... & ) gives rc=0. GitHub names the single-shard job 'workspace-test-shard (1, 1)'; rows 2 and 3 no longer exist. PMAT-Ticket: PMAT-5926 PMAT-Batch: BATCH-aprender-build-speed Co-Authored-By: Claude Sonnet 5.5 --- ci/step-budgets.tsv | 4 +--- scripts/check_ladder_cells_producer.sh | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ci/step-budgets.tsv b/ci/step-budgets.tsv index 75120c1ac2..d7fa299239 100644 --- a/ci/step-budgets.tsv +++ b/ci/step-budgets.tsv @@ -1,6 +1,4 @@ # jobstepbudget_seconds. A step over 1.5x its budget is flagged by scripts/ci/step_budget_check.sh. # Seeded 2026-10-19 from MG run 39205769238 (shard norm ~24 min) and the guard-cargo green time after #5712 (~45 min x86-main). The owner tunes them from data. x86-main Sections (wait for every one to finish) 2700 -workspace-test-shard (1) Sections 1440 -workspace-test-shard (2) Sections 1440 -workspace-test-shard (3) Sections 1440 +workspace-test-shard (1, 1) Sections 1440 diff --git a/scripts/check_ladder_cells_producer.sh b/scripts/check_ladder_cells_producer.sh index 4db96dca67..3da8ee7157 100755 --- a/scripts/check_ladder_cells_producer.sh +++ b/scripts/check_ladder_cells_producer.sh @@ -32,9 +32,9 @@ reap_work() { trap reap_work EXIT # EXIT does not run on SIGKILL (a cancelled or timed-out job): a detached watcher repeats the sweep once this shell is gone # (8 fake serves leaked on lambda, 2026-10-21). The dir is removed by the same name, never by a pattern. -setsid bash -c 'while kill -0 "$1" 2>/dev/null; do sleep 2; done +( setsid bash -c 'while kill -0 "$1" 2>/dev/null; do sleep 2; done for p in /proc/[0-9]*; do c=$(tr "\0" " " 2>/dev/null < "$p/cmdline") || continue; case "$c" in *"$2/"*) kill -TERM "${p#/proc/}" 2>/dev/null ;; esac; done - rm -rf -- "${2:?}"' _ "$$" "$WORK" < /dev/null > /dev/null 2>&1 & + rm -rf -- "${2:?}"' _ "$$" "$WORK" < /dev/null > /dev/null 2>&1 & ) FAKE="$ROOT/scripts/lib/model_ladder_cells_produce_cases/fake_apr.py" cat > "$WORK/ladder.yaml" <<'EOF'