diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de2f17321f..8f7307b7f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,11 +51,11 @@ jobs: # runs all parts to completion on the biggest runner in its class, nextest # full-core, sccache warm. Same test set as today (Σ executed unchanged)." # - # The five, by job id: x86-main, gx10, yoga, determinism, and mac-check -- the - # ruling's "mac-refusal": the macOS lane, whose one expected failure is the - # Linux-only profiler's compile_error! refusal (excluded by name below). It - # keeps its context name `mac-check` and, as on main before #4433, is NOT in - # `gate`'s needs (its own comment: first-green proof before it may block). + # The five, by job id, were: x86-main, gx10, yoga, determinism, and mac-check + # (the ruling's "mac-refusal"). gx10, yoga and mac-check were never in `gate`'s + # needs, so none could block a merge or a release; Q6 quorum, C324 moved all + # three to LAB (nightly only): gx10/yoga to gpu-lab-nightly.yml, mac-check to + # nightly.yml. x86-main and determinism are the fat jobs this workflow runs. # # The job BODIES did not change: they moved, verbatim, to ci/sections.yml # (and the pinned sovereign-ci reusable workflow to ci/vendor/). Each fat job @@ -83,7 +83,9 @@ jobs: # commit the queue had just passed. On that push, mg-reuse reads the merge_group # run on the same sha instead (P1-P4 in scripts/ci_mg_reuse.sh), and reuses only # when that run RAN both jobs and they passed -- never a queue run that itself - # reused. mac-check is never reused on a push. + # reused. mac-check is never reused on a push. Q6 quorum, C324: mac-check left + # this workflow, so its `mac` key is no longer an output here; ci_mg_reuse.sh + # still computes it (its case table and check_ci_push_reuse.sh read it). mg-reuse: if: github.event_name == 'merge_group' || (github.event_name == 'push' && github.ref == 'refs/heads/main') runs-on: [self-hosted, Linux, X64, clean-room] @@ -95,7 +97,6 @@ jobs: outputs: x86: ${{ steps.key.outputs.x86 }} det: ${{ steps.key.outputs.det }} - mac: ${{ steps.key.outputs.mac }} cite: ${{ steps.key.outputs.cite }} steps: # Full history: the key reads the queue head's parent, the base's parent and @@ -122,7 +123,7 @@ jobs: *) out=$(bash scripts/ci_mg_reuse.sh resolve --event "$EVT" --ref "$QREF" --queue-head "$QHEAD" --base "$QBASE" --repo "$REPO") ;; esac printf '%s\n' "$out" - printf '%s\n' "$out" | grep -E '^(x86|det|mac|cite|pr_head)=' >> "$GITHUB_OUTPUT" + printf '%s\n' "$out" | grep -E '^(x86|det|cite|pr_head)=' >> "$GITHUB_OUTPUT" # The same case table on every event, so a change that breaks the key is RED on # its own PR, not first in the queue. gate requires it. @@ -312,112 +313,12 @@ jobs: export GIT_CONFIG_VALUE_0 setsid --wait bash scripts/check_deny_exemptions_live.sh - # The GPU decision stays OFF the GPU hosts, as before #4433: an `if:` that - # could only be evaluated on gx10/yoga would queue the job to answer it, and - # a queued job has already claimed the fleet's one GB10 / one RTX 4060 (and - # displaced a pending job in its perf-* group). A ten-second decision on the - # clean-room pool, not a sixth fat job; gx10 and yoga run only on a - # pull_request that it says touches the GPU set, exactly as gpu-quick and - # cuda-unit did. scripts/tests/ci_gpu_touched_test.sh pins this shape. - gpu-touched: - runs-on: [self-hosted, X64, Linux, clean-room] - timeout-minutes: 10 - if: github.event_name == 'pull_request' - outputs: - gpu_touched: ${{ steps.decide.outputs.gpu_touched }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FAT_VARS_JSON: ${{ toJSON(vars) }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Sections - id: run - run: >- - python3 scripts/ci/fat_driver.py run - --sections 'gpu-touched' - --results "$RUNNER_TEMP/fat/results.json" - # #4948: a cancelled run never reaches a section's own container removal, - # and its fat--
container then sits on `tail -f /dev/null` - # for days. always() runs on cancel too; on the passing path the driver - # has already removed every container and this finds none. Cancelled - # before the checkout, no driver ran and there is no script to call. - - name: Remove this job's section containers (runs on cancel too) - if: always() - run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi - - name: The decision, as this job's output - id: decide - run: | - set -euo pipefail - jq -r '"gpu_touched=" + ((."gpu-touched".outputs.gpu_touched // "0") | tostring)' \ - "$RUNNER_TEMP/fat/results.json" | tee -a "$GITHUB_OUTPUT" - - gx10: - runs-on: [self-hosted, Linux, ARM64, cuda, gx10, ephemeral, docker] - timeout-minutes: 30 - needs: [gpu-touched] - if: github.event_name == 'pull_request' && needs.gpu-touched.outputs.gpu_touched == '1' - # Shared with the other gx10 perf workflows (qwen-story-daily, silicon-nightly, - # cuda-nightly, b2-gpu): one GPU job on the box at a time. - concurrency: - group: perf-gx10 - cancel-in-progress: false - outputs: - results: ${{ steps.run.outputs.results }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FAT_VARS_JSON: ${{ toJSON(vars) }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Sections - id: run - run: >- - python3 scripts/ci/fat_driver.py run - --sections 'gpu-touched,gpu-quick' - --results "$RUNNER_TEMP/fat/results.json" - # #4948: a cancelled run never reaches a section's own container removal, - # and its fat--
container then sits on `tail -f /dev/null` - # for days. always() runs on cancel too; on the passing path the driver - # has already removed every container and this finds none. Cancelled - # before the checkout, no driver ran and there is no script to call. - - name: Remove this job's section containers (runs on cancel too) - if: always() - run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi - - yoga: - runs-on: [self-hosted, Linux, X64, cuda, yoga, ephemeral, docker] - timeout-minutes: 50 - needs: [gpu-touched] - if: github.event_name == 'pull_request' && needs.gpu-touched.outputs.gpu_touched == '1' - concurrency: - group: perf-yoga - cancel-in-progress: false - outputs: - results: ${{ steps.run.outputs.results }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FAT_VARS_JSON: ${{ toJSON(vars) }} - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Sections - id: run - run: >- - python3 scripts/ci/fat_driver.py run - --sections 'gpu-touched,cuda-unit' - --results "$RUNNER_TEMP/fat/results.json" - # #4948: a cancelled run never reaches a section's own container removal, - # and its fat--
container then sits on `tail -f /dev/null` - # for days. always() runs on cancel too; on the passing path the driver - # has already removed every container and this finds none. Cancelled - # before the checkout, no driver ran and there is no script to call. - - name: Remove this job's section containers (runs on cancel too) - if: always() - run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi + # gpu-touched, gx10 and yoga LEFT this workflow: Q6 quorum, C324 (DELETE-TO-LAB + # 3/3). gpu-quick and cuda-unit were never in the gate, so they could not block + # a merge or a release, and a check that cannot block is LAB: it never runs in + # a PR or merge-queue job. The sections keep their bodies in ci/sections.yml; + # their one home is .github/workflows/gpu-lab-nightly.yml (gx10 in perf-gx10, + # yoga in perf-yoga). scripts/tests/ci_gpu_touched_test.sh pins that shape. determinism: needs: [mg-reuse, x86-main] @@ -479,39 +380,11 @@ jobs: path: ${{ runner.temp }}/fat/artifacts/raster-receipt/ if-no-files-found: ignore - # PACK-001: mini-m4 had taken ZERO jobs in the 24 h measured -- nothing in any - # workflow named macOS. The multiplatform dogfood requires a mini receipt at - # post-publish and the three macOS-only defect classes on record (#2567 aarch64 - # scalar GEMV, #2568 an OOM guard that reads /proc/meminfo, #2572 a metal-only - # dep) were each found only by building there. This lane is the cheap version: - # a locked workspace check plus the CLI's lib tests, on the pinned toolchain - # (rust-toolchain.toml; rustup on mini holds 1.93.0). No docker on macOS, so - # the runner's own persistent target dir is used and nothing is bind-mounted. - # Not in `gate`'s needs yet: first-green proof on a real PR before it may block. - mac-check: - needs: [mg-reuse] - if: ${{ !cancelled() && needs.mg-reuse.outputs.mac != '1' }} - runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] - timeout-minutes: 60 - env: - CARGO_INCREMENTAL: 0 - CARGO_TERM_COLOR: never - steps: - - uses: actions/checkout@v4 - - name: Toolchain is the pinned one - run: | - set -euo pipefail - rustup show active-toolchain - cargo --version - # First run on mini (PR #3488, 2026-09-18) found the first macOS-only fact - # in two minutes: crates/aprender-profile/src/main.rs is a compile_error! - # on non-Linux by design ("renacer requires Linux (ptrace syscall - # tracing)"). That crate is excluded here on purpose; any OTHER crate that - # fails to check on macOS is a finding this lane exists to surface. - - name: cargo check --workspace --all-targets --locked (minus the Linux-only profiler) - run: cargo check --workspace --all-targets --locked --exclude aprender-profile - - name: apr-cli lib tests (nextest, ci profile) - run: cargo nextest run --profile ci -p apr-cli --lib + # mac-check (PACK-001) LEFT this workflow: Q6 quorum, C324 (DELETE-TO-LAB 3/3). + # It was never in `gate`'s needs, so it could not block a merge or a release, + # and a check that cannot block is LAB: it never runs in a PR or merge-queue + # job. Its home is nightly.yml job `mac-check` (same mini labels, same two + # cargo steps), which no publish job needs. # Poka-yoke (aprender#2269): fail fast if ANY self-hosted job omits a # discriminating runner label, so a bare [self-hosted, X64, Linux] selector @@ -676,7 +549,8 @@ jobs: res() { printf '%s' "${1:-null}" | SEC="$2" jq -r '(. // {})[env.SEC].result // "missing"'; } # The old gate's needs, in its order: guard-tree, guard-cargo, ci (the # sovereign-ci gate), workspace-test, determinism-compare. gpu-quick and - # cuda-unit were never in the gate; they still are not. + # cuda-unit were never in the gate; Q6 quorum, C324 moved them to LAB + # (gpu-lab-nightly.yml), so no job of this workflow runs them. # workspace-test is its own job now (three shard jobs + the Σ fan-in). [ "$WT" = success ] || { echo "workspace-test failed: $WT"; exit 1; } [ "$MST" = success ] || { echo "mg-reuse-self-test failed: $MST"; exit 1; } diff --git a/.github/workflows/gpu-lab-nightly.yml b/.github/workflows/gpu-lab-nightly.yml new file mode 100644 index 0000000000..1511e678ea --- /dev/null +++ b/.github/workflows/gpu-lab-nightly.yml @@ -0,0 +1,71 @@ +# GPU LAB nightly — the one home of the gpu-quick (gx10) and cuda-unit (yoga) +# sections. Q6 quorum, C324 (DELETE-TO-LAB 3/3): neither was ever in ci.yml's +# gate, so neither could block a merge or a release; a check that cannot block +# is LAB, and LAB never runs in a PR, merge-queue or release job. These two jobs +# are ci.yml's former `gx10` and `yoga` fat jobs, minus the pull_request trigger +# and the gpu-touched PR-diff decision: on schedule and workflow_dispatch the +# gpu-touched section answers the whole GPU set (ci/sections.yml). +# +# No pull_request, merge_group or push trigger, on purpose. +name: GPU LAB nightly + +on: + schedule: + - cron: '0 21 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + gx10: + runs-on: [self-hosted, Linux, ARM64, cuda, gx10, ephemeral, docker] + timeout-minutes: 30 + # Shared with the other gx10 perf workflows (qwen-story-daily, silicon-nightly, + # cuda-nightly, b2-gpu): one GPU job on the box at a time. + concurrency: + group: perf-gx10 + cancel-in-progress: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FAT_VARS_JSON: ${{ toJSON(vars) }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Sections + run: >- + python3 scripts/ci/fat_driver.py run + --sections 'gpu-touched,gpu-quick' + --results "$RUNNER_TEMP/fat/results.json" + # #4948: a cancelled run never reaches a section's own container removal, + # and its fat--
container then sits on `tail -f /dev/null` + # for days. always() runs on cancel too; on the passing path the driver + # has already removed every container and this finds none. Cancelled + # before the checkout, no driver ran and there is no script to call. + - name: Remove this job's section containers (runs on cancel too) + if: always() + run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi + + yoga: + runs-on: [self-hosted, Linux, X64, cuda, yoga, ephemeral, docker] + timeout-minutes: 50 + concurrency: + group: perf-yoga + cancel-in-progress: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FAT_VARS_JSON: ${{ toJSON(vars) }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Sections + run: >- + python3 scripts/ci/fat_driver.py run + --sections 'gpu-touched,cuda-unit' + --results "$RUNNER_TEMP/fat/results.json" + # #4948: as in gx10 above. + - name: Remove this job's section containers (runs on cancel too) + if: always() + run: if [ -f scripts/ci/fat_teardown.sh ]; then bash scripts/ci/fat_teardown.sh "$GITHUB_RUN_ID" "$RUNNER_TEMP"; fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfabf0079..b3d83ccbed 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -235,8 +235,8 @@ jobs: # ── Build apr: macOS aarch64 natively on mini-m4 (#3204) ── # Host-native: there is no docker on macOS and no glibc floor to pin, so the - # toolchain is rust-toolchain.toml via the runner's rustup, as in ci.yml's - # mac-check. A LITERAL runs-on list, not fromJSON, so check_runner_labels.sh + # toolchain is rust-toolchain.toml via the runner's rustup, as in the + # mac-check job below. A LITERAL runs-on list, not fromJSON, so check_runner_labels.sh # sees the apple-silicon label. Gated on the same `gate` decision as the Linux # matrix (car's `check-activity` job does not exist on this branch -- folded # into `gate`'s reused/build/red-ci/ci-pending verdict, #4189). One box with @@ -292,6 +292,38 @@ jobs: apr-aarch64-apple-darwin.tar.gz apr-aarch64-apple-darwin.tar.gz.sha256 + # ── mac-check: LAB, moved here from ci.yml (Q6 quorum, C324) ── + # PACK-001's macOS lane: a locked workspace check plus the CLI's lib tests on + # the pinned toolchain. It was never in ci.yml's gate, so it could not block a + # merge or a release, and a check that cannot block is LAB: nightly only. The + # mini is one 16 GB box, so this runs AFTER build-darwin (always(): a red build + # still gets its check) and nothing waits on it -- neither `publish` nor + # `publish-darwin` needs this job, so it never holds the nightly publish. + # crates/aprender-profile is a compile_error! on non-Linux by design + # ("renacer requires Linux (ptrace syscall tracing)"), so it is excluded by + # name; any OTHER crate that fails to check on macOS is a finding. + mac-check: + needs: [gate, build-darwin] + if: always() && needs.gate.result == 'success' && needs.gate.outputs.decision != 'reused' + runs-on: [self-hosted, macOS, ARM64, apple-silicon, m4, mini] + timeout-minutes: 60 + env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: never + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.gate.outputs.sha }} + - name: Toolchain is the pinned one + run: | + set -euo pipefail + rustup show active-toolchain + cargo --version + - name: cargo check --workspace --all-targets --locked (minus the Linux-only profiler) + run: cargo check --workspace --all-targets --locked --exclude aprender-profile + - name: apr-cli lib tests (nextest, ci profile) + run: cargo nextest run --profile ci -p apr-cli --lib + # ── Publish per arch, then the manifest ────────────────── publish: needs: [gate, build] diff --git a/ci/sections.yml b/ci/sections.yml index 1dfd1089dc..aef7b9778f 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -2987,10 +2987,19 @@ jobs: # `schedule`, over the PRs that merged. # ========================================================================== - # ADVISORY GPU JOBS AT PR TIME + # GPU LAB SECTIONS (were: ADVISORY GPU JOBS AT PR TIME) # spec docs/specifications/06x-release-schedule.md §2 row 67-C1 (`gpu-quick`, # gx10) and §2 row 67-D1 (`cuda-unit`, yoga). # + # Q6 quorum, C324 (DELETE-TO-LAB 3/3): a check that cannot block a merge or a + # release is LAB, and LAB never runs in a PR, merge-queue or release job. These + # three sections no longer run in ci.yml at all. Their one home is + # .github/workflows/gpu-lab-nightly.yml (schedule + workflow_dispatch, gx10 in + # perf-gx10, yoga in perf-yoga). On those events gpu-touched does not select: + # there is no PR diff, so it answers the whole GPU set (gpu_touched=1, + # cuda_lint=1). The paragraphs below are the PR-time history, kept for why + # the shapes are what they are. + # # WHY THEY ARE NOT IN `gate.needs`. # They are ADVISORY in 0.67, deliberately. Each becomes a REQUIRED check # only after FIVE CONSECUTIVE GREEN NIGHTLIES on its own host — rows 68-C3 @@ -3031,7 +3040,8 @@ jobs: # whole point of splitting it out. runs-on: [self-hosted, X64, Linux, clean-room] timeout-minutes: 10 - if: github.event_name == 'pull_request' + # Q6 quorum, C324: LAB, nightly only (gpu-lab-nightly.yml). + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' outputs: gpu_touched: ${{ steps.decide.outputs.gpu_touched }} gpu_crates: ${{ steps.decide.outputs.gpu_crates }} @@ -3046,33 +3056,22 @@ jobs: # governs — a stale selector is caught on the same PR that broke it. - name: The GPU selector can still go RED (case table, no GPU, no cargo) run: bash scripts/ci_gpu_touched.sh --self-test - - name: Does this PR touch the GPU set? (rows 67-C1 / 67-D1) + # Q6 quorum, C324: the PR-diff selection (`ci_gpu_touched.sh --diff-from` + # against GITHUB_BASE_REF) left with the PR rows. A nightly has no PR + # diff and owes the whole GPU set, so the answer is the full set, stated. + - name: The nightly owes the whole GPU set (rows 67-C1 / 67-D1) id: decide run: | set -euo pipefail - # Shallow checkout: no merge-base. Tip-vs-tip is a SUPERSET of the - # PR's diff, which is the safe direction for a selector — it can only - # run the GPU jobs when they were not needed, never skip them when - # they were. - git fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_BASE_REF}:refs/remotes/origin/${GITHUB_BASE_REF}" - git diff --name-only "origin/${GITHUB_BASE_REF}" HEAD > "$RUNNER_TEMP/touched.txt" - printf 'touched (tip vs tip): %s path(s)\n' \ - "$(wc -l < "$RUNNER_TEMP/touched.txt" | tr -d ' ')" - # No `|| true` and no pipe-swallowed status: the script exits 2 when - # the diff is undecidable, and an undecidable selection must fail the - # decision job rather than answer a green 0 that skips both GPU jobs - # forever. - bash scripts/ci_gpu_touched.sh --diff-from "$RUNNER_TEMP/touched.txt" \ - > "$RUNNER_TEMP/gpu.txt" - cat "$RUNNER_TEMP/gpu.txt" - grep -E '^(gpu_touched|gpu_crates|cuda_lint|reason)=' "$RUNNER_TEMP/gpu.txt" >> "$GITHUB_OUTPUT" + printf 'gpu_touched=1\ncuda_lint=1\nreason=%s: no PR diff to select from; the whole GPU set is owed\n' \ + "$GITHUB_EVENT_NAME" | tee -a "$GITHUB_OUTPUT" # ROW 67-C1 — `gpu-quick` on gx10 (GB10, sm_121, aarch64). # Promotion to required: row 68-C3, after 5 consecutive green gx10 nightlies. gpu-quick: needs: [gpu-touched] - if: github.event_name == 'pull_request' && needs.gpu-touched.outputs.gpu_touched == '1' + # Q6 quorum, C324: LAB, nightly only (gpu-lab-nightly.yml, perf-gx10). + if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.gpu-touched.outputs.gpu_touched == '1' # LITERAL, not an expression: a guard can only check labels it can see, and # `runs-on: ${{ fromJSON(...) }}` hides them from check_runner_labels.sh. # `ephemeral, docker` is the DISPOSABLE gx10 runner (operator rule): its @@ -3127,7 +3126,7 @@ jobs: procs=${procs:-0} echo "gx10: GPU memory.used=${used} MiB, existing compute procs=${procs}" if [ "${used:-0}" -gt 2000 ] || [ "${procs:-0}" -gt 0 ]; then - echo "::notice::gpu-quick: GPU busy (${used:-?} MiB, ${procs:-?} procs) — yielding to training, skipping the GPU leg on this PR" + echo "::notice::gpu-quick: GPU busy (${used:-?} MiB, ${procs:-?} procs) — yielding to training, skipping the GPU leg this run" echo "proceed=false" >> "$GITHUB_OUTPUT"; exit 0 fi echo "proceed=true" >> "$GITHUB_OUTPUT" @@ -3165,7 +3164,8 @@ jobs: # gpu_touched=1 runs every step. cuda_lint=1 alone (an apr-cli-only diff, # #4336) runs ONLY the clippy step: the test steps carry their own # gpu_touched `if:`, so yoga is held for a check build, not the test builds. - if: github.event_name == 'pull_request' && (needs.gpu-touched.outputs.gpu_touched == '1' || needs.gpu-touched.outputs.cuda_lint == '1') + # Q6 quorum, C324: LAB, nightly only (gpu-lab-nightly.yml, perf-yoga). + if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (needs.gpu-touched.outputs.gpu_touched == '1' || needs.gpu-touched.outputs.cuda_lint == '1') runs-on: [self-hosted, Linux, X64, cuda, yoga, ephemeral, docker] # 15 -> 35 (#3810): the derived cuda-only set needs a second, default-feature # test build of aprender-serve to list against. MEASURED on this PR's own run diff --git a/contracts/apr-required-checks-v1.yaml b/contracts/apr-required-checks-v1.yaml index 04aebf470a..3c1f88ff86 100644 --- a/contracts/apr-required-checks-v1.yaml +++ b/contracts/apr-required-checks-v1.yaml @@ -111,6 +111,8 @@ scope: > workflow_dispatch only; B1 counts an invocation of scripts/review_receipts_nightly.sh, the nightly walker that runs Arm 4 on every pull request merged in the window. + Q6 quorum, C324: mac-check left ci.yml for nightly.yml (LAB). The key still + computes its mac flag, and the equations below still define it. status: active metadata: @@ -454,7 +456,7 @@ proof_obligations: statement: "gate and ci / gate share one MG-REUSE-VERDICT block that reuses only on x86=det=1 with mg-reuse and x86-main-advisories success on merge_group or on a push to main (#4748), and fails on a split, flags set on any other event or by a failed mg-reuse, or a reused x86-main whose advisory job did not succeed." discharged_by: falsification_tests[10] - id: RC-OB-012 - statement: "ci.yml wires the rule: mg-reuse runs only on merge_group and on a push to main; x86-main, determinism and mac-check need it and skip only on their own flag; x86-main-advisories runs exactly when x86 = 1; gate needs mg-reuse, x86-main-advisories and mg-reuse-self-test." + statement: "ci.yml wires the rule: mg-reuse runs only on merge_group and on a push to main; x86-main and determinism need it and skip only on their own flag; mac-check is LAB and is not a ci.yml job (Q6 quorum, C324), so the mac flag is computed but read by no ci.yml job; x86-main-advisories runs exactly when x86 = 1; gate needs mg-reuse, x86-main-advisories and mg-reuse-self-test." discharged_by: falsification_tests[11] - id: RC-OB-013 statement: "On a push to main, x86=det=1 only when the merge_group run on the pushed single-parent commit S (queue base = S's parent = before) completed, RAN x86-main and determinism (x86-main-advisories skipped) and passed them, gate and ci / gate; mac is never reused on a push; every other input and every failed lookup gives 0." @@ -698,7 +700,7 @@ falsification_tests: that line is an equivalent mutant: the catch-all `*)` still fails.) - id: RC-F-012 - rule: "ci.yml wires the reuse key into the three jobs and both aggregators (T43)" + rule: "ci.yml wires the reuse key into x86-main, determinism and both aggregators (T43; mac-check left ci.yml, Q6 quorum, C324)" prediction: >- Against ci.yml the wiring rows pass; against origin/main's ci.yml before this rule (no mg-reuse job) the same self-test FAILS, 23 rows diff --git a/docs/roadmaps/entries/PMAT-4955.yaml b/docs/roadmaps/entries/PMAT-4955.yaml new file mode 100644 index 0000000000..9fba294372 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4955.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4955 + github_issue: 4955 + item_type: task + title: 'Q6: non-blocking PR-path rows to nightly LAB' + status: planned + priority: high + assigned_to: null + created: 2026-10-09T09:05:55Z + updated: 2026-10-09T09:05:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index d1b846cf90..7f6c90029c 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24571,3 +24571,21 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 3998 +- id: PMAT-4955 + github_issue: 4955 + item_type: task + title: 'Q6: non-blocking PR-path rows to nightly LAB' + status: planned + priority: high + assigned_to: null + created: 2026-10-09T09:05:55Z + updated: 2026-10-09T09:05:55Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3998 diff --git a/scripts/check_determinism_wait.sh b/scripts/check_determinism_wait.sh index dcdf380364..2ca21d1274 100644 --- a/scripts/check_determinism_wait.sh +++ b/scripts/check_determinism_wait.sh @@ -120,7 +120,9 @@ row() { # row # plant : a copy of the real ci.yml with one edit inside the determinism job; an edit that # changes nothing is an error (exit 2), never a row that passes for the wrong reason. plant() { - sed -E "/^ determinism:\$/,/^ mac-check:\$/{$2}" "$CI" > "$1" + # The range ends at the NEXT job key, whatever it is (it named mac-check, which left ci.yml: + # Q6 quorum, C324). sed tests the end only from the line after the start. + sed -E "/^ determinism:\$/,/^ [A-Za-z0-9_-]+:\$/{$2}" "$CI" > "$1" if cmp -s "$1" "$CI"; then printf 'ENV planted edit did not apply: %s\n' "$2"; exit 2; fi } diff --git a/scripts/check_guard_steps_isolated.sh b/scripts/check_guard_steps_isolated.sh index 879851864e..17e268966e 100755 --- a/scripts/check_guard_steps_isolated.sh +++ b/scripts/check_guard_steps_isolated.sh @@ -21,7 +21,7 @@ # default workflows: every .github/workflows/*.y{a,}ml and .github/actions/*/action.y{a,}ml, # plus ci/*.yml and ci/vendor/*.yml (the fat-job sections since #4441) # (every job that calls a guard runs on a self-hosted Linux runner -- review of #4133 -# measured the runs-on of all 13 such workflows; ci.yml's mac-check job is self-hosted +# measured the runs-on of all 13 such workflows; nightly.yml's mac-check job (from ci.yml, Q6/C324) is self-hosted # macOS and calls no guard) # Exit: 0 every invocation isolated · 1 an unisolated invocation (named) · 2 usage/ENV. set -uo pipefail diff --git a/scripts/ci_mg_reuse.sh b/scripts/ci_mg_reuse.sh index cb8478d8c7..82126de34d 100755 --- a/scripts/ci_mg_reuse.sh +++ b/scripts/ci_mg_reuse.sh @@ -21,6 +21,8 @@ # determinism-compare waits on x86-main's X64 artifact and the gate reads # both jobs' section results, so they are reused together, and only when # the head run's `gate` and `ci / gate` succeeded too. mac-check stands alone. +# (Q6 quorum, C324: mac-check left ci.yml for nightly.yml. Its `mac` key is still +# computed, and its rows still pin it, but no ci.yml job reads it.) # Why K1-K3 make the head run's origin/main comparand equal B: H contains B, so # every run on H started after B was on main; B was main's tip when the queue # entry was cut, and main only moves forward. So main's tip was B for the whole @@ -460,8 +462,12 @@ STUB wire x86-main "if: \${{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' }}" wire determinism 'needs: [mg-reuse, x86-main]' # T42: determinism also waits on x86-main wire determinism "if: \${{ !cancelled() && needs.mg-reuse.outputs.det != '1' }}" - wire mac-check 'needs: [mg-reuse]' - wire mac-check "if: \${{ !cancelled() && needs.mg-reuse.outputs.mac != '1' }}" + # Q6 quorum, C324: mac-check is LAB and left ci.yml (nightly.yml now). The `mac` key is + # still computed above; ci.yml must neither run the job nor export the key. + if [ -z "$(job mac-check)" ]; then pass=$((pass + 1)); else + fail=$((fail + 1)); echo "FAIL wiring: ci.yml runs mac-check, a LAB job (Q6 quorum, C324)"; fi + if grep -qF 'outputs.mac' <<<"$(job mg-reuse)"; then + fail=$((fail + 1)); echo "FAIL wiring: mg-reuse still exports mac, which no ci.yml job reads"; else pass=$((pass + 1)); fi wire x86-main-advisories "if: \${{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' }}" wire x86-main-advisories 'cargo deny check advisories' wire x86-main-advisories 'bash scripts/check_deny_exemptions_live.sh' diff --git a/scripts/tests/ci_gpu_touched_test.sh b/scripts/tests/ci_gpu_touched_test.sh index 95e23d00fb..4728da0122 100644 --- a/scripts/tests/ci_gpu_touched_test.sh +++ b/scripts/tests/ci_gpu_touched_test.sh @@ -15,6 +15,9 @@ # 0.67; rows 68-C3 / 68-D2 promote them to required after five # consecutive green nightlies). Every one of those is a property of # the YAML, so it is read from the YAML. +# (c) Q6 quorum, C324 (DELETE-TO-LAB 3/3): the two jobs are LAB. They left +# ci.yml (no PR, merge-queue or release job runs them) and run only in +# .github/workflows/gpu-lab-nightly.yml, on schedule and dispatch. # # bash scripts/tests/ci_gpu_touched_test.sh set -uo pipefail @@ -23,6 +26,7 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$REPO_ROOT" || exit 2 SCRIPT="scripts/ci_gpu_touched.sh" WF=".github/workflows/ci.yml" +NW=".github/workflows/gpu-lab-nightly.yml" # Q6 quorum, C324: the LAB home # #4433: the job BODIES moved verbatim to ci/sections.yml; gx10/yoga/gpu-touched # in ci.yml are the fat jobs that run them and must keep the same decision shape. SEC="ci/sections.yml" @@ -76,6 +80,9 @@ jobq() { # jobq jobq_wf() { # jobq_wf -- a job in the workflow itself jobq_in "$WF" "$@" } +jobq_nw() { # jobq_nw -- a job in the LAB nightly + jobq_in "$NW" "$@" +} jobq_in() { python3 - "$1" "$2" "$3" <<'PY' import sys, yaml @@ -110,8 +117,8 @@ row 0 "the clippy --features cuda step has NO step if: — it runs whenever the "^True$" jobq cuda-unit "any('check_clippy_cuda.sh' in (s.get('run') or '') and not s.get('if') for s in job['steps'])" for j in gpu-quick cuda-unit; do - row 0 "$j runs ONLY on pull_request — never merge_group, never push (the queue stays under 20 min)" \ - "^True$" jobq "$j" "\"github.event_name == 'pull_request'\" in job.get('if','')" + row 0 "$j runs ONLY on schedule / workflow_dispatch — LAB, never pull_request, merge_group or push (Q6, C324)" \ + "^True$" jobq "$j" "\"(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') &&\" in job.get('if','') and not any(e in job.get('if','') for e in ('pull_request', 'merge_group', \"'push'\"))" row 0 "$j is gated on the decision job's gpu_touched (cuda-unit ALSO on cuda_lint, #4336; skipped, not queued, otherwise)" \ "^True$" jobq "$j" "\"gpu_touched\" in job.get('if','') and 'gpu-touched' in job.get('needs',[])" row 0 "$j is NOT continue-on-error — an honest red on the PR is the point" \ @@ -120,18 +127,32 @@ for j in gpu-quick cuda-unit; do "^True$" jobq "$j" "any('ci_self_hosted_preflight.sh' in (s.get('run') or '') for s in job['steps'])" done -for q in jobq jobq_wf; do - row 0 "the decision job itself runs on the clean-room pool, so a non-GPU PR never holds a GPU runner ($q)" \ - "clean-room" $q gpu-touched "job['runs-on']" - row 0 "the decision job publishes gpu_touched as a job output ($q)" \ - "gpu_touched" $q gpu-touched "sorted(job.get('outputs',{}))" -done -# #4433: the fat jobs that run gpu-quick / cuda-unit carry the decision at JOB level, -# so GitHub skips them off the GPU host instead of queueing them on it. -for j in gx10 yoga; do - row 0 "fat job $j runs ONLY on pull_request and only when gpu-touched said 1 (skipped, not queued)" \ - "^True$" jobq_wf "$j" "\"github.event_name == 'pull_request'\" in job.get('if','') and \"gpu_touched == '1'\" in job.get('if','') and 'gpu-touched' in job.get('needs',[])" -done +row 0 "the decision section is documented on the clean-room pool" \ + "clean-room" jobq gpu-touched "job['runs-on']" +row 0 "the decision section publishes gpu_touched as a job output" \ + "gpu_touched" jobq gpu-touched "sorted(job.get('outputs',{}))" +row 0 "the decision section is LAB too: schedule / workflow_dispatch only (Q6, C324)" \ + "^True$" jobq gpu-touched "job.get('if','') == \"github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'\"" +row 0 "the decision section's own case table still runs before it answers" \ + "^True$" jobq gpu-touched "'ci_gpu_touched.sh --self-test' in (job['steps'][1].get('run') or '')" + +# Q6 quorum, C324: no job of ci.yml runs a GPU LAB section, and none is named gx10/yoga/gpu-touched. +row 0 "ci.yml has no gpu-touched / gx10 / yoga job (they left the PR path)" \ + "^NONE$" jobq_in "$WF" gate \ + "'NONE' if not ({'gpu-touched','gx10','yoga','gpu-quick','cuda-unit'} & set(doc['jobs'])) else sorted({'gpu-touched','gx10','yoga','gpu-quick','cuda-unit'} & set(doc['jobs']))" +row 0 "no ci.yml step runs a GPU LAB section through fat_driver" \ + "^NONE$" jobq_in "$WF" gate \ + "'NONE' if not __import__('re').search(r\"--sections '[^']*\\b(gpu-touched|gpu-quick|cuda-unit)\\b\", __import__('json').dumps(doc['jobs'])) else 'FOUND'" + +# The LAB home: schedule + workflow_dispatch only, the fat jobs' runners and perf-* groups. +row 0 "gpu-lab-nightly.yml triggers on schedule and workflow_dispatch ONLY" \ + "^\['schedule', 'workflow_dispatch'\]$" jobq_nw gx10 "sorted(doc[True])" +row 0 "gx10 runs gpu-touched,gpu-quick on the disposable gx10 runner in perf-gx10" \ + "^True$" jobq_nw gx10 \ + "job['runs-on'] == ['self-hosted','Linux','ARM64','cuda','gx10','ephemeral','docker'] and job['concurrency']['group'] == 'perf-gx10' and any(\"--sections 'gpu-touched,gpu-quick'\" in (s.get('run') or '') for s in job['steps'])" +row 0 "yoga runs gpu-touched,cuda-unit on the disposable yoga runner in perf-yoga" \ + "^True$" jobq_nw yoga \ + "job['runs-on'] == ['self-hosted','Linux','X64','cuda','yoga','ephemeral','docker'] and job['concurrency']['group'] == 'perf-yoga' and any(\"--sections 'gpu-touched,cuda-unit'\" in (s.get('run') or '') for s in job['steps'])" # ADVISORY in 0.67. This is the row that has to be DELETED, not edited, when # 68-C3 / 68-D2 promote the jobs — which is the point of pinning it.