diff --git a/.github/workflows/beat-speed-nightly.yml b/.github/workflows/beat-speed-nightly.yml index eb80311551..7da58bbb2d 100644 --- a/.github/workflows/beat-speed-nightly.yml +++ b/.github/workflows/beat-speed-nightly.yml @@ -31,17 +31,13 @@ name: BEAT Speed (nightly) on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:45 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:45 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "45 0 * * *" # 00:45 UTC -> lands ~05:45 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: jobs: @@ -79,6 +75,12 @@ jobs: BEAT_LOG: ${{ github.workspace }}/beat-speed-measurements.log steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # PERF-031 measurement-environment receipt. # diff --git a/.github/workflows/book-contracts.yml b/.github/workflows/book-contracts.yml index b28d4404fe..f0d6c2eb9b 100644 --- a/.github/workflows/book-contracts.yml +++ b/.github/workflows/book-contracts.yml @@ -35,9 +35,14 @@ on: - ".github/workflows/book-contracts.yml" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '45 22 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] # #3676: no concurrency group meant every superseded PR push ran the ~18 min @@ -62,6 +67,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -106,6 +117,12 @@ jobs: needs: chapter-examples-compile steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -157,6 +174,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -189,6 +212,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Every chapter has a contract run: | FAIL=0 @@ -249,6 +278,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: No legacy imports in chapter examples run: | FAIL=0 @@ -290,6 +325,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: No dead links in SUMMARY.md run: | FAIL=0 diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index f695c6ed4e..a27fa9640a 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -51,9 +51,14 @@ on: - "crates/apr-cli/src/tool_commands.rs" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '30 22 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] permissions: contents: read @@ -75,6 +80,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Self-hosted (#3073): tools go into this run's private $RUNNER_TEMP, never a $HOME path that every # runner on the box execs from (the aprender#2353 race; check_cargo_install_private_root.sh enforces diff --git a/.github/workflows/conleche-nightly.yml b/.github/workflows/conleche-nightly.yml index 767e167d96..2e5bdae3c8 100644 --- a/.github/workflows/conleche-nightly.yml +++ b/.github/workflows/conleche-nightly.yml @@ -21,10 +21,13 @@ name: conleche-nightly on: - schedule: - # 21:23 UTC -> lands ~02:23 UTC. GitHub dispatches schedules ~5h late on this account (#3292); the slot is - # free and clear of guards-nightly (21:47) and cuda-nightly (20:30). - - cron: '23 21 * * *' + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: concurrency: @@ -49,6 +52,12 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Verdict case table (the classifier this job trusts) run: bash crates/aprender-contracts-staging/lean/conleche.sh --self-test - name: Pinned elan in the job's own directory (sha256-checked, never a host install) diff --git a/.github/workflows/coverage-nightly.yml b/.github/workflows/coverage-nightly.yml index c5745712b6..83da55bcd9 100644 --- a/.github/workflows/coverage-nightly.yml +++ b/.github/workflows/coverage-nightly.yml @@ -21,17 +21,13 @@ name: Coverage Nightly # measurement ever worked. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 22 * * *' # 22:00 UTC -> lands ~03:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} # #3676: no v* tag trigger here. The release tag's coverage is owned by ci.yml # (`coverage_on: tag`, which sovereign-ci's gate requires on a tag push), so a tag @@ -109,6 +105,12 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Provision cargo-llvm-cov WITHOUT touching ~/.cargo/bin (#2353). # diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index c33e72b56d..0c3e608577 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -44,17 +44,13 @@ name: CUDA Nightly (GPU QLoRA falsifiers) # without deleting the workflow. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 01:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 01:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "30 20 * * *" # 20:30 UTC -> lands ~01:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: silicon: @@ -165,7 +161,7 @@ jobs: # NOT_RUN is not PASS. mkdir -p "$PERF041_OUT" printf '{"host":"%s","status":"NOT_RUN","reason":"yield to training (%s MiB, %s procs)","started_utc":"%s","commit":"%s","exit":null,"max_m":null,"cc":null,"sha256":null}\n' \ - "${{ matrix.name }}" "${used:-?}" "${procs:-?}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${{ github.sha }}" \ + "${{ matrix.name }}" "${used:-?}" "${procs:-?}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${{ github.event.workflow_run.head_sha || github.sha }}" \ > "$PERF041_OUT/marker.json" echo "proceed=false" >> "$GITHUB_OUTPUT"; exit 0 fi @@ -174,11 +170,20 @@ jobs: - uses: actions/checkout@v7 if: steps.decide.outputs.proceed == 'true' - # The first thing gx10 does once it has a tree. NOT before `decide` and - # NOT before the checkout, for the reason the comment below spells out: + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' && (steps.decide.outputs.proceed == 'true') + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + + # The first thing this job does once its tree is C (the step above). NOT before + # `decide`, and NOT before the checkout, for the reason the comment below spells out: # on a yield night there is no working tree, so a step that `bash`es a # repo path exits 127 and turns a deliberately-yielded night RED. It # carries the same `if:` as the checkout it follows. + # It comes AFTER the C step, not between the checkout and it: there it + # ran main's head's copy of the preflight, so it did not judge C's own. # # gx10 is the box where run 34448908554 died on `gh: command not found` # after a 21 MB CUDA build. Everything below this line assumes cargo, @@ -802,6 +807,12 @@ jobs: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: flock -E 175 -w 600 /run/lock/fleet-gpu/gpu.lock steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Put the runner's Rust toolchain on PATH run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 2adaaed942..7bc926acaf 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -9,17 +9,13 @@ name: examples-nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '30 22 * * *' # 22:30 UTC -> lands ~03:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: filter: @@ -41,6 +37,12 @@ jobs: timeout-minutes: 300 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: dogfood_examples.sh case table (hermetic, scratch workspace) run: bash scripts/dogfood_examples.sh --selftest diff --git a/.github/workflows/guards-nightly.yml b/.github/workflows/guards-nightly.yml index 9e045048e1..b97951f71c 100644 --- a/.github/workflows/guards-nightly.yml +++ b/.github/workflows/guards-nightly.yml @@ -37,20 +37,13 @@ name: guards-nightly on: - schedule: - # 02:47Z -- an off-minute, and a slot no other nightly holds: cuda-nightly - # 01:30 (10 min), silicon-nightly 03:30, qwen-story-daily 04:17. A run of - # this job measured ~27 min of steps, so it lands clear of all three. - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 02:47 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 02:47 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '47 21 * * *' # 21:47 UTC -> lands ~02:47 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: # One at a time. Two of these jobs would share $GUARD_CARGO_HOME for the same @@ -99,6 +92,12 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # THE MANIFEST IS THE JOIN KEY, AND IT IS CHECKED BEFORE ANYTHING RUNS. # # A step that leaves ci.yml and never arrives here is a guard that was @@ -177,6 +176,8 @@ jobs: - name: aprender-profile lib tests (was gated by nothing) run: cargo test -p aprender-profile --lib - name: model-tests falsification suites (was gated by nothing, aprender#2522) -- SATD ceiling measured on the comparand (BSE-03) + env: + NIGHTLY_C: ${{ env.NIGHTLY_C }} # the C step wrote it to GITHUB_ENV; empty on workflow_dispatch run: | set -euo pipefail # The warm sccache lives under the runner user's data dir on the persistent intel @@ -218,7 +219,7 @@ jobs: # export-ignore, so an ignored source file would vanish from the # comparand and lower the measured count (review m2-prb-review, 4/5 lanes). comparand="$RUNNER_TEMP/comparand"; git worktree remove --force "$comparand" 2>/dev/null || rm -rf "$comparand" - git worktree add --detach "$comparand" origin/main + git worktree add --detach "$comparand" "${NIGHTLY_C:-origin/main}" trap 'git worktree remove --force "$comparand" 2>/dev/null || true' EXIT DR_ENV=(-e SATD_BASELINE=0) set +e @@ -227,7 +228,7 @@ jobs: n=$(printf '%s\n' "$out" | sed -n 's/.*BROKEN -- \([0-9][0-9]*\) markers in production source.*/\1/p' | head -1) if [ "$mrc" -eq 0 ] && [ -z "$n" ] && printf '%s\n' "$out" | grep -q '^test result: ok'; then n=0; fi case "$n" in ''|*[!0-9]*) printf '%s\n' "$out" | tail -20; echo "FAIL: the comparand measurement did not yield a count (rc=$mrc)"; exit 1 ;; esac - echo "SATD_BASELINE=$n measured on origin/main $(git rev-parse --short origin/main) by the HEAD-built suite" + echo "SATD_BASELINE=$n measured on ${NIGHTLY_C:+nightly C }$(git rev-parse --short "${NIGHTLY_C:-origin/main}") by the HEAD-built suite" # 3. the suites, judged against the measured ceiling DR_ENV=(-e "SATD_BASELINE=$n") dr "$GITHUB_WORKSPACE" cargo test -p aprender-core --features model-tests \ @@ -246,7 +247,7 @@ jobs: printf 'budget (PMAT-1098, row 67-E3). Manifest: `scripts/guards_nightly_manifest.txt`.\n\n' printf '| fact | value |\n|---|---|\n' printf '| ref | `%s` |\n' "${GITHUB_REF_NAME}" - printf '| commit | `%s` |\n' "${GITHUB_SHA}" + printf '| commit | `%s` |\n' "${NIGHTLY_C:-$GITHUB_SHA}" printf '| steps moved here | %s |\n' "$(grep -cvE '^[[:space:]]*(#|$)' scripts/guards_nightly_manifest.txt)" printf '| result | %s |\n' "${JOB_STATUS}" printf '\nA red run here is a defect that reached main: the PR path no longer\n' @@ -254,3 +255,4 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" env: JOB_STATUS: ${{ job.status }} + NIGHTLY_C: ${{ env.NIGHTLY_C }} diff --git a/.github/workflows/install-script.yml b/.github/workflows/install-script.yml index f8daec95c5..1aa5d4bfcb 100644 --- a/.github/workflows/install-script.yml +++ b/.github/workflows/install-script.yml @@ -33,9 +33,14 @@ on: - ".github/workflows/nightly.yml" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '0 23 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] # #3676: a newer push to the same PR cancels the superseded run @@ -52,6 +57,12 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # pmat/bashrs are pinned-fleet tools, never installed per-job (BSE-10a, # PMAT-1066) — see ci.yml's identical step for why. diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index 299a3e0113..caef0af505 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -7,8 +7,16 @@ name: Mutants Nightly # names itself. on: - schedule: - - cron: '0 3 * * *' # 03:00 UTC: the quietest PR hours (03-05Z, 10 of 300 PR CI runs 2026-09-29..10-06), so the PR gpu job on the same runner waits least (infra#1468) + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + # infra#1468 wanted 03-05Z, the quietest PR hours, so the PR gpu job on the + # same runner waits least. The pick's 20:30Z cron is dispatched 4-7h late + # (coverage-nightly.yml), so this run starts at about 00:30-03:30Z. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} permissions: @@ -25,7 +33,7 @@ jobs: concurrency: # PP-19: the gpu runner is a perf host; serialise against every other consumer of it (check_perf_concurrency_groups.sh) group: perf-gx10 cancel-in-progress: false - timeout-minutes: 180 # ends by 06:00Z, before PR traffic rises; a run cut off is red and the next night re-judges from the last success + timeout-minutes: 180 # a 00:30-03:30Z start ends by about 03:30-06:30Z, later when queued behind the perf group; a run cut off is red and the next night re-judges from the last success env: IMAGE: localhost:5000/sovereign-ci:stable MUTANTS_JOBS: ${{ vars.MUTANTS_NIGHTLY_JOBS || '8' }} @@ -42,6 +50,12 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Everything merged since the last nightly that JUDGED main, as a diff id: day # Not a rolling "24 hours ago" window: that has no memory, so one missed or failed run would drop a day of diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 71b75c67a2..abc301a166 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -26,17 +26,13 @@ name: Nightly Bench on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "30 0 * * *" # 00:30 UTC -> lands ~05:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -64,6 +60,12 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # The receipt validator's own case table must discriminate before any # number it blesses means anything. diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml new file mode 100644 index 0000000000..311213c8fa --- /dev/null +++ b/.github/workflows/nightly-pick.yml @@ -0,0 +1,79 @@ +# Nightly pick: ONE job picks the night's candidate C and publishes it as refs/heads/nightly/. +# +# Every producer measures that C, the nightly train prints its line for that C, and a release +# promotes that C. The ref is created once and never moved (a create-only push: its lease expects the ref absent), so a +# merge to main after the pick cannot change the night's line. Contract: contracts/nightly-pick-v1.yaml. +# +# WHY A PICK, NOT CRON SPACING. Scheduled events here fire 1h43m to 5h48m late (14 nights measured), +# so producers that each read main's head at their own start judge different commits in one night. +# A producer chains from this workflow (workflow_run on "Nightly pick", or a dispatch carrying the +# night) and runs `scripts/release/nightly_pick.sh verify --night N --sha `. +# +# The schedule below is a fallback trigger only. A dispatch is the reliable one, and a re-pick is +# idempotent: it prints KEPT and leaves the ref as it is. The night is the UTC date of (created_at - 12h) +# of this run, so a pick that fires late still lands on its own night, and a rerun the next evening +# lands on the night its producers build (they read the same created_at), never on the next one. +# +# The ref is a BRANCH because repository rulesets cover only branches and tags; a ruleset on refs/heads/nightly/* +# (update, deletion and non-fast-forward refused; creation allowed; no bypass) keeps it immutable on the server. It is +# pushed with this workflow's GITHUB_TOKEN, so creating it fires no workflow. The rolling tag refs/tags/nightly is never C. +# +# REPORT-ONLY. Nothing reads refs/heads/nightly/* yet; no job here gates anything. +name: Nightly pick + +on: + schedule: + - cron: '30 20 * * *' + workflow_dispatch: + +permissions: + contents: write + actions: read # this run's own created_at, which is the night (a rerun keeps it) + +concurrency: + group: nightly-pick + cancel-in-progress: false + +jobs: + pick: + if: github.ref == 'refs/heads/main' + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 10 + outputs: + night: ${{ steps.pick.outputs.night }} + c: ${{ steps.pick.outputs.c }} + steps: + - name: Checkout main + uses: actions/checkout@v7 + with: + ref: main + fetch-depth: 0 + + - name: Picker case table + run: bash scripts/release/nightly_pick.sh --self-test + + - name: Pick C for the night + id: pick + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # the clock the producers read (workflow_run.created_at), not now: a rerun after noon UTC the next + # day would otherwise pick the next night while every chained producer builds this one + at="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at)" + night="$(bash scripts/release/nightly_pick.sh night --at "$at")" + rc=0 + bash scripts/release/nightly_pick.sh pick --night "$night" --sha "$GITHUB_SHA" || rc=$? + [ "$rc" -eq 0 ] || { echo "pick exited $rc"; exit "$rc"; } + c="$(bash scripts/release/nightly_pick.sh resolve --night "$night")" + { + echo "night=$night" + echo "c=$c" + } >> "$GITHUB_OUTPUT" + { + echo "### Nightly pick" + echo "" + echo "| night | C | main at pick |" + echo "|---|---|---|" + echo "| $night | \`$c\` | \`$GITHUB_SHA\` |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfabf0079..a27e21a8c1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,17 +36,13 @@ name: Nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 04:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 04:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 23 * * *' # 23:00 UTC -> lands ~04:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -78,6 +74,12 @@ jobs: uses: actions/checkout@v7 with: ref: main + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Case table run: | @@ -121,6 +123,12 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ needs.gate.outputs.sha }} + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # A sibling container through the mounted socket, as in binary-release.yml's # CPU lanes: every bind-mount path must exist on the HOST (the work root is @@ -257,6 +265,19 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + # the night table on this runner's own BSD userland: a GNU-only call there (date -d) fails the C step above + # every night and skips publish-darwin; the case tables name it on any event, not only on a chained night + - name: Night table self-test on macOS (BSD userland) + run: | + set -euo pipefail + bash scripts/release/nightly_pick.sh --self-test + bash scripts/release/nightly_c_checkout.sh --self-test - name: Build apr (host-native, pinned toolchain) run: | @@ -304,6 +325,12 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ needs.gate.outputs.sha }} + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # A runner that died uploaded nothing; merge records that arch red, so # an empty download must not stop the other arch from publishing. diff --git a/.github/workflows/qwen-hunt-nightly.yml b/.github/workflows/qwen-hunt-nightly.yml new file mode 100644 index 0000000000..fc1a52d745 --- /dev/null +++ b/.github/workflows/qwen-hunt-nightly.yml @@ -0,0 +1,189 @@ +# The qwen story's pmat bug-hunt manifest, in its own job (#4875). +# +# The hunt's coverage gaps read coverage-nightly's coverage JSON, and only when +# that file was measured on the commit being hunted (scripts/lib_story_pmat.sh). +# qwen-story-daily starts from the nightly pick, as coverage-nightly does, and +# coverage takes 60-70 min against the story job's 30, so inside the story the +# gaps read not_measured on most nights. This workflow starts when Coverage +# Nightly finishes instead, downloads THAT run's coverage-json artifact, and +# hunts the night's C: the same eight hunts the story's beats name +# (STORY_HUNTS), via scripts/story_pmat_hunt.sh. The story itself runs with +# PMAT_HUNT=0 now; a dispatch can still turn its hunts on. +# +# One hop from the pick, never two: Coverage Nightly chains from "Nightly pick", +# and scripts/release/check_producers_chain.sh (HOPS) holds both links. The C +# step reads the COVERAGE run's created_at and conclusion, so a coverage run +# that did not succeed makes this run not_measured. + +name: qwen-hunt-nightly + +on: + workflow_run: + # The workflow NAME (coverage-nightly.yml `name:`), not the file name. + workflows: ["Coverage Nightly"] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + file_issue: + description: 'File a tracking issue if the hunt fails or the manifest grows' + required: false + default: 'true' + +permissions: + contents: read + issues: write # to open/comment on the tracking issue + actions: read # to download coverage-nightly's coverage-json artifact + +concurrency: + group: qwen-hunt-nightly + cancel-in-progress: false + +jobs: + hunt: + name: Qwen story pmat hunt + # pmat and the checkout only: no apr, GPU or model, so the clean-room + # pool, as guards-nightly, and never a perf host. + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + + - name: pmat must match the pinned fleet versions (never install) + run: | + export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" + setsid --wait bash scripts/check_tool_versions.sh + + # The triggering coverage run's own artifact, matched on the coverage.sha it + # records, never on that run's head_sha: a merge between the pick's start and + # its finish moves head_sha off C while coverage measured C. lib_story_pmat.sh + # uses the file only when coverage.sha is this checkout's HEAD; any other + # file, or none, makes the gaps not_measured and the hunt red. + - name: Fetch the triggering coverage run's coverage JSON + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # workflow_run: the Coverage Nightly run that started this one. + # workflow_dispatch: empty, so look one up for the checked-out commit. + COV_RUN_ID: ${{ github.event.workflow_run.id }} + run: | + set -uo pipefail + dir="$RUNNER_TEMP/coverage-json" + rm -rf "${dir:?}" + run_id="$COV_RUN_ID" + if [ -z "$run_id" ]; then + run_id=$(gh run list -R "$GITHUB_REPOSITORY" -w coverage-nightly.yml --commit "$(git rev-parse HEAD)" \ + --limit 5 --json databaseId,status --jq '[.[] | select(.status == "completed")][0].databaseId // empty') || run_id="" + fi + if [ -z "$run_id" ]; then + echo "::warning::no completed coverage-nightly run for $(git rev-parse HEAD) - pmat hunt coverage gaps will be not_measured" + exit 0 + fi + if ! gh run download "$run_id" -R "$GITHUB_REPOSITORY" -n coverage-json -D "$dir"; then + echo "::warning::coverage-nightly run $run_id has no coverage-json artifact - pmat hunt coverage gaps will be not_measured" + exit 0 + fi + echo "STORY_COVERAGE_FILE=$dir/coverage.json" >> "$GITHUB_ENV" + echo "STORY_COVERAGE_SHA=$(cat "$dir/coverage.sha" 2>/dev/null)" >> "$GITHUB_ENV" + echo "coverage JSON from coverage-nightly run $run_id, measured on $(cat "$dir/coverage.sha" 2>/dev/null || echo '(none recorded)'); HEAD is $(git rev-parse HEAD)" + + # ${PIPESTATUS[0]}: the hunt's status, not tee's (FALSIFY-QWEN-STORY-009). + - name: Run scripts/story_pmat_hunt.sh + id: hunt + continue-on-error: true + shell: bash + run: | + set +e + set -o pipefail + export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" + bash scripts/story_pmat_hunt.sh 2>&1 | tee /tmp/hunt.log + echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + + - name: Extract pmat bug-hunt manifest + id: manifest + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + # Rows are indented; anchor on leading whitespace, as qwen-story-daily does. + mkdir -p /tmp/qwen-hunt-artifacts + grep -E '^[[:space:]]+(gap|churn|fault)[[:space:]]' /tmp/hunt.log \ + > /tmp/qwen-hunt-artifacts/pmat-manifest.txt || true + LINES=$(wc -l < /tmp/qwen-hunt-artifacts/pmat-manifest.txt) + echo "manifest_lines=$LINES" >> "$GITHUB_OUTPUT" + PREV_LINES=$(gh run list --workflow=qwen-hunt-nightly.yml --status=success --limit=1 \ + --json databaseId --jq '.[0].databaseId' 2>/dev/null \ + | xargs -I {} gh run download {} -n pmat-manifest -D /tmp/prev 2>/dev/null \ + && wc -l < /tmp/prev/pmat-manifest.txt 2>/dev/null || echo 0) + GROWTH=$((LINES - PREV_LINES)) + echo "growth=$GROWTH" >> "$GITHUB_OUTPUT" + echo "Previous manifest: $PREV_LINES lines, current: $LINES lines, growth: $GROWTH" + + - name: Upload pmat manifest artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: pmat-manifest + path: /tmp/qwen-hunt-artifacts/pmat-manifest.txt + retention-days: 90 + + - name: Upload hunt log artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: hunt-log + path: /tmp/hunt.log + retention-days: 30 + + # Notifies, never judges (continue-on-error), as in qwen-story-daily: a + # notification failure must not mask or fake the hunt's own result. + - name: File / update tracking issue on failure + continue-on-error: true + if: | + (steps.hunt.outputs.exit_code != '0' || + fromJson(steps.manifest.outputs.growth || '0') > 5) && + (github.event.inputs.file_issue != 'false') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -uo pipefail + for L in qwen-story-daily regression; do + gh label create "$L" --description "Filed by qwen-story-daily" --force >/dev/null 2>&1 || true + done + TITLE="qwen-hunt-nightly: hunt exit=${{ steps.hunt.outputs.exit_code }} manifest_growth=${{ steps.manifest.outputs.growth }}" + EXISTING=$(gh issue list --label qwen-story-daily --state open --limit 1 \ + --json number --jq '.[0].number // empty') + { + echo "## Qwen story pmat hunt — $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "" + echo "- Hunt exit code: \`${{ steps.hunt.outputs.exit_code }}\`" + echo "- Pmat manifest lines: \`${{ steps.manifest.outputs.manifest_lines }}\` (growth: ${{ steps.manifest.outputs.growth }})" + echo "- Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "" + echo "### Hunt log (tail)" + echo '```' + tail -50 /tmp/hunt.log + echo '```' + } > /tmp/issue-body.md + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --body-file /tmp/issue-body.md + else + gh issue create --title "$TITLE" --label qwen-story-daily,regression --body-file /tmp/issue-body.md + fi + + # always(): without a status function GitHub prepends success(), and any + # earlier failure would skip the verdict. + - name: Fail the job if the hunt failed + if: always() && steps.hunt.outputs.exit_code != '0' + run: | + echo "::error::pmat hunt exited ${{ steps.hunt.outputs.exit_code }} - see the hunt-log artifact" + exit 1 diff --git a/.github/workflows/qwen-story-daily.yml b/.github/workflows/qwen-story-daily.yml index 43ac938667..162590e578 100644 --- a/.github/workflows/qwen-story-daily.yml +++ b/.github/workflows/qwen-story-daily.yml @@ -1,4 +1,4 @@ -# Nightly Qwen-story regression run (after coverage-nightly) + pmat bug-hunt manifest. +# Nightly Qwen-story regression run (chained from the nightly pick); its pmat bug-hunt runs in qwen-hunt-nightly.yml. # # Contract: contracts/qwen-story-v1.yaml § FALSIFY-QWEN-STORY-006. # Runs `scripts/qwen-story.sh` against the canonical Qwen model registry, @@ -13,22 +13,23 @@ name: qwen-story-daily on: - # #4715: started by coverage-nightly finishing, not by its own cron. The pmat - # hunt reads coverage gaps only from coverage-nightly's coverage-json artifact - # for the sha it checks out. With two crons (coverage 22:00Z, story 23:17Z) and - # coverage taking >= 60 min, that artifact was rarely there yet. workflow_run - # gives the same sha by construction. No cron alongside it: a second trigger - # is a second run that races the first. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + # The pmat hunt runs in qwen-hunt-nightly.yml (#4875), chained from Coverage + # Nightly so it reads that run's coverage of the same C. Coverage starts from the + # same pick and takes 60-70 min, past this job's 30, so here PMAT_HUNT defaults + # to 0; a dispatch with pmat_hunt=1 still hunts inside the story. workflow_run: - # The workflow NAME (coverage-nightly.yml `name:`), not the file name. - workflows: [Coverage Nightly] + workflows: ["Nightly pick"] types: [completed] + branches: [main] workflow_dispatch: inputs: pmat_hunt: - description: 'Run pmat bug-hunt manifest (1 = yes, 0 = skip for speed)' + description: 'Also run the pmat bug-hunt here (1 = yes; nightly it runs in qwen-hunt-nightly)' required: false - default: '1' + default: '0' file_issue: description: 'File a tracking issue if manifest grows' required: false @@ -72,16 +73,18 @@ jobs: # Default to the runner user's $HOME/models (the models live there, e.g. # /home/noah/models). The old hardcoded /home/runner/models was the retired # runner's path → every beat SKIPped (missing model) even once scheduled. - PMAT_HUNT: ${{ github.event.inputs.pmat_hunt || '1' }} + PMAT_HUNT: ${{ github.event.inputs.pmat_hunt || '0' }} RUST_BACKTRACE: '1' steps: - uses: actions/checkout@v7 with: - # On workflow_run, GITHUB_SHA is main's tip, not the commit coverage - # measured. Check out the measured commit, so the hunt's sha check - # compares like with like. - ref: ${{ github.event.workflow_run.head_sha || github.sha }} fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Install apr from HEAD shell: bash @@ -144,19 +147,16 @@ jobs: shell: bash env: GH_TOKEN: ${{ github.token }} - # workflow_run: the run that triggered this one. workflow_dispatch: - # empty, so look up a completed coverage-nightly run for the sha. - COV_RUN_ID: ${{ github.event.workflow_run.id }} - COV_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} + # Look the coverage run up by the commit this job measures: the night's + # C on workflow_run (the triggering run is the pick, which has no + # coverage artifact), main's sha on workflow_dispatch. + COV_SHA: ${{ env.NIGHTLY_C || github.sha }} run: | set -uo pipefail dir="$RUNNER_TEMP/coverage-json" rm -rf "${dir:?}" - run_id="$COV_RUN_ID" - if [ -z "$run_id" ]; then - run_id=$(gh run list -R "$GITHUB_REPOSITORY" -w coverage-nightly.yml --commit "$COV_SHA" \ - --limit 5 --json databaseId,status --jq '[.[] | select(.status == "completed")][0].databaseId // empty') || run_id="" - fi + run_id=$(gh run list -R "$GITHUB_REPOSITORY" -w coverage-nightly.yml --commit "$COV_SHA" \ + --limit 5 --json databaseId,status --jq '[.[] | select(.status == "completed")][0].databaseId // empty') || run_id="" if [ -z "$run_id" ]; then echo "::warning::no completed coverage-nightly run for $COV_SHA - pmat hunt coverage gaps will be not_measured" exit 0 diff --git a/.github/workflows/silicon-nightly.yml b/.github/workflows/silicon-nightly.yml index ae5ed579b9..9d273d7b19 100644 --- a/.github/workflows/silicon-nightly.yml +++ b/.github/workflows/silicon-nightly.yml @@ -21,17 +21,13 @@ name: Silicon Nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '30 22 * * *' # 22:30 UTC -> lands ~03:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: axis: @@ -56,6 +52,12 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Probe the instrument before believing it. The label matcher must classify # six committed cases — including a runner whose labels are a strict SUBSET @@ -110,6 +112,12 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH @@ -151,6 +159,12 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH diff --git a/.github/workflows/toolchain-ceiling.yml b/.github/workflows/toolchain-ceiling.yml index e400cd22f9..6fdde0353d 100644 --- a/.github/workflows/toolchain-ceiling.yml +++ b/.github/workflows/toolchain-ceiling.yml @@ -23,17 +23,13 @@ name: Toolchain Ceiling # clippy component, broken version comparator) — see its header. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 0 * * *' # 00:00 UTC -> lands ~05:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} permissions: @@ -64,6 +60,12 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Workspace builds on its DECLARED MSRV run: setsid --wait bash scripts/check_msrv.sh @@ -78,6 +80,12 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Clippy on current stable (ceiling gate) run: | @@ -102,6 +110,12 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Self-test — an unreached feature is refused, and a planted cuda-only finding reddens only the cuda-reaching axes run: | export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" diff --git a/Makefile b/Makefile index a6f6fca66b..bc68df2217 100644 --- a/Makefile +++ b/Makefile @@ -1575,7 +1575,7 @@ oracle-owl-check: oracle-owl || { echo "FAIL: tests/oracle/tbox-differential.json differs from a fresh run — commit it"; exit 1; } # ── BLD-002 R4: nightly evidence train (report-only) ──────────────────────────────────────────────────────── -# One line a night for main's head: RELEASABLE H= or NOT RELEASABLE: , . The timer runs a bundle copied +# One line a night for main: RELEASABLE H=, J the head unless the line prints a lag note or NOT RELEASABLE: , . The timer runs a bundle copied # out of git at pinned shas, never the working tree; all three default to HEAD, since they ship in one tree. OUT (and optionally INBOX) come from the command line: # make nightly-train-install OUT= [INBOX=] pin, self-test, install + enable the daily 04:45 UTC user timer # make nightly-train-run run the installed unit once, by hand, and print its line diff --git a/README.md b/README.md index f17a99341a..78a644977b 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1894** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1895** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1894 contracts across inference, training, quantization, attention, FFN, +The tree carries 1895 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/contracts.nt b/contracts/contracts.nt index eb0769e00f..54824ebe55 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -12254,6 +12254,11 @@ "contracts/nightly-greens-v1.yaml"^^ . "nightly-greens-v1"^^ . "pattern"^^ . + . + . + "contracts/nightly-pick-v1.yaml"^^ . + "nightly-pick-v1"^^ . + "pattern"^^ . . . "contracts/nightly-train-v1.yaml"^^ . diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml new file mode 100644 index 0000000000..57d415de55 --- /dev/null +++ b/contracts/nightly-pick-v1.yaml @@ -0,0 +1,101 @@ +metadata: + version: 1.0.0 + created: '2026-10-04' + author: PAIML Engineering + description: "One job picks the night's candidate C once and publishes it as the ref refs/heads/nightly/. Every producer measures that C, the nightly train prints its line for that C, and a release promotes that C. The ref is created once and never moved, so a merge to main after the pick cannot change the night's line. scripts/release/nightly_pick.sh picks, resolves and verifies; .github/workflows/nightly-pick.yml is the one job. Report-only: no producer, train or release reads the ref yet." + kind: pattern + references: + - "scripts/release/nightly_pick.sh: the command line (night, pick, resolve, verify), the case table (--self-test) and the planted mutants (--mutants)" + - "scripts/lib/nightly_pick.sh: the picker, an option-neutral sourced library that fails by return status" + - ".github/workflows/nightly-pick.yml: the one job that picks C, on a schedule or a dispatch from main" + - ".github/workflows/nightly.yml: its gate job already resolves main's head once per run and builds that sha; this generalises that from one workflow to the night" + +five_whys: + symptom: "On 2026-10-02 the scheduled nightly producers judged three different commits of main in one night (scheduled runs of 2026-09-29 to 2026-10-03, read from the runs API), so their results did not describe one candidate." + why_1: "Each scheduled workflow reads main's head at its own start time." + why_2: "Scheduled events on this account fire 1h43m to 5h48m late (14 nights measured), so the start times spread over hours." + why_3: "Merges to main land inside that spread, so early and late producers see different heads." + why_4: "Nothing names the night's candidate, so neither the train nor a release can say which commit the night measured." + why_5: "Cron spacing was used as the ordering, and the lateness makes cron spacing meaningless." + root_cause: "The night had no single picked commit. The remedy is one pick per night, published as an immutable ref, with the night chained from that pick instead of from cron spacing." + +equations: + night: + formula: "night(t) = UTC date of (t − 12 h)" + domain: "t = the pick run's created_at in UTC, which a rerun keeps and every chained producer reads (check_producers_chain P7); a one-hop producer (HOPS) reads its upstream run's created_at, the same night unless that upstream is rerun the next evening, which its same-sha rule reads as not_measured" + codomain: "YYYY-MM-DD" + invariants: + - "a night runs from 12:00Z to 12:00Z, so a pick that fires hours late still lands on the night it was meant for" + + pick: + formula: "pick(n, s) = ref(n) if ref(n) exists; else create ref(n) := s when s is reachable from main, then return ref(n)" + domain: "n a night, s a 40-hex commit id, ref(n) = refs/heads/nightly/ on the remote" + codomain: "C = ref(n), printed as PICKED (this call created it) or KEPT (it existed)" + invariants: + - "ref(n) is created once and never moved: the push carries a lease that expects the ref absent (a plain push is not enough: it fast-forwards an existing branch). It is a branch because repository rulesets cover only branches and tags; a ruleset on refs/heads/nightly/* refusing update, deletion and non-fast-forward (creation allowed, no bypass) holds the same rule on the server, and an existing ref is returned unchanged" + - "a commit not on main is refused and no ref is created; a commit whose ancestry cannot be read is not_measured, never refused as RED and never picked" + - "only the exact branch refs/heads/nightly/ is C: the rolling tag refs/tags/nightly is never read, and a branch named nightly (a directory/file clash) makes the pick not_measured" + - "a lost race returns the winner as KEPT, never PICKED, also when the winner is an ancestor of the loser's commit" + + verify: + formula: "verify(n, s) = OK if s = ref(n); RED if ref(n) exists and s ≠ ref(n); NOT_MEASURED if ref(n) is absent or unreadable" + domain: "n a night, s the commit a producer was handed" + codomain: "exit 0 OK | 1 RED | 2 NOT_MEASURED | 3 caller error" + invariants: + - "a producer handed main's head instead of C is RED" + - "an unreadable remote is NOT_MEASURED, never OK" + - "another remote ref whose name ends in refs/heads/nightly/ (git matches a pattern and a push destination by tail) makes the night ambiguous: NOT_MEASURED, never a C" + +proof_obligations: + - id: NP-INV-001 + type: invariant + property: "A merge to main after the pick does not change the night's line" + formal: "pick(n, s₁) = s₁ ∧ main advances to s₂ ⇒ pick(n, s₂) = s₁ ∧ resolve(n) = s₁ ∧ line(n) unchanged" + applies_to: pick + - id: NP-INV-002 + type: invariant + property: "The night's ref is never moved" + formal: "∀ n: once ref(n) = c, every later pick, including a racing one, leaves ref(n) = c" + applies_to: pick + - id: NP-INV-003 + type: invariant + property: "Only a commit on main can be the night's C" + formal: "s ∉ ancestors(main) ⇒ pick(n, s) = RED ∧ ref(n) absent" + applies_to: pick + - id: NP-INV-004 + type: invariant + property: "A producer that measured another commit is RED" + formal: "ref(n) = c ∧ s ≠ c ⇒ verify(n, s) = RED" + applies_to: verify + - id: NP-INV-005 + type: invariant + property: "No pick or no remote is not_measured, never OK" + formal: "ref(n) absent ∨ remote unreadable ⇒ resolve(n) and verify(n, s) exit 2" + applies_to: verify + - id: NP-INV-006 + type: invariant + property: "A late pick lands on its own night" + formal: "t ∈ [n 12:00Z, n+1 12:00Z) ⇒ night(t) = n" + applies_to: night + +falsification_tests: + - id: FALSIFY-NP-001 + rule: "The picker is right on every row of its case table, including the planted merge after the pick" + prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; a decoy branch whose name ends in refs/heads/nightly/ makes the night ambiguous and not_measured; the rolling tag is never C; a branch named nightly is not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" + test: "bash scripts/release/nightly_pick.sh --self-test" + if_fails: "A merge after the pick can change what the night measured, so producers, the train and a release can disagree on the commit they judged" + + - id: FALSIFY-NP-002 + rule: "The case table can still fail on the defects it names" + prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, a read widened so the rolling tag refs/tags/nightly is seen, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, either one checked line by line so a well-formed first line carries the rest, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + test: "bash scripts/release/nightly_pick.sh --mutants" + if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" + +qa_gate: + id: QA-NP-001 + name: "One immutable C per night, picked from main and published as a ref" + description: "scripts/release/nightly_pick.sh creates refs/heads/nightly/ once from a commit on main, never moves it, and verifies that a producer measured that commit. Report-only until the producers, the train and the release read it." + checks: + - "night" + - "pick" + - "verify" diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index 6c3213ef20..335fbd3d87 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' scheduled producers recorded for C, and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night of the newest Nightly pick run created at or before it, and of the night before for a run created 12:00Z-17:59Z, is J; a rerun keeps its created_at, so an older pick run updated after that one, a pick-run list that was not read, or a full list of 100 pick runs whose oldest is less than 30 days (the rerun limit) before the run, is not_measured), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" @@ -21,17 +21,17 @@ five_whys: equations: lane_state: - formula: "state(lane, C) = not_measured if the lane has no producer, the read failed, no run of the producer on main is for C, the newest such run is unfinished, no matched job ran, or its attempt is unread; red if any matched job (every leg of a repeated name votes) ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 or later; else green" - domain: "one lane of the constant lane table, and the producer runs read for main's head C" + formula: "state(lane, J) = not_measured if the lane has no producer, the read failed, no run of the producer on main is for J, the newest such run is unfinished, no matched job ran, or its attempt is unread; red if any matched job (every leg of a repeated name votes) ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 or later; else green" + domain: "one lane of the constant lane table, and the producer runs read for the judged commit J, the one commit every lane is judged on, at most 6 commits behind main's head C" codomain: "{green, red, not_measured} for each lane" invariants: - - "a run on another commit never stands for C" + - "a run on another commit never stands for J; all lanes are judged on the one J, never a per-lane mix" - "an unfinished run, an unread attempt or a failed read is not_measured, never green" - "a retried success is red: a retried green is not a green" - - "a run whose matched jobs were skipped, cancelled or neutral, or fewer than the lane requires, measured nothing, and the next older run for C is taken; only the matched jobs decide, the run's own conclusion is not read" + - "a run whose matched jobs were skipped, cancelled or neutral, or fewer than the lane requires, measured nothing, and the next older run for J is taken; only the matched jobs decide, the run's own conclusion is not read" verdict_line: - formula: "RELEASABLE H=C iff every verdict lane is green; else NOT RELEASABLE: L, R where L is the first failing verdict lane, red before not_measured, then by hours lost, then table order, and R its run id or not_measured" + formula: "RELEASABLE H=J iff every verdict lane is green; else NOT RELEASABLE: L, R where L is the first failing verdict lane, red before not_measured, then by hours lost, then table order, and R its run id or not_measured" domain: "lane_state over the constant lane table; info lanes are read and reported but never enter the verdict" codomain: "exactly one line on stdout per run" invariants: @@ -43,18 +43,18 @@ equations: proof_obligations: - id: NT-INV-001 type: invariant - property: "RELEASABLE only when every verdict lane is green on C" - formal: "line = RELEASABLE(C) ⇒ ∀ l ∈ verdict: state(l, C) = green" + property: "RELEASABLE only when every verdict lane is green on J" + formal: "line = RELEASABLE(J) ⇒ ∀ l ∈ verdict: state(l, J) = green" applies_to: verdict_line - id: NT-INV-002 type: invariant property: "A lane that cannot be read is never green" - formal: "∀ l: read_failed ∨ no_run(l, C) ∨ unfinished(l, C) ∨ attempt_unread(l, C) ∨ job_list_truncated(l, C) ∨ rollup_truncated(C) ∨ no_producer(l) ⇒ state(l, C) = not_measured" + formal: "∀ l: read_failed ∨ no_run(l, J) ∨ unfinished(l, J) ∨ attempt_unread(l, J) ∨ job_list_truncated(l, J) ∨ rollup_truncated(J) ∨ no_producer(l) ⇒ state(l, J) = not_measured" applies_to: lane_state - id: NT-INV-003 type: invariant property: "A retried success is red" - formal: "∀ l: success(run(l, C)) ∧ attempt(run(l, C)) ≥ 2 ⇒ state(l, C) = red" + formal: "∀ l: success(run(l, J)) ∧ attempt(run(l, J)) ≥ 2 ⇒ state(l, J) = red" applies_to: lane_state - id: NT-INV-004 type: invariant @@ -68,8 +68,8 @@ proof_obligations: applies_to: verdict_line - id: NT-INV-006 type: invariant - property: "A void run is never the evidence; the next older run on C is read instead. Void is decided by the matched jobs only" - formal: "∀ r: (matched_jobs(r) = ∅ ∨ (¬failed(r) ∧ ∃ j ∈ matched_jobs(r): conclusion(j) ∈ {skipped, cancelled, neutral}) ∨ (¬failed(r) ∧ |distinct_jobs(r)| < need(l))) ⇒ pick(l, C) ≠ r" + property: "A void run is never the evidence; the next older run on J is read instead. Void is decided by the matched jobs only" + formal: "∀ r: (matched_jobs(r) = ∅ ∨ (¬failed(r) ∧ ∃ j ∈ matched_jobs(r): conclusion(j) ∈ {skipped, cancelled, neutral}) ∨ (¬failed(r) ∧ |distinct_jobs(r)| < need(l))) ⇒ pick(l, J) ≠ r" applies_to: lane_state - id: NT-INV-007 type: invariant @@ -114,26 +114,26 @@ proof_obligations: - id: NT-INV-015 type: invariant property: "Under --exit-verdict exit 0 means RELEASABLE and every NOT RELEASABLE exits non-zero; a RELEASABLE run still exits non-zero when a later step fails (inbox read-back, NT-INV-013); without the flag the status says only whether the train ran" - formal: "exit_verdict(run) ⇒ (exit(run) = 0 ⇒ line(run) = RELEASABLE(C)) ∧ (line(run) ≠ RELEASABLE(C) ⇒ exit(run) ≠ 0)" + formal: "exit_verdict(run) ⇒ (exit(run) = 0 ⇒ line(run) = RELEASABLE(J)) ∧ (line(run) ≠ RELEASABLE(J) ⇒ exit(run) ≠ 0)" applies_to: verdict_line falsification_tests: - id: FALSIFY-NT-001 rule: "The judge is right on every row of its case table" - prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox" + prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on J; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run, on a run whose night picked another commit or whose pick was not read, and on a run created 12:00Z-17:59Z whose night before picked another commit; a schedule lane ignores the picks; the query reads refs/heads/nightly/ and only dated night names count; the judged commit is the newest of the last 7 on which every verdict producer has a measured run, a newer partial commit never wins, the lag cap counts commits without runs, and a red on J is red; a night with no pick ref is not_measured for a chained lane while a pick read that failed is not_measured for every lane and never the fallback" test: "bash scripts/release/nightly_train.sh --self-test" if_fails: "The train can print RELEASABLE for a commit some release-day check never measured, or stay silent when it fails" - id: FALSIFY-NT-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit) turns at least one row RED, after the unmutated copy is shown green in the same directory" + prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit, a run of any event counted, a chained run counted without its night's pick, any night's pick counted, a night window shifted at either end, the night-before check dropped or shortened or read from the same night, every lane checking the picks, the picks not carried by --from, any ref name read as a night, the picks dropped from the query) turns at least one row RED, after the unmutated copy is shown green in the same directory" test: "bash scripts/release/nightly_train.sh --mutants" if_fails: "The case table is theater: it would pass a judge that calls main releasable on evidence it never read" qa_gate: id: F-NT-001 name: "Main's head is called releasable only from green evidence for every release-day check" - description: "scripts/release/nightly_train.sh reads the night's results for main's head and prints RELEASABLE H= or NOT RELEASABLE: , . Its case table pins every lane state and the verdict set; its mutants prove the table can fail." + description: "scripts/release/nightly_train.sh reads the night's results for one judged commit J of main (the head, or the newest commit up to 6 behind it that every verdict producer ran on) and prints RELEASABLE H= or NOT RELEASABLE: , . Its case table pins every lane state and the verdict set; its mutants prove the table can fail." checks: - "lane_state" - "verdict_line" diff --git a/contracts/qwen-story-v1.yaml b/contracts/qwen-story-v1.yaml index 56db2a6e45..d2635ac085 100644 --- a/contracts/qwen-story-v1.yaml +++ b/contracts/qwen-story-v1.yaml @@ -79,7 +79,7 @@ proof_obligations: - id: QSA60C-INV-003 type: invariant property: "pmat audit runs against the right modules" - formal: "Each beat's pmat_hunt call references the apr command-handler files it just exercised" + formal: "Each beat's line of STORY_HUNTS (scripts/lib_story_pmat.sh), run by story_hunt N, names the apr command-handler files it just exercised" applies_to: pmat_audit_per_beat falsification_tests: @@ -107,8 +107,8 @@ falsification_tests: - id: FALSIFY-QWEN-STORY-004 rule: "PMAT audit is wired per beat" - prediction: "Each beat function calls pmat_hunt at the end" - test: "[ \"$(grep -cE 'pmat_hunt ' scripts/qwen-story.sh)\" -ge '8' ]" + prediction: "Each beatN_*() function ends with story_hunt N, its line of the shared STORY_HUNTS table (#4875), and no story_hunt sits in another function" + test: "[ \"$(awk '/^beat[1-8]_[a-z_]+[(][)] *[{]/ { b = substr($0, 5, 1); if (def[b]++) bad = 1; last = \"\"; next } /^[[:space:]]*[^#[:space:]]/ && /(^|[^[:alnum:]_])(function[[:space:]]+beat[1-8]_|beat[1-8]_[[:alnum:]_]*[[:space:]]*[(][[:space:]]*[)])/ { bad = 1 } /^}/ { if (b != \"\" && last == \" story_hunt \" b) ok[b] = 1; b = \"\"; next } $1 == \"story_hunt\" && ($0 != \" story_hunt \" b || seen[b]++) { bad = 1 } /^[[:space:]]*[^#[:space:]]/ { last = $0 } END { for (i = 1; i <= 8; i++) n += ok[i]; print (bad ? 0 : n) }' scripts/qwen-story.sh)\" = '8' ]" if_fails: "Bug-hunting layer disconnected from the story" - id: FALSIFY-QWEN-STORY-005 diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 8f3c2ebae1..db06005d34 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "22d29f68ea3d5a9c055c0926eef4843419a2a9e79b5e3cd0736eceb9500216cb", + "census_id_set_sha256": "2c164810c4aaf96500c9ea1affec5e75d302f2b83f7040ce9dd3d6b59ec68a05", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "b0f342a30bc3dd54ee19541aa44e179fc345fdb5", + "reasoner_git_sha": "5da18cd6b3e38a0855d670bde6af3278cd21eacc", "checkable_n": 84, "result": { "kind": "model", diff --git a/docs/audits/impl-PMAT-4798-receipt.md b/docs/audits/impl-PMAT-4798-receipt.md new file mode 100644 index 0000000000..e939378a3c --- /dev/null +++ b/docs/audits/impl-PMAT-4798-receipt.md @@ -0,0 +1,45 @@ +# Receipt: PMAT-4798 (aprender#4798) nightly judge: newest commit every verdict lane ran on + +Change: `scripts/release/nightly_train.sh`, the matching amendment of `contracts/nightly-train-v1.yaml` (C becomes J), one +`Makefile` comment, and the roadmap entry `docs/roadmaps/entries/PMAT-4798.yaml`. Authored fresh on the c310 pick-window +proof (#4749, `a7w/4749-c310` @ 067e2e0646). Not touched: `.github/workflows/nightly-train.yml` (its step label still +says "main's head"; a workflow edit needs its own gate-keeping proof). + +## Ticket intent +The nightly judged C = main's head at the read and counted a lane only from a run on C, so any merge after the producers +fired voided the night. Now it judges J = the newest commit, at or before the read, at most 6 commits behind the head, on +which EVERY verdict lane has a measured (green or red) run. None such = judged on the head, where a lane without a run +reads not_measured. There is no other fallback. The line prints `(judged , head , lag k commits)` when J is +not the head. + +## How c310's rule carries to J +- A plain (schedule, push) run stands for the commit it ran on: it counts for J when its head_sha is J. +- A chained (`workflow_run`) run's tree is the pick of its night, never its head_sha, so it stands for that pick only: + it counts for J when `chainc(created_at)` == J: the night of the newest Nightly pick run created at or before it (a + rerun keeps its created_at, so a rerun the next evening is the old night), not_measured when an older pick run was + updated after that one, no pick run was seen, or the list is full (100 runs) and reaches back less than 30 days, the + rerun limit (a rerun of an older run would be unseen); for a run created 12:00Z-17:59Z also `prevc` == J. The J search + applies the same rule when it asks whether a commit has a measured run on every lane. +- Candidates are main's head (rank 0) and its history from the SAME GraphQL query (`history(first: 7)`), written to + `RAW/rank.tsv`; lag is the rank of J (exact). One J for all lanes, never a per-lane mix. A rank list whose rank 0 is + not the head read is ignored. + +## Two unlike failures (a7's condition) +- "No pick for that night": the nightly refs were read and that night has none. The chained lane is not_measured with the + reason "measured the pick no pick for that night" (c310's behaviour); plain lanes are unaffected. Case row + `a_chained_run_without_a_read_pick_is_not_measured`; read-level row `an_absent_pick_is_a_good_read`. +- "Pick read failed": the refs field is not an array (fetch/parse error). `picks_read` writes `pickfail`; `evaluate` turns + the read into `failed: pick read: ...` and EVERY lane is not_measured, with no fallback. Rows + `a_failed_pick_read_is_not_measured_never_a_fallback`, `a_pick_read_with_no_refs_field_failed`. +- Mutants that merge them: `m66_failed_pick_read_falls_back` (evaluate ignores `pickfail`) and + `m67_failed_pick_read_is_an_absent_pick` (the array check is dropped, so a failed read looks like an absent pick). + +## What is measured +- `--self-test`: 89/89 rows. `--mutants`: 69/69 killed (m43-m69; m55-m65 and m68-m69 the J search, m66 and m67 the pick split). +- Merge-queue history on main is not strictly first-parent; those commits carry no main runs and only shorten the + lookback. Conservative. + +## Not claimed +- No replay of 2026-10-05: the saved raw predates the history read, so a replay reads only the head. The first + measurement is the next live read. +- If the verdict is ever wired to the release, require tree(J) == tree(MC). Not done here. diff --git a/docs/audits/quorum-PMAT-4798.json b/docs/audits/quorum-PMAT-4798.json new file mode 100644 index 0000000000..26b22e19b7 --- /dev/null +++ b/docs/audits/quorum-PMAT-4798.json @@ -0,0 +1,283 @@ +{ + "ticket": "PMAT-4798", + "base": "067e2e0646", + "base_resolved": "067e2e0646", + "base_note": "no origin/067e2e0646 exists; judged against the local ref", + "head": "bd003f5592c61c510c633770615ba8e530cc73ee", + "diff_sha256": "47d2e9bf5150d31a95ca25468005a3645e7eafe0aafc0ce5bd4e6a1ca218b9e0", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 62508, + "prompt_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "author": { + "model": "claude-sonnet-5-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff completely and correctly implements the requirements of PMAT-4798. It properly queries the most recent 7 commits from main, establishes `J` as the newest commit (up to `MAX_LAG=6` commits behind the head) on which every verdict lane has a measured run, and defaults back to the head if no such commit exists. The output format `(judged , head , lag k commits)` is accurately appended to the verdict when `J` differs from the head. The tests, mutants, and invariants rigorously verify the edge cases such as partial commits, failed GraphQL reads, chained run pick resolutions, and lag limits. The release path behavior correctly matches the contract, and no constraints are weakened.", + "findings": [], + "raw_bytes": 2698, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "role": "independent", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "executor": "agy", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "1d52976fdfc391af107971499f018f7eda64b3691f937db9c6bbbffa0cb24d8b", + "output_sha256": "5c4e9e3cc7572c4202ede4e934e0332f7f6dec1fc419716715c45f6091690ae5", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements PMAT-4798: J search selects the newest commit (at most 6 behind head) on which every verdict lane has a measured run. Chained runs counted by pick, plain runs by head_sha. Two-unlike-failures split correctly handles absent pick (good read, no pickfail) vs. failed pick read (pickfail written, all lanes not_measured). Lag printed when J ≠ head. GraphQL queries 7 commits via history. Self-test comprehensive (87 rows planned per receipt). Mutants m55–m67 target J logic; prior mutants updated to reference J. No refutations found.", + "findings": [], + "raw_bytes": 2068, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "executor": "claude-code", + "grounding_check": "parity", + "role": "counted", + "model": "claude-haiku-4-5", + "model_measured": "claude-haiku-4-5", + "model_source": "flag", + "family": "claude", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "2df2e42cc426663958e4f76331cef19495720fd8c5e2b8c560ea2c3c31014cc2", + "output_sha256": "2c569d4903394f5f534b78eba9e083c4d1f6d493aa21cd756bf856315c389422", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "claude-haiku-4-5", + "exhausted": false, + "attempts": [ + { + "model": "claude-haiku-4-5", + "family": "claude", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "All changes align with ticket PMAT-4798. No refutations found.", + "findings": [], + "raw_bytes": 1389, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "role": "independent", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "executor": "agy", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "b3c73fc67fa2e6ca379f4f80fa48707cabe10c3cbb49bc9731edd33680d1ab4a", + "output_sha256": "dff13e691b5a04980af31525d3b7ef4c5e2ebb17b1705b624a0206c002166d2e", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "failed to send message: send failed; already reported to the user: Eligibility check failed: failed to get load code assist response: UNAVAILABLE (code 503): The service is currently unavailable." + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: failed to send message: send failed; already reported to the user: Eligibility check failed: failed to get load code assist response: UNAVAILABLE (code 503): The service is currently unavailable.) (PMAT-321)" + ], + "fallback": { + "same_family_width": 2, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "claude-haiku-4-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "claude-sonnet-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "claude-opus-5-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "degraded": null, + "prah": { + "source": "none", + "path": null, + "why": "no --prah, none on PATH, and no install receipt at /file" + }, + "bucket": { + "ledger": "/state", + "window_s": 18000, + "pace": "off", + "buckets": { + "claude-gpt": { + "bucket": "claude-gpt", + "state": "unmeasured", + "reason": "agy-bucket.sh could not read the ledger; the bucket is not consulted this round" + }, + "gemini": { + "bucket": "gemini", + "state": "unmeasured", + "reason": "agy-bucket.sh could not read the ledger; the bucket is not consulted this round" + } + }, + "closed": [] + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "cheap_seat": "claude-haiku-4-5", + "advisory_lane": { + "state": "unavailable", + "counts": false, + "row": { + "NotRun": "Busy" + }, + "verdict": "unavailable", + "why": "no host answered inside the budget", + "served_by": null, + "gpu_proof": null, + "apr": null, + "apr_binary_sha256": null, + "apr_tag": null, + "apr_commit": null, + "apr_build_why": null, + "model": null, + "model_sha256": null, + "rc": 2, + "wall_s": 1, + "collected_s": 1242, + "budget_s": 196, + "budget_basis": "ledger: 2 x p95 98 s over 126 -cuda Verdict rows", + "brief": { + "bytes": 62508, + "sent_bytes": 62677, + "max_bytes": 196608 + }, + "trace": { + "input_sha256": "ef0432863a345be5255db200fd42f06dfd530c6e4f8ad5e8da44155f86645043", + "raw": "advisory.json" + }, + "attempts": [ + { + "host": "", + "ok": false, + "wall_s": 1, + "error": "busy: executor yielded (unit inactive; last yield: 2026-10-03T00:47:22Z host=-a5b5 event=stopped gpu-priority lock=/tmp/apr-gpu.priority)", + "ts": "2026-10-05T22:07:00.887Z", + "wall_ms": 363, + "request_id": "01a10e1b-1f15-75fe-866f-2a0d4a995c20", + "load1": 0.15, + "cold": null + } + ], + "ledger": "/state", + "raw": "advisory.json", + "agrees_with_counted": null, + "counted": "PASS" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-sonnet-5-5/claude same_family=1/2" + } +} diff --git a/docs/roadmaps/entries/PMAT-4798.yaml b/docs/roadmaps/entries/PMAT-4798.yaml new file mode 100644 index 0000000000..1c8fa7f1da --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4798.yaml @@ -0,0 +1,20 @@ +- id: PMAT-4798 + github_issue: 4798 + item_type: task + title: 'nightly_train: judge the newest commit all verdict producers share, not main''s head' + status: inprogress + priority: high + assigned_to: aprender-w4798 + created: 2026-10-05T14:47:26Z + updated: 2026-10-05T14:47:26Z + spec: null + acceptance_criteria: + - 'The nightly judge judges the newest commit, at most 6 behind main''s head, on which every verdict lane has a measured run; none = the head (not_measured); lag printed (judged, head, lag k commits)' + - 'Self-test and mutants pin it; the release path is unchanged' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Landing waits on Q5 shared (operator, via the cop).' + epic: 4678 diff --git a/scripts/check_coverage_has_producers.sh b/scripts/check_coverage_has_producers.sh index daf1f06208..b4b2dbfd78 100644 --- a/scripts/check_coverage_has_producers.sh +++ b/scripts/check_coverage_has_producers.sh @@ -12,8 +12,11 @@ # pre-publish dogfood uses measures coverage itself, reads nothing from CI. # R2 scripts/dogfood.sh still runs `make ... coverage-check` -- the release # CONSUMER exists (without it the chain would be vacuously intact). -# R3 .github/workflows/coverage-nightly.yml triggers on `schedule:` and NOT on -# `v*` tags -- the nightly trend stays, and a tag is measured ONCE, by ci.yml. +# R3 .github/workflows/coverage-nightly.yml has a NIGHTLY trigger and NOT `v*` tags -- +# the nightly trend stays, and a tag is measured ONCE, by ci.yml. Nightly is +# `schedule:`, or (T44, #4686) `workflow_run:` on the "Nightly pick" workflow, +# `types: [completed]` -- the chain that measures the night's C. A workflow_run +# on any other workflow is not nightly. # R4 the Makefile defines a numeric COV_FLOOR -- the floor `make coverage` enforces. # R5 ci/sections.yml (#4433) hands sovereign-ci `coverage_on: tag` (no `skip_coverage: true`) # AND its own `on:` has `push: tags: ['v*']` -- the reusable's two required @@ -36,6 +39,17 @@ on_block() { } # live -> the file with comments stripped (a setting in a comment is not a setting) live() { sed 's/#.*$//' "$1"; } +# nightly_trigger -> 0 when the block has `schedule:`, or a `workflow_run:` whose +# `workflows:` names exactly "Nightly pick" and whose `types:` includes completed (T44). +nightly_trigger() { + grep -qE '^[[:space:]]+schedule:' <<<"$1" && return 0 + awk ' + /^[[:space:]]+workflow_run:/ { match($0, /^ */); wi = RLENGTH; f = 1; next } + f && /[^[:space:]]/ { match($0, /^ */); if (RLENGTH <= wi) f = 0 } + f && /^[[:space:]]+workflows:[[:space:]]*\[[[:space:]]*["\047]?Nightly pick["\047]?[[:space:]]*\]/ { w = 1 } + f && /^[[:space:]]+types:[[:space:]]*\[.*completed/ { t = 1 } + END { exit !(w && t) }' <<<"$1" +} V_TAGS_RE="^[[:space:]]+tags:[[:space:]]*\[[^]]*['\"]?v\*['\"]?" judge() { # judge -> 0 all links hold, 1 a link broke, 2 ENV @@ -67,11 +81,11 @@ judge() { # judge -> 0 all links hold, 1 a link broke, 2 ENV else printf 'FAIL R2 scripts/dogfood.sh no longer runs coverage-check -- nothing checks coverage at the release\n'; bad=1; fi blk=$(on_block "$wf") - if ! grep -qE '^[[:space:]]+schedule:' <<<"$blk"; then - printf 'FAIL R3 coverage-nightly.yml must trigger on schedule (on: block, comments ignored)\n'; bad=1 + if ! nightly_trigger "$blk"; then + printf 'FAIL R3 coverage-nightly.yml must trigger nightly: schedule, or workflow_run on "Nightly pick" completed (on: block, comments ignored)\n'; bad=1 elif grep -qE "$V_TAGS_RE" <<<"$blk"; then printf 'FAIL R3 coverage-nightly.yml also triggers on v* tags -- ci.yml owns tag coverage (R5); a tag would be measured twice\n'; bad=1 - else printf 'ok R3 coverage-nightly.yml triggers on schedule and not on v* tags\n'; fi + else printf 'ok R3 coverage-nightly.yml triggers nightly and not on v* tags\n'; fi if grep -qE '^COV_FLOOR[[:space:]]*:?=[[:space:]]*[0-9]+' "$mk"; then printf 'ok R4 the Makefile defines a numeric COV_FLOOR\n' @@ -114,6 +128,14 @@ if [ "${1:-}" = "--self-test" ]; then m_r2_comment() { printf '#!/usr/bin/env bash\n# gate coverage make -C x coverage-check\n' > scripts/dogfood.sh; } m_r3_nosched() { sed -i '/schedule:/d; /cron:/d' .github/workflows/coverage-nightly.yml; } m_r3_tagsback() { printf "on:\n schedule:\n - cron: '0 22 * * *'\n push:\n tags: ['v*']\njobs: {}\n" > .github/workflows/coverage-nightly.yml; } + # T44 (#4686): the chained form, as the producers carry it, and its near misses. + chain() { printf "on:\n workflow_run:\n workflows: [%s]\n types: [%s]\n branches: [main]\n workflow_dispatch: {}\njobs: {}\n" "$1" "$2" > .github/workflows/coverage-nightly.yml; } + m_r3_chain() { chain '"Nightly pick"' completed; } + m_r3_chain_sq() { chain "'Nightly pick'" completed; } + m_r3_chain_other(){ chain '"Nightly"' completed; } + m_r3_chain_two() { chain '"Nightly pick", "CI"' completed; } + m_r3_chain_req() { chain '"Nightly pick"' requested; } + m_r3_chain_cmt() { m_r3_chain; sed -i 's/^ workflows:/ # workflows:/' .github/workflows/coverage-nightly.yml; } m_r4() { sed -i '/^COV_FLOOR/d' Makefile; } m_r5_skip() { sed -i 's/coverage_on: tag/skip_coverage: true/' ci/sections.yml; } m_r5_both() { printf ' skip_coverage: true\n' >> ci/sections.yml; } @@ -138,6 +160,12 @@ if [ "${1:-}" = "--self-test" ]; then row 1 "R2: the call only in a COMMENT -> RED" m_r2_comment row 1 "R3: nightly schedule removed -> RED" m_r3_nosched row 1 "R3: the nightly's v* tag trigger back (a tag measured twice) -> RED" m_r3_tagsback + row 0 "R3: workflow_run on \"Nightly pick\" completed (T44 chain) -> PASS" m_r3_chain + row 0 "R3: the chain, single-quoted -> PASS" m_r3_chain_sq + row 1 "R3: workflow_run on another workflow -> RED" m_r3_chain_other + row 1 "R3: workflow_run on Nightly pick AND another -> RED" m_r3_chain_two + row 1 "R3: workflow_run on Nightly pick, types [requested] -> RED" m_r3_chain_req + row 1 "R3: the chain's workflows: only in a COMMENT -> RED" m_r3_chain_cmt row 1 "R4: COV_FLOOR removed -> RED" m_r4 row 1 "R5: skip_coverage: true instead of coverage_on (the #3688 red) -> RED" m_r5_skip row 1 "R5: skip_coverage: true alongside coverage_on -> RED" m_r5_both diff --git a/scripts/check_no_pipe_into_grep_q.sh b/scripts/check_no_pipe_into_grep_q.sh index fe39a6ccf8..08d6476f3d 100755 --- a/scripts/check_no_pipe_into_grep_q.sh +++ b/scripts/check_no_pipe_into_grep_q.sh @@ -38,7 +38,7 @@ SELF="scripts/check_no_pipe_into_grep_q.sh" # it is NOT fed by a pipe, so `cmd || grep -q PAT file` has no producer to kill # and no SIGPIPE to misread. The shipped regex matched the SECOND bar of `||` # and flagged 2 safe sites in this tree (dogfood.sh:1477 greps a FILE, -# qwen-story.sh:281 greps a HERE-STRING) plus every new one written since. The +# qwen-story.sh:275 greps a HERE-STRING) plus every new one written since. The # case table below now has a row for it in both directions, because a regex # without a discriminating row is how all five previous versions of this pattern # shipped wrong. diff --git a/scripts/check_silicon_coverage.sh b/scripts/check_silicon_coverage.sh index e251eb03a5..51778faa05 100755 --- a/scripts/check_silicon_coverage.sh +++ b/scripts/check_silicon_coverage.sh @@ -376,7 +376,9 @@ build_jobs() { _bj_since="$(epoch_day "$_bj_cut")" || return 1 _bj_runs="$(mktemp)" || return 1 for _bj_wf in "$@"; do - for _bj_ev in schedule workflow_dispatch; do + # workflow_run: the nightlies chain from "Nightly pick" (T44, #4686); + # schedule stays for the runs from before the chain, inside the lookback. + for _bj_ev in schedule workflow_run workflow_dispatch; do _bj_one="$(mktemp)" || return 1 list_window "$_bj_wf" "$_bj_ev" "$_bj_one" || { rm -f "${_bj_one:?}" "${_bj_runs:?}"; return 1; } # CROSS-CHECK EVERY LISTING (PMAT-3337 §8, widened). The API can hand @@ -599,7 +601,7 @@ st_page() { st_finish() { _sf_since="$(epoch_day $(( ST_NOW - 30 * 86400 )))" _sf_since_e="$(date -u -d "$_sf_since" +%s)" # bashrs disable-line=DET002 - for _sf_ev in schedule workflow_dispatch; do + for _sf_ev in schedule workflow_run workflow_dispatch; do _sf_key="repos/$ST_REPO/actions/runs?event=${_sf_ev}&per_page=100" if [ "$ST_ERROR" = 1 ]; then st_route "$_sf_key" @ERROR; continue; fi awk -F'\t' -v ev="$_sf_ev" '$3 == ev && $6 == 1' "$ST/u/runs.tsv" \ @@ -608,7 +610,7 @@ st_finish() { st_route "$_sf_key" "pages/global-$_sf_ev.json" done for _sf_wf in sil.yml gpu.yml noise.yml silicon-nightly.yml; do - for _sf_ev in schedule workflow_dispatch; do + for _sf_ev in schedule workflow_run workflow_dispatch; do awk -F'\t' -v wf="$_sf_wf" -v ev="$_sf_ev" -v s="$_sf_since_e" \ '$2 == wf && $3 == ev && $5 >= s' "$ST/u/runs.tsv" \ | sort -t"$TAB" -k5,5nr > "$ST/u/l.tsv" @@ -708,11 +710,11 @@ st_real() { # st_real_nightlies : one Silicon Nightly a day, newest # first. The newest is run 36364433328 with its four jobs as the API returned # them (2026-09-28): names, conclusions and runs-on labels verbatim. -st_real_nightlies() { - _rn_h="$1"; _rn_i=0 +st_real_nightlies() { # args: hours before now, count, event (default schedule) + _rn_h="$1"; _rn_i=0; _rn_ev="${3:-schedule}" while [ "$_rn_i" -lt "$2" ]; do _rn_id=$(( 36364433328 - _rn_i )) - st_run "$_rn_id" silicon-nightly.yml schedule $(( _rn_h + 24 * _rn_i )) 1 \ + st_run "$_rn_id" silicon-nightly.yml "$_rn_ev" $(( _rn_h + 24 * _rn_i )) 1 \ "x86_64-cpu|success|self-hosted,clean-room,intel" \ "coverage|success|self-hosted,clean-room,intel" \ "aarch64-cuda-sm121|success|self-hosted,gpu,gx10,cuda,blackwell" \ @@ -818,6 +820,31 @@ selftest_rows() { st_check j-short-always 2 '^ NO-GO +x86_64-cpu +REQUIRED: inconsistent listing .*listed 4 of 4.*total 13' \ || _rows_bad=$((_rows_bad + 1)) + # (k) T44 (#4686): the producers run on workflow_run from "Nightly pick", no + # longer on schedule. Runs of the chain are evidence like scheduled runs; a + # guard that lists only schedule / workflow_dispatch reads every chained + # night as no run at all. + st_reset + st_run 9001 sil.yml workflow_run 12 1 "$CPU_JOB"; st_run 9101 gpu.yml workflow_run 20 1 "$GPU_JOB" + st_finish + _rows=$((_rows + 1)) + st_check k-chained 0 '^ ok +gpu-axis ' || _rows_bad=$((_rows_bad + 1)) + + # (l) the switch-over night: the last scheduled run is stale (8 days), the + # chained run is fresh. The newest run of EITHER event is the evidence. + st_reset + st_run 9001 sil.yml schedule 192 1 "$CPU_JOB"; st_run 9002 sil.yml workflow_run 12 1 "$CPU_JOB" + st_run 9101 gpu.yml schedule 200 1 "$GPU_JOB"; st_run 9102 gpu.yml workflow_run 20 1 "$GPU_JOB" + st_finish + _rows=$((_rows + 1)) + st_check l-switch-over 0 '^ ok +gpu-axis ' || _rows_bad=$((_rows_bad + 1)) + + # (m) the real silicon-nightly jobs, chained: both real axes GREEN. + st_reset; st_real || return 2; st_real_nightlies 10 13 workflow_run; st_finish + _rows=$((_rows + 1)) + st_check m-real-chained 0 '^ ok +aarch64-cuda-sm121 +success .*Silicon Nightly / aarch64-cuda-sm121' \ + || _rows_bad=$((_rows_bad + 1)) + # (e) the API errors: refuse, exactly as before. st_reset; ST_ERROR=1 st_run 9001 sil.yml schedule 12 1 "$CPU_JOB"; st_run 9101 gpu.yml schedule 20 1 "$GPU_JOB" diff --git a/scripts/check_story_pmat_hunt.sh b/scripts/check_story_pmat_hunt.sh index 424944e43b..2b701a9e61 100755 --- a/scripts/check_story_pmat_hunt.sh +++ b/scripts/check_story_pmat_hunt.sh @@ -31,6 +31,15 @@ cd "$(dirname "$0")/.." || exit 1 LIB="scripts/lib_story_pmat.sh" STORY="scripts/qwen-story.sh" +# The awk FALSIFY-QWEN-STORY-004 runs: prints how many of beats 1-8 end with their +# own story_hunt N, or 0 when any story_hunt call is elsewhere, re-indented or repeated, +# or when a beatN_ function is defined twice or in another form (bash runs the last definition). +STORY_HUNT_BINDING='/^beat[1-8]_[a-z_]+[(][)] *[{]/ { b = substr($0, 5, 1); if (def[b]++) bad = 1; last = ""; next } +/^[[:space:]]*[^#[:space:]]/ && /(^|[^[:alnum:]_])(function[[:space:]]+beat[1-8]_|beat[1-8]_[[:alnum:]_]*[[:space:]]*[(][[:space:]]*[)])/ { bad = 1 } +/^}/ { if (b != "" && last == " story_hunt " b) ok[b] = 1; b = ""; next } +$1 == "story_hunt" && ($0 != " story_hunt " b || seen[b]++) { bad = 1 } +/^[[:space:]]*[^#[:space:]]/ { last = $0 } +END { for (i = 1; i <= 8; i++) n += ok[i]; print (bad ? 0 : n) }' [ -f "$LIB" ] || { echo "check_story_pmat_hunt: missing $LIB"; exit 1; } fails=0 @@ -256,23 +265,63 @@ fi # -- 8. Every path the story hunts still exists ----------------------------- # Cause 3. Static, because a path can rot without anyone running the nightly. -if [ -f "$STORY" ]; then - missing="" - checked=0 - while read -r p; do - checked=$((checked + 1)) - [ -e "$p" ] || missing="$missing $p" - done < <(grep -oE '(crates|src)/[A-Za-z0-9_./-]+' "$STORY" \ - | grep -E '\.rs$|/serve$' | sort -u) - want "every source path hunted by qwen-story.sh exists" "" "$missing" - # Fail closed: a discovery step that found nothing must not report success. - if [ "$checked" -ge 15 ]; then - ok "path discovery found $checked hunted paths" - else - bad "path discovery found enough hunted paths" ">= 15" "$checked" - fi +# The paths live in one table, STORY_HUNTS (lib_story_pmat.sh, #4875), read by +# both the story's beats and the hunt-only nightly. +missing="" +checked=0 +while read -r p; do + checked=$((checked + 1)) + [ -e "$p" ] || missing="$missing $p" +done < <(sed -e 's/^[^|]*|//' <<< "$STORY_HUNTS" | tr ' ' '\n' | grep -v '^$' | sort -u) +want "every source path hunted by the story exists" "" "$missing" +# Fail closed: a discovery step that found nothing must not report success. +if [ "$checked" -ge 15 ]; then + ok "path discovery found $checked hunted paths" +else + bad "path discovery found enough hunted paths" ">= 15" "$checked" fi +# -- 10. One table, two readers (#4875) -------------------------------------- +# The story runs line N at the end of beat N; story_pmat_hunt.sh runs all eight. +# A beat that stops calling its line, or a line no beat calls, is a hunt the +# nightly no longer runs where the story said it would. +want "STORY_HUNTS has one line per beat" "8" "$(grep -c '|' <<< "$STORY_HUNTS")" +want "qwen-story.sh calls story_hunt 1..8 once each, in beat order" "1 2 3 4 5 6 7 8" \ + "$(sed -n -e 's/^ story_hunt \([0-9][0-9]*\)$/\1/p' "$STORY" | tr '\n' ' ' | sed -e 's/ $//')" +# File order is not enough: a hunt moved into the previous beat keeps the order. +# Each beatN_*() must END with story_hunt N (its last live line before the closing +# brace), and no story_hunt may sit in another function. FALSIFY-QWEN-STORY-004 +# runs the same awk. +want "each beatN_*() ends with story_hunt N, and no hunt sits elsewhere" "8" \ + "$(awk "$STORY_HUNT_BINDING" "$STORY")" +# Bash runs the LAST definition of a name, so a second beatN_ function, in any +# form and anywhere in the file, is a beat whose hunt may never run: each must read 0. +for c in 'beat3_explore() {| :|}' 'function beat3_explore {| :|}' 'beat3_explore () {| :|}' 'beat3_other() {| :|}' ' beat3_explore() { :; }'; do + { printf '%s\n' "$c" | tr '|' '\n'; cat "$STORY"; } > "$TMP/story-dup" + want "a second beat3 definition ($c) reads 0" "0" "$(awk "$STORY_HUNT_BINDING" "$TMP/story-dup")" +done +want "qwen-story.sh names no hunt path itself" "" "$(grep -n -E '^[[:space:]]*pmat_hunt ' "$STORY")" +: > "$FAILLOG" +story_hunt 9 >/dev/null 2>&1; rc=$? +want "a line STORY_HUNTS lacks returns 1" "1" "$rc" +want "a line STORY_HUNTS lacks tallies a failure" "pmat-hunt beat 9 :: STORY_HUNTS has no line 9" "$(cat "$FAILLOG")" +: > "$FAILLOG" +out=$(STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" PMAT_HUNT=1 story_hunt 2); rc=$? +want "story_hunt 2 hunts beat 2's label" "1" "$(grep -c -F -e '-- pmat bug-hunt manifest (qa validate lint) --' <<< "$out")" +want "story_hunt 2 returns 0 with rows" "0" "$rc" + +# The hunt-only nightly, end to end against the stub: eight headers and rc 0 on +# a coverage file of HEAD; rc 2 (never 0) when that file is absent; rc 1 when +# pmat is missing, because the library's silent skip would otherwise pass. +HUNT="scripts/story_pmat_hunt.sh" +out=$(STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" PMAT_HUNT=0 bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: coverage of HEAD returns 0 (PMAT_HUNT=0 in its env is overridden)" "0" "$rc" +want "story_pmat_hunt.sh: runs all eight hunts" "8" "$(grep -c -e '-- pmat bug-hunt manifest (' <<< "$out")" +out=$(env -u STORY_COVERAGE_FILE -u STORY_COVERAGE_SHA bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: no coverage file returns 2" "2" "$rc" +out=$(PMAT_BIN="$TMP/bin/no-such-pmat" STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: no pmat returns 1" "1" "$rc" + echo if [ "$fails" -eq 0 ]; then echo "check_story_pmat_hunt: OK - the manifest produces rows, and an empty one fails the beat" diff --git a/scripts/check_workflow_path_filters.sh b/scripts/check_workflow_path_filters.sh index 7b47f2fd6a..dbd38ae61b 100644 --- a/scripts/check_workflow_path_filters.sh +++ b/scripts/check_workflow_path_filters.sh @@ -36,7 +36,7 @@ # five wrong guard regexes in this repo were caught by tables, none by review. # # bash scripts/check_workflow_path_filters.sh # check -# bash scripts/check_workflow_path_filters.sh --self-test # 4-case table +# bash scripts/check_workflow_path_filters.sh --self-test # 9-row table set -uo pipefail @@ -44,6 +44,7 @@ SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0] REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WF_DIR="${REPO_ROOT}/.github/workflows" FILTER_DUMP="${REPO_ROOT}/scripts/lib/workflow_path_filters.py" +NIGHTLY_AWK="${REPO_ROOT}/scripts/lib/workflow_runs_nightly.awk" # Workflows that RUN code from a crate must watch that crate's source. # Format: | @@ -69,22 +70,23 @@ check_workflow() { rm -f /tmp/wfpf_err.$$ [ -z "$out" ] && return 0 - local push pr sched + local push pr push="$(printf '%s\n' "$out" | awk -F'\t' '$1=="PUSH"{print $2}' | sort -u)" pr="$(printf '%s\n' "$out" | awk -F'\t' '$1=="PR"{print $2}' | sort -u)" - sched="$(printf '%s\n' "$out" | awk -F'\t' '$1=="SCHEDULE"{print $2}')" - # Rule 3 (#3676): a path-FILTERED workflow must also run on `schedule:`. The + # Rule 3 (#3676): a path-FILTERED workflow must also run nightly: a `schedule:` + # cron, or the chain from "Nightly pick" the nightly producers start on (T44, + # #4686; scripts/lib/workflow_runs_nightly.awk). The # filter is a claim that nothing outside its paths can break the gate; the # nightly run is what turns a wrong claim into a red within a day instead of # the three months book.yml sat dark. Applies when either event is filtered. local filtered=0 if [ -n "$push" ] && [ "$push" != '' ]; then filtered=1; fi if [ -n "$pr" ] && [ "$pr" != '' ]; then filtered=1; fi - if [ "$filtered" = 1 ] && [ "$sched" != yes ]; then - printf '\nFAIL %s: path-filtered, but no `schedule:` trigger.\n' "$name" + if [ "$filtered" = 1 ] && ! awk -f "$NIGHTLY_AWK" "$wf"; then + printf '\nFAIL %s: path-filtered, but no nightly trigger (a `schedule:` cron, or workflow_run on "Nightly pick").\n' "$name" printf ' A change outside the filter that breaks this gate is never seen;\n' - printf ' add a nightly cron so the claim is re-checked every day.\n' + printf ' add a nightly cron, or chain from "Nightly pick", so the claim is re-checked every day.\n' return 1 fi @@ -188,8 +190,25 @@ on: paths: ["book/**", "scripts/book-gate.sh"] YML + # 7 - symmetric, chained from "Nightly pick" instead of a cron (T44, #4686). MUST pass. + cat > "$TD/wf7.yml" <<'YML' +on: + push: + paths: ["book/**"] + pull_request: + paths: ["book/**"] + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] +YML + # 8 - chained from some other workflow: not the nightly (rule 3). MUST fail. + sed 's/"Nightly pick"/"CI"/' "$TD/wf7.yml" > "$TD/wf8.yml" + # 9 - the chain commented out is no trigger (rule 3). MUST fail. + sed 's/^ workflow_run:/ # workflow_run:/; s/^ workflows:/ # workflows:/' "$TD/wf7.yml" > "$TD/wf9.yml" + fails=0 - for c in 1 4 5 6; do + for c in 1 4 5 6 8 9; do if check_workflow "$TD/wf${c}.yml" >/dev/null 2>&1; then printf 'FAIL row %s NOT flagged - the guard is blind to a real defect shape\n' "$c" fails=$((fails + 1)) @@ -197,7 +216,7 @@ YML printf 'ok row %s flagged (must turn RED)\n' "$c" fi done - for c in 2 3; do + for c in 2 3 7; do if check_workflow "$TD/wf${c}.yml" >/dev/null 2>&1; then printf 'ok row %s clean (must stay GREEN)\n' "$c" else @@ -207,9 +226,9 @@ YML done if [ "$fails" -ne 0 ]; then - printf '\nSELF-TEST FAILED (%s/6 wrong)\n' "$fails"; exit 1 + printf '\nSELF-TEST FAILED (%s/9 wrong)\n' "$fails"; exit 1 fi - printf '\nSELF-TEST PASSED (6/6)\n' + printf '\nSELF-TEST PASSED (9/9)\n' exit 0 fi diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh new file mode 100644 index 0000000000..aa7f477b23 --- /dev/null +++ b/scripts/lib/nightly_pick.sh @@ -0,0 +1,133 @@ +# shellcheck shell=bash +# nightly_pick.sh (lib) — one candidate C per night, published as an immutable ref (see scripts/release/nightly_pick.sh). +# +# SOURCED, so option-neutral: no shell options are changed here, nothing exits; every function fails by return status. +# . scripts/lib/nightly_pick.sh || exit 1 +# +# The night's ref is refs/heads/nightly/, NIGHT = the UTC date of (pick time - 12 h): a night runs 12:00Z to 12:00Z, +# so a scheduled pick that fires hours late still lands on the night it was meant for. The ref is created once and +# never moved: a second pick the same night keeps the first C, whatever main is by then. Every producer, the train +# and the release read C from that ref, never from main's head. +# It is a BRANCH because repository rulesets cover only branches and tags: a ruleset on refs/heads/nightly/* (update, +# deletion and non-fast-forward refused, creation allowed, no bypass) makes it immutable on the server as well. The +# pick pushes with the workflow's GITHUB_TOKEN, so creating the branch fires no workflow. Only that exact name is C: +# the rolling tag refs/tags/nightly, or any other ref whose name merely ends in the night's name, is never read as C. +# +# Return codes: 0 ok; 1 refused (RED); 2 not_measured (could not read or write the remote); 3 caller error. +# +# np_night [EPOCH] print the night of EPOCH (default now) +# np_date DAYS print the UTC date DAYS after 1970-01-01 (no date -d: portable to BSD date) +# np_days YYYY-MM-DD print its days after 1970-01-01; 3 for a day the calendar lacks +# np_epoch YYYY-MM-DDTHH:MM:SSZ print its epoch; 3 for any other form. No date -d anywhere: the darwin runner has BSD date +# np_pick REPO REMOTE NIGHT SHA create refs/heads/nightly/NIGHT -> SHA unless it exists; prints "C=" + PICKED|KEPT +# np_resolve REPO REMOTE NIGHT print the night's C; 2 when the night has no pick +# np_verify REPO REMOTE NIGHT SHA +# 0 when SHA is the night's C, 1 when it is not, 2 when the night has no pick + +np_night() { # [EPOCH] + local e="${1:-}" s + [ -n "$e" ] || e="$(date -u +%s)" || return 3 + case "$e" in '' | *[!0-9]*) printf 'NP caller error: epoch %s is not a number\n' "$e" >&2; return 3 ;; esac + s=$((e - 43200)) + np_date $((s >= 0 ? s / 86400 : -((86399 - s) / 86400))) +} + +np_date() { # DAYS since 1970-01-01 -> YYYY-MM-DD, in integer arithmetic: date -d @N is GNU-only and the darwin + # runner's BSD date refuses it (H. Hinnant's civil_from_days, proleptic Gregorian) + local z era doe yoe doy mp m + z=$(($1 + 719468)) + era=$(((z >= 0 ? z : z - 146096) / 146097)); doe=$((z - era * 146097)) + yoe=$(((doe - doe / 1460 + doe / 36524 - doe / 146096) / 365)) + doy=$((doe - (365 * yoe + yoe / 4 - yoe / 100))); mp=$(((5 * doy + 2) / 153)) + m=$((mp < 10 ? mp + 3 : mp - 9)) + printf '%04d-%02d-%02d\n' $((yoe + era * 400 + (m <= 2))) "$m" $((doy - (153 * mp + 2) / 5 + 1)) +} + +np_days() { # YYYY-MM-DD -> days since 1970-01-01 (H. Hinnant's days_from_civil); 3 for a date the calendar lacks + local y m d days back + [[ $1 =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})$ ]] || return 3 + # a leading 1 keeps "08" decimal: 108 - 100 + y=$((1${BASH_REMATCH[1]} - 10000)); m=$((1${BASH_REMATCH[2]} - 100)); d=$((1${BASH_REMATCH[3]} - 100)) + [ "$m" -gt 2 ] || y=$((y - 1)) + days=$(((y / 400) * 146097 + (y % 400) * 365 + (y % 400) / 4 - (y % 400) / 100 + (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1 - 719468)) + # back through np_date: 2026-02-30 comes back 2026-03-02, so it is refused + back="$(np_date "$days")" + [ "$back" = "$1" ] || return 3 + printf '%s\n' "$days" +} + +np_epoch() { # YYYY-MM-DDTHH:MM:SSZ (the form workflow_run.created_at takes) -> epoch; 3 for any other form + local ymd hh mm ss days + [[ $1 =~ ^([0-9]{4}-[0-9]{2}-[0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$ ]] || return 3 + ymd="${BASH_REMATCH[1]}"; hh=$((1${BASH_REMATCH[2]} - 100)); mm=$((1${BASH_REMATCH[3]} - 100)); ss=$((1${BASH_REMATCH[4]} - 100)) + (( hh < 24 && mm < 60 && ss < 60 )) || return 3 + days="$(np_days "$ymd")" || return 3 + printf '%s\n' $((days * 86400 + hh * 3600 + mm * 60 + ss)) +} + +np__args() { # NIGHT [SHA] + # [[ =~ ]] anchors the whole string; `printf | grep -x` matched per line, so "2026-10-09x" passed + [[ $1 =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || { printf 'NP caller error: night %s is not YYYY-MM-DD\n' "$1" >&2; return 3; } + [ "$#" -lt 2 ] || [[ $2 =~ ^[0-9a-f]{40}$ ]] || { printf 'NP caller error: sha %s is not 40 hex\n' "$2" >&2; return 3; } +} + +np__read() { # REPO REMOTE NIGHT -> the ref's sha on stdout; 0 found, 1 absent, 2 unreadable + local out rc=0 + out="$(git -C "$1" ls-remote --refs "$2" "refs/heads/nightly/$3" 2>/dev/null)" || rc=$? + [ "$rc" -eq 0 ] || return 2 + # exact name only. ls-remote matches a pattern by its tail, and so does a push destination, so another ref ending in + # refs/heads/nightly/N (a branch refs/heads/refs/heads/nightly/N) would be read as C, and would swallow the pick's push. + # Such a name makes the night ambiguous: unreadable (2), never a C. + if printf '%s\n' "$out" | awk -v r="refs/heads/nightly/$3" 'NF && $2 != r { found = 1 } END { exit !found }'; then + printf 'NP: refs/heads/nightly/%s is ambiguous on the remote (another ref ends in that name)\n' "$3" >&2; return 2 + fi + out="$(printf '%s\n' "$out" | awk -v r="refs/heads/nightly/$3" '$2 == r { print $1; exit }')" + [ -n "$out" ] || return 1 + printf '%s\n' "$out" +} + +np_resolve() { # REPO REMOTE NIGHT + local c rc=0 + np__args "$3" || return 3 + c="$(np__read "$1" "$2" "$3")" || rc=$? + case "$rc" in + 0) printf '%s\n' "$c" ;; + 1) printf 'NP NOT_MEASURED: night %s has no pick\n' "$3" >&2; return 2 ;; + *) printf 'NP NOT_MEASURED: cannot read refs/heads/nightly/%s\n' "$3" >&2; return 2 ;; + esac +} + +np_verify() { # REPO REMOTE NIGHT SHA + local c rc=0 + np__args "$3" "$4" || return 3 + c="$(np_resolve "$1" "$2" "$3")" || rc=$? + [ "$rc" -eq 0 ] || return "$rc" + if [ "$c" = "$4" ]; then printf 'NP OK: %s is the C of night %s\n' "${4:0:10}" "$3"; return 0; fi + printf 'NP RED: %s is not the C of night %s (C=%s) -- measure C, not main\n' "${4:0:10}" "$3" "${c:0:10}" + return 1 +} + +np_pick() { # REPO REMOTE NIGHT SHA + local c rc=0 + np__args "$3" "$4" || return 3 + c="$(np__read "$1" "$2" "$3")" || rc=$? + if [ "$rc" -eq 0 ]; then + printf 'C=%s KEPT night %s (picked earlier; main now %s does not move it)\n' "$c" "$3" "${4:0:10}"; return 0 + fi + [ "$rc" -eq 1 ] || { printf 'NP NOT_MEASURED: cannot read refs/heads/nightly/%s; nothing picked\n' "$3"; return 2; } + # C must be on main: a dispatch from a branch must never become the night + git -C "$1" fetch -q "$2" main 2>/dev/null || { printf 'NP NOT_MEASURED: cannot fetch main; nothing picked\n'; return 2; } + rc=0; git -C "$1" merge-base --is-ancestor "$4" FETCH_HEAD 2>/dev/null || rc=$? + case "$rc" in + 0) ;; + 1) printf 'NP RED: %s is not on main; nothing picked\n' "${4:0:10}"; return 1 ;; + *) printf 'NP NOT_MEASURED: cannot tell whether %s is on main (merge-base rc=%s); nothing picked\n' "${4:0:10}" "$rc"; return 2 ;; + esac + # create-only: the lease with an empty expected value accepts the push only while the ref does not exist. A plain + # push is NOT create-only: it fast-forwards an existing branch, and main only moves + # forward, so a racing later pick would move the night's C. + rc=0; git -C "$1" push -q --force-with-lease="refs/heads/nightly/$3:" "$2" "$4:refs/heads/nightly/$3" 2>/dev/null || rc=$? + c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/heads/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } + if [ "$c" = "$4" ]; then printf 'C=%s PICKED night %s\n' "$c" "$3"; return 0; fi + printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3"; return 0 +} diff --git a/scripts/lib/workflow_runs_nightly.awk b/scripts/lib/workflow_runs_nightly.awk new file mode 100644 index 0000000000..1a072a60a2 --- /dev/null +++ b/scripts/lib/workflow_runs_nightly.awk @@ -0,0 +1,33 @@ +# workflow_runs_nightly.awk — does a workflow re-check itself every night? Rule 3 of +# scripts/check_workflow_path_filters.sh (#3676), moved here from workflow_path_filters.py (T44, #4686). +# +# Exit 0 when the workflow's on: block has either +# - a `schedule:` with at least one cron entry that has a value, or +# - the chain: `workflow_run:` with `workflows: ["Nightly pick"]` (that one workflow) and `types:` naming +# `completed`; the nightly producers start on the pick's run instead of their own cron (T44). +# Exit 1 otherwise: no on: block, `schedule: []`, a schedule with no cron, a chain from another workflow. +# Comment lines are ignored, so a commented-out trigger is no trigger. A cron is accepted in the forms the YAML +# reader of the old rule accepted: `- cron: X`, `- {cron: X}` and `schedule: [{cron: X}]`; an empty or quoted-empty +# value is no cron. A sequence item may sit at the key's own indent (` schedule:` then ` - cron: X`). +# +# awk -f scripts/lib/workflow_runs_nightly.awk WORKFLOW.yml +function ind(s) { match(s, /^ */); return RLENGTH } +function list1(s) { + sub(/^[^:]*:[[:space:]]*/, "", s); sub(/[[:space:]]+$/, "", s) + if (s !~ /^\[.*\]$/) return "\001" + s = substr(s, 2, length(s) - 2); gsub(/^[[:space:]]+|[[:space:]]+$/, "", s) + if (s ~ /^".*"$/ || s ~ /^\047.*\047$/) s = substr(s, 2, length(s) - 2) + return s +} +function hascron(s) { return s ~ /cron:[[:space:]]*["\047]?[^[:space:],}"\047#]/ } +/^on:[[:space:]]*$/ { on = 1; next } +/^[[:space:]]*(#|$)/ { next } +on && ind($0) == 0 { on = 0 } +!on { next } +ind($0) == 2 && !/^ - / { k = $0; sub(/^ +/, "", k); sub(/:.*/, "", k); sc = (k == "schedule"); wr = (k == "workflow_run") + if (sc && hascron($0)) cron = 1 + next } +sc && /^[[:space:]]+- [{]?[[:space:]]*cron:/ && hascron($0) { cron = 1 } +wr && ind($0) == 4 && /^ +workflows:/ && list1($0) == "Nightly pick" { w = 1 } +wr && ind($0) == 4 && /^ +types:[[:space:]]*\[/ && $0 ~ /[[,[:space:]]completed[],[:space:]]/ { t = 1 } +END { exit !(cron || (w && t)) } diff --git a/scripts/lib_story_pmat.sh b/scripts/lib_story_pmat.sh index ec3129ed9f..d99ab0c94f 100755 --- a/scripts/lib_story_pmat.sh +++ b/scripts/lib_story_pmat.sh @@ -215,3 +215,30 @@ pmat_hunt() { fi return 0 } + +# The eight beats' hunts, one line per beat in beat order: "