From c593a5e6d4d90d43f5d6427ae6fa84bb377c82e4 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 13:47:49 +0200 Subject: [PATCH 01/34] feat(release): nightly pick, one immutable candidate per night MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One job picks the night's candidate C (main's head at pick time) and publishes it as refs/nightly/, create-only and never forced. Producers verify they measured C, the train and a release resolve C, so a merge to main after the pick cannot change the night's line (operator ask). - scripts/lib/nightly_pick.sh: option-neutral lib (night, pick, resolve, verify); 0 ok, 1 RED, 2 NOT_MEASURED, 3 caller error. - scripts/release/nightly_pick.sh: CLI, --self-test 25/25 (incl. the planted merge after the pick and its counter-row), --mutants 10/10. - contracts/nightly-pick-v1.yaml: NP-INV-001..006, FALSIFY-NP-001/002. - .github/workflows/nightly-pick.yml: report-only pick job; scheduled events fire hours late, so producers chain from the pick, not cron. Nothing reads refs/nightly/* yet and nothing is blocking. ont-delta: none — new pattern contract, no new ontology class or predicate Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-pick.yml | 68 ++++++++++++ contracts/nightly-pick-v1.yaml | 99 +++++++++++++++++ scripts/lib/nightly_pick.sh | 77 +++++++++++++ scripts/release/nightly_pick.sh | 172 +++++++++++++++++++++++++++++ 4 files changed, 416 insertions(+) create mode 100644 .github/workflows/nightly-pick.yml create mode 100644 contracts/nightly-pick-v1.yaml create mode 100644 scripts/lib/nightly_pick.sh create mode 100755 scripts/release/nightly_pick.sh diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml new file mode 100644 index 0000000000..f53c90df6b --- /dev/null +++ b/.github/workflows/nightly-pick.yml @@ -0,0 +1,68 @@ +# Nightly pick: ONE job picks the night's candidate C and publishes it as refs/nightly/. +# +# Every producer measures that C, the nightly train prints its line for that C, and a release +# promotes that C. The ref is created once and never moved (create-only push, never forced), so a +# merge to main after the pick cannot change the night's line. Contract: contracts/nightly-pick-v1.yaml. +# +# WHY A PICK, NOT CRON SPACING. Scheduled events here fire 1h43m to 5h48m late (14 nights measured), +# so producers that each read main's head at their own start judge different commits in one night. +# A producer chains from this workflow (workflow_run on "Nightly pick", or a dispatch carrying the +# night) and runs `scripts/release/nightly_pick.sh verify --night N --sha `. +# +# The schedule below is a fallback trigger only. A dispatch is the reliable one, and a re-pick is +# idempotent: it prints KEPT and leaves the ref as it is. The night is the UTC date of (now - 12h), +# so a pick that fires late still lands on its own night. +# +# REPORT-ONLY. Nothing reads refs/nightly/* yet; no job here gates anything. +name: Nightly pick + +on: + schedule: + - cron: '30 20 * * *' + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: nightly-pick + cancel-in-progress: false + +jobs: + pick: + if: github.ref == 'refs/heads/main' + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 10 + outputs: + night: ${{ steps.pick.outputs.night }} + c: ${{ steps.pick.outputs.c }} + steps: + - name: Checkout main + uses: actions/checkout@v7 + with: + ref: main + fetch-depth: 50 + + - name: Picker case table + run: bash scripts/release/nightly_pick.sh --self-test + + - name: Pick C for the night + id: pick + run: | + set -euo pipefail + night="$(bash scripts/release/nightly_pick.sh night)" + rc=0 + bash scripts/release/nightly_pick.sh pick --night "$night" --sha "$GITHUB_SHA" || rc=$? + [ "$rc" -eq 0 ] || { echo "pick exited $rc"; exit "$rc"; } + c="$(bash scripts/release/nightly_pick.sh resolve --night "$night")" + { + echo "night=$night" + echo "c=$c" + } >> "$GITHUB_OUTPUT" + { + echo "### Nightly pick" + echo "" + echo "| night | C | main at pick |" + echo "|---|---|---|" + echo "| $night | \`$c\` | \`$GITHUB_SHA\` |" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml new file mode 100644 index 0000000000..4bd03eb44b --- /dev/null +++ b/contracts/nightly-pick-v1.yaml @@ -0,0 +1,99 @@ +metadata: + version: 1.0.0 + created: '2026-10-04' + author: PAIML Engineering + description: "One job picks the night's candidate C once and publishes it as the ref refs/nightly/. Every producer measures that C, the nightly train prints its line for that C, and a release promotes that C. The ref is created once and never moved, so a merge to main after the pick cannot change the night's line. scripts/release/nightly_pick.sh picks, resolves and verifies; .github/workflows/nightly-pick.yml is the one job. Report-only: no producer, train or release reads the ref yet." + kind: pattern + references: + - "scripts/release/nightly_pick.sh: the command line (night, pick, resolve, verify), the case table (--self-test) and the planted mutants (--mutants)" + - "scripts/lib/nightly_pick.sh: the picker, an option-neutral sourced library that fails by return status" + - ".github/workflows/nightly-pick.yml: the one job that picks C, on a schedule or a dispatch from main" + - ".github/workflows/nightly.yml: its gate job already resolves main's head once per run and builds that sha; this generalises that from one workflow to the night" + +five_whys: + symptom: "On 2026-10-02 the scheduled nightly producers judged three different commits of main in one night (scheduled runs of 2026-09-29 to 2026-10-03, read from the runs API), so their results did not describe one candidate." + why_1: "Each scheduled workflow reads main's head at its own start time." + why_2: "Scheduled events on this account fire 1h43m to 5h48m late (14 nights measured), so the start times spread over hours." + why_3: "Merges to main land inside that spread, so early and late producers see different heads." + why_4: "Nothing names the night's candidate, so neither the train nor a release can say which commit the night measured." + why_5: "Cron spacing was used as the ordering, and the lateness makes cron spacing meaningless." + root_cause: "The night had no single picked commit. The remedy is one pick per night, published as an immutable ref, with the night chained from that pick instead of from cron spacing." + +equations: + night: + formula: "night(t) = UTC date of (t − 12 h)" + domain: "t = the pick's wall-clock time in UTC" + codomain: "YYYY-MM-DD" + invariants: + - "a night runs from 12:00Z to 12:00Z, so a pick that fires hours late still lands on the night it was meant for" + + pick: + formula: "pick(n, s) = ref(n) if ref(n) exists; else create ref(n) := s when s is reachable from main, then return ref(n)" + domain: "n a night, s a 40-hex commit id, ref(n) = refs/nightly/ on the remote" + codomain: "C = ref(n), printed as PICKED (this call created it) or KEPT (it existed)" + invariants: + - "ref(n) is created once and never moved: no force push, and an existing ref is returned unchanged" + - "a commit not on main is refused and no ref is created" + - "a lost race returns the winner as KEPT, never PICKED" + + verify: + formula: "verify(n, s) = OK if s = ref(n); RED if ref(n) exists and s ≠ ref(n); NOT_MEASURED if ref(n) is absent or unreadable" + domain: "n a night, s the commit a producer was handed" + codomain: "exit 0 OK | 1 RED | 2 NOT_MEASURED | 3 caller error" + invariants: + - "a producer handed main's head instead of C is RED" + - "an unreadable remote is NOT_MEASURED, never OK" + +proof_obligations: + - id: NP-INV-001 + type: invariant + property: "A merge to main after the pick does not change the night's line" + formal: "pick(n, s₁) = s₁ ∧ main advances to s₂ ⇒ pick(n, s₂) = s₁ ∧ resolve(n) = s₁ ∧ line(n) unchanged" + applies_to: pick + - id: NP-INV-002 + type: invariant + property: "The night's ref is never moved" + formal: "∀ n: once ref(n) = c, every later pick, including a racing one, leaves ref(n) = c" + applies_to: pick + - id: NP-INV-003 + type: invariant + property: "Only a commit on main can be the night's C" + formal: "s ∉ ancestors(main) ⇒ pick(n, s) = RED ∧ ref(n) absent" + applies_to: pick + - id: NP-INV-004 + type: invariant + property: "A producer that measured another commit is RED" + formal: "ref(n) = c ∧ s ≠ c ⇒ verify(n, s) = RED" + applies_to: verify + - id: NP-INV-005 + type: invariant + property: "No pick or no remote is not_measured, never OK" + formal: "ref(n) absent ∨ remote unreadable ⇒ resolve(n) and verify(n, s) exit 2" + applies_to: verify + - id: NP-INV-006 + type: invariant + property: "A late pick lands on its own night" + formal: "t ∈ [n 12:00Z, n+1 12:00Z) ⇒ night(t) = n" + applies_to: night + +falsification_tests: + - id: FALSIFY-NP-001 + rule: "The picker is right on every row of its case table, including the planted merge after the pick" + prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" + test: "bash scripts/release/nightly_pick.sh --self-test" + if_fails: "A merge after the pick can change what the night measured, so producers, the train and a release can disagree on the commit they judged" + + - id: FALSIFY-NP-002 + rule: "The case table can still fail on the defects it names" + prediction: "Each planted mutant (a force push that moves the ref, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + test: "bash scripts/release/nightly_pick.sh --mutants" + if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" + +qa_gate: + id: QA-NP-001 + name: "One immutable C per night, picked from main and published as a ref" + description: "scripts/release/nightly_pick.sh creates refs/nightly/ once from a commit on main, never moves it, and verifies that a producer measured that commit. Report-only until the producers, the train and the release read it." + checks: + - "night" + - "pick" + - "verify" diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh new file mode 100644 index 0000000000..1e9b472772 --- /dev/null +++ b/scripts/lib/nightly_pick.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash +# nightly_pick.sh (lib) — one candidate C per night, published as an immutable ref (#C310-2, see the checker). +# +# SOURCED, so option-neutral: no shell options are changed here, nothing exits; every function fails by return status. +# . scripts/lib/nightly_pick.sh || exit 1 +# +# The night's ref is refs/nightly/, NIGHT = the UTC date of (pick time - 12 h): a night runs 12:00Z to 12:00Z, +# so a scheduled pick that fires hours late still lands on the night it was meant for. The ref is created once and +# never moved: a second pick the same night keeps the first C, whatever main is by then. Every producer, the train +# and the release read C from that ref, never from main's head. +# +# Return codes: 0 ok; 1 refused (RED); 2 not_measured (could not read or write the remote); 3 caller error. +# +# np_night [EPOCH] print the night of EPOCH (default now) +# np_pick REPO REMOTE NIGHT SHA create refs/nightly/NIGHT -> SHA unless it exists; prints "C=" + PICKED|KEPT +# np_resolve REPO REMOTE NIGHT print the night's C; 2 when the night has no pick +# np_verify REPO REMOTE NIGHT SHA +# 0 when SHA is the night's C, 1 when it is not, 2 when the night has no pick + +np_night() { # [EPOCH] + local e="${1:-}" + [ -n "$e" ] || e="$(date -u +%s)" || return 3 + case "$e" in '' | *[!0-9]*) printf 'NP caller error: epoch %s is not a number\n' "$e" >&2; return 3 ;; esac + date -u -d "@$((e - 43200))" +%F +} + +np__args() { # NIGHT [SHA] + printf '%s\n' "$1" | grep -qxE '[0-9]{4}-[0-9]{2}-[0-9]{2}' || { printf 'NP caller error: night %s is not YYYY-MM-DD\n' "$1" >&2; return 3; } + [ "$#" -lt 2 ] || printf '%s\n' "$2" | grep -qxE '[0-9a-f]{40}' || { printf 'NP caller error: sha %s is not 40 hex\n' "$2" >&2; return 3; } +} + +np__read() { # REPO REMOTE NIGHT -> the ref's sha on stdout; 0 found, 1 absent, 2 unreadable + local out rc=0 + out="$(git -C "$1" ls-remote --refs "$2" "refs/nightly/$3" 2>/dev/null)" || rc=$? + [ "$rc" -eq 0 ] || return 2 + [ -n "$out" ] || return 1 + printf '%s\n' "${out%%[[:space:]]*}" +} + +np_resolve() { # REPO REMOTE NIGHT + local c rc=0 + np__args "$3" || return 3 + c="$(np__read "$1" "$2" "$3")" || rc=$? + case "$rc" in + 0) printf '%s\n' "$c" ;; + 1) printf 'NP NOT_MEASURED: night %s has no pick\n' "$3" >&2; return 2 ;; + *) printf 'NP NOT_MEASURED: cannot read refs/nightly/%s\n' "$3" >&2; return 2 ;; + esac +} + +np_verify() { # REPO REMOTE NIGHT SHA + local c rc=0 + np__args "$3" "$4" || return 3 + c="$(np_resolve "$1" "$2" "$3")" || rc=$? + [ "$rc" -eq 0 ] || return "$rc" + if [ "$c" = "$4" ]; then printf 'NP OK: %s is the C of night %s\n' "${4:0:10}" "$3"; return 0; fi + printf 'NP RED: %s is not the C of night %s (C=%s) -- measure C, not main\n' "${4:0:10}" "$3" "${c:0:10}" + return 1 +} + +np_pick() { # REPO REMOTE NIGHT SHA + local c rc=0 + np__args "$3" "$4" || return 3 + c="$(np__read "$1" "$2" "$3")" || rc=$? + if [ "$rc" -eq 0 ]; then + printf 'C=%s KEPT night %s (picked earlier; main now %s does not move it)\n' "$c" "$3" "${4:0:10}"; return 0 + fi + [ "$rc" -eq 1 ] || { printf 'NP NOT_MEASURED: cannot read refs/nightly/%s; nothing picked\n' "$3"; return 2; } + # C must be on main: a dispatch from a branch must never become the night + git -C "$1" fetch -q "$2" main 2>/dev/null || { printf 'NP NOT_MEASURED: cannot fetch main; nothing picked\n'; return 2; } + git -C "$1" merge-base --is-ancestor "$4" FETCH_HEAD 2>/dev/null || { printf 'NP RED: %s is not on main; nothing picked\n' "${4:0:10}"; return 1; } + # create-only: no force, so a ref another picker created first is never overwritten + git -C "$1" push -q "$2" "$4:refs/nightly/$3" 2>/dev/null || rc=$? + c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } + if [ "$c" = "$4" ]; then printf 'C=%s PICKED night %s\n' "$c" "$3"; return 0; fi + printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3" +} diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh new file mode 100755 index 0000000000..d110217da0 --- /dev/null +++ b/scripts/release/nightly_pick.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# nightly_pick.sh — pick the night's candidate C once and publish it as refs/nightly/. +# +# The rule (operator): one job picks C once a night and publishes it as a ref; every producer measures that C; the +# train prints its line for that C; a release promotes that C. Planted falsifier: a merge to main after C is picked +# must not change the night's line. +# +# Why a ref and not cron spacing: scheduled events on this account fire 1h43m-5h48m late (14 nights measured), so +# workflows that each read main's head at their own late start judge different commits. The night is chained from the +# one pick instead: producers start on the pick (workflow_run or dispatch) and verify the commit they were handed. +# +# Usage: +# nightly_pick.sh night [--at EPOCH] +# nightly_pick.sh pick [--repo DIR] [--remote NAME] [--night N | --at EPOCH] --sha SHA +# nightly_pick.sh resolve [--repo DIR] [--remote NAME] --night N +# nightly_pick.sh verify [--repo DIR] [--remote NAME] --night N --sha SHA +# nightly_pick.sh --self-test | --mutants +# Exit: 0 ok; 1 RED (refused); 2 not_measured; 3 caller error. Report-only: nothing reads this yet. +set -uo pipefail + +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_PATH="$HERE/$(basename -- "${BASH_SOURCE[0]}")" +LIB="$HERE/../lib/nightly_pick.sh" + +caller_error() { printf 'nightly_pick: caller error: %s\n' "$1" >&2; exit 3; } + +# ------------------------------------------------------------------------------------ the case table ---- +CASES=0; FAILED=0 +row() { # NAME WANT-RC MUST MUSTNOT -- CMD... + local name="$1" want="$2" must="$3" mustnot="$4" out rc + shift 5 + out="$("$@" 2>&1)"; rc=$? + CASES=$((CASES + 1)) + if [ "$rc" != "$want" ]; then printf 'RED %-44s rc=%s want %s\n' "$name" "$rc" "$want"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$mustnot" ] && printf '%s\n' "$out" | grep -qF -e "$mustnot"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi + printf 'ok %s\n' "$name" +} +same() { # NAME A B : two outputs must be equal + CASES=$((CASES + 1)) + if [ -n "$2" ] && [ "$2" = "$3" ]; then printf 'ok %s\n' "$1"; else printf 'RED %-44s %s != %s\n' "$1" "$2" "$3"; FAILED=$((FAILED + 1)); fi +} +commit() { # REPO MSG -> new head sha + git -C "$1" -c core.hooksPath=/dev/null -c user.name=fixture -c user.email=fixture@example.invalid commit -q --allow-empty -m "$2" && git -C "$1" rev-parse HEAD +} +# the train's line for a night, as the train must print it: from the night's ref, never from main +train_line() { local c; c="$(bash "$SCRIPT_PATH" resolve --repo "$1" --night "$2")" || return $?; printf 'NOT RELEASABLE: stub [C=%s]\n' "${c:0:10}"; } +# the line as a train that reads main's head prints it: the defect this row removes +main_line() { printf 'NOT RELEASABLE: stub [C=%s]\n' "$(git -C "$1" ls-remote "$2" refs/heads/main | cut -c1-10)"; } + +self_test() { + local FX W c1 c2 c3 br n=2026-10-04 l1 l2 m1 m2 + FX="$(mktemp -d "${TMPDIR:-/tmp}/np-st.XXXXXX")" || caller_error "no temp dir" + W="$FX/w" + git init -q --bare -b main "$FX/origin.git" && git clone -q "$FX/origin.git" "$W" 2>/dev/null || caller_error "no fixture repo" + git -C "$W" checkout -q -b main 2>/dev/null + c1="$(commit "$W" one)" && git -C "$W" push -q origin main || caller_error "fixture push" + + row night_mid_evening_is_today 0 "2026-10-04" "" -- bash "$SCRIPT_PATH" night --at 1791154800 # 2026-10-04T23:00Z + row night_late_fire_is_still_the_night 0 "2026-10-04" "" -- bash "$SCRIPT_PATH" night --at 1791179280 # 2026-10-05T05:48Z + row night_turns_at_noon 0 "2026-10-05" "" -- bash "$SCRIPT_PATH" night --at 1791201600 # 2026-10-05T12:00Z + row unpicked_night_is_not_measured 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night "$n" + row first_pick_publishes_the_ref 0 "C=$c1 PICKED night $n" "" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "$n" --sha "$c1" + row the_ref_is_on_the_remote 0 "$c1" "" -- git -C "$W" ls-remote "$FX/origin.git" "refs/nightly/$n" + l1="$(train_line "$W" "$n")"; m1="$(main_line "$W" "$FX/origin.git")" + # THE PLANT: a merge to main after the pick + c2="$(commit "$W" merged-after-the-pick)" && git -C "$W" push -q origin main || caller_error "fixture push" + row a_second_pick_keeps_the_first_c 0 "C=$c1 KEPT night $n" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "$n" --sha "$c2" + row resolve_after_a_merge_is_still_c 0 "$c1" "$c2" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night "$n" + l2="$(train_line "$W" "$n")"; m2="$(main_line "$W" "$FX/origin.git")" + same a_merge_after_the_pick_does_not_change_the_line "$l1" "$l2" + CASES=$((CASES + 1)) # the counter-row: reading main's head DOES change the line, so the plant can fail + if [ "$m1" != "$m2" ]; then printf 'ok %s\n' main_head_reader_changes_the_line; else printf 'RED %-44s the plant did not move main\n' main_head_reader_changes_the_line; FAILED=$((FAILED + 1)); fi + row producer_handed_c_verifies 0 "NP OK" "" -- bash "$SCRIPT_PATH" verify --repo "$W" --night "$n" --sha "$c1" + row producer_handed_main_head_is_red 1 "is not the C of night $n" "NP OK" -- bash "$SCRIPT_PATH" verify --repo "$W" --night "$n" --sha "$c2" + row verify_on_an_unpicked_night_is_not_measured 2 "has no pick" "NP OK" -- bash "$SCRIPT_PATH" verify --repo "$W" --night 2026-10-03 --sha "$c1" + row next_night_picks_its_own_c 0 "C=$c2 PICKED night 2026-10-05" "" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-05 --sha "$c2" + row the_old_night_still_resolves_to_its_c 0 "$c1" "$c2" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night "$n" + git -C "$W" checkout -q -b side && br="$(commit "$W" branch-only)" && git -C "$W" push -q origin side && git -C "$W" checkout -q main + row a_commit_not_on_main_is_refused 1 "is not on main" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-06 --sha "$br" + row a_refused_pick_publishes_no_ref 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-06 + c3="$(commit "$W" racer)" && git -C "$W" push -q origin main && git -C "$W" push -q origin "$c3:refs/nightly/2026-10-07" + row a_pick_never_overwrites_an_existing_ref 0 "C=$c3 KEPT night 2026-10-07" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-07 --sha "$c2" + # a race: the ref is absent when read and created by another picker before the push lands + mkdir -p "$FX/bin" && printf '%s\n' '#!/usr/bin/env bash' \ + "case \" \$* \" in" \ + " *' ls-remote '*) [ -f '$FX/seen' ] || { : > '$FX/seen'; exit 0; } ;;" \ + " *' push '*) '$(command -v git)' -C '$FX/origin.git' update-ref refs/nightly/2026-10-10 '$c3' ;;" \ + "esac" "exec '$(command -v git)' \"\$@\"" > "$FX/bin/git" && chmod +x "$FX/bin/git" + row a_lost_race_keeps_the_winner 0 "C=$c3 KEPT night 2026-10-10 (another pick won the race)" "PICKED" -- env PATH="$FX/bin:$PATH" bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-10 --sha "$c2" + row unreachable_remote_is_not_measured 2 "NOT_MEASURED" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --remote "$FX/absent.git" --night 2026-10-08 --sha "$c2" + row unreachable_remote_resolve_is_not_measured 2 "cannot read" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --remote "$FX/absent.git" --night "$n" + row malformed_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-1-4 --sha "$c1" + row malformed_sha_is_a_caller_error 3 "not 40 hex" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 --sha "${c1:0:10}" + row pick_without_sha_is_a_caller_error 3 "caller error" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 + row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" repick + rm -rf -- "${FX:?}" + printf '%s %s/%s case rows green\n' "$([ "$FAILED" -eq 0 ] && echo SELF-TEST-GREEN || echo SELF-TEST-RED)" "$((CASES - FAILED))" "$CASES" + [ "$FAILED" -eq 0 ] +} + +# ------------------------------------------------------------------------------------ the mutants ---- +# namesed expression on the lib. Each must change the lib and turn the case table RED. +MUTANTS='m01_pick_moves_the_ref s/push -q "\$2" "\$4:refs\/nightly\/\$3"/push -q -f "$2" "$4:refs\/nightly\/$3"/ +m02_existing_ref_ignored s/ if \[ "\$rc" -eq 0 \]; then$/ if false; then/ +m03_branch_commit_accepted s/merge-base --is-ancestor "\$4" FETCH_HEAD 2>\/dev\/null ||/true ||/ +m04_verify_accepts_any_sha s/if \[ "\$c" = "\$4" \]; then printf .NP OK/if true; then printf '"'"'NP OK/ +m05_unpicked_night_resolves s/1) printf .NP NOT_MEASURED: night %s has no pick\\n. "\$3" >\&2; return 2 ;;/1) printf "\\n"; return 0 ;;/ +m06_unreadable_is_absent s/\[ "\$rc" -eq 0 \] || return 2$/[ "$rc" -eq 0 ] || return 1/ +m07_night_is_the_fire_date s/\$((e - 43200))/$e/ +m08_bad_night_accepted s/grep -qxE .\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}. || {/true || {/ +m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ +m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"$/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/' +mutants() { + local tmp name expr killed=0 total=0 errors=0 out + tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" + mkdir -p "$tmp/release" "$tmp/lib" + cp "$SCRIPT_PATH" "$tmp/release/" || caller_error "cannot copy the checker" + while IFS="$(printf '\t')" read -r name expr; do + [ -n "$name" ] || continue + total=$((total + 1)) + sed -e "$expr" "$LIB" > "$tmp/lib/nightly_pick.sh" + if cmp -s "$LIB" "$tmp/lib/nightly_pick.sh"; then + printf 'ERROR %-32s the patch did not apply (an error, never a survivor)\n' "$name"; errors=$((errors + 1)); continue + fi + if out="$(bash "$tmp/release/nightly_pick.sh" --self-test 2>&1)"; then + printf 'SURVIVED %s\n' "$name" + else + killed=$((killed + 1)) + printf 'killed %-32s %s\n' "$name" "$(printf '%s\n' "$out" | grep -c -e '^RED ')" + fi + done <<< "$MUTANTS" + rm -rf -- "${tmp:?}" + printf 'MUTANTS killed=%s total=%s errors=%s\n' "$killed" "$total" "$errors" + [ "$killed" -eq "$total" ] && [ "$errors" -eq 0 ] +} + +# ------------------------------------------------------------------------------------ main ---- +[ -f "$LIB" ] || caller_error "no library at $LIB" +# shellcheck source=../lib/nightly_pick.sh +. "$LIB" || caller_error "cannot source $LIB" + +cmd="${1:-}"; [ "$#" -gt 0 ] && shift +case "$cmd" in + --self-test) self_test; exit $? ;; + --mutants) mutants; exit $? ;; + night | pick | resolve | verify) ;; + -h | --help) sed -n '2,19p' "$SCRIPT_PATH"; exit 0 ;; + *) caller_error "unknown command '$cmd' (night|pick|resolve|verify|--self-test|--mutants)" ;; +esac +repo=.; remote=origin; night=''; at=''; sha='' +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) repo="${2:-}"; shift 2 || caller_error "--repo DIR" ;; + --remote) remote="${2:-}"; shift 2 || caller_error "--remote NAME" ;; + --night) night="${2:-}"; shift 2 || caller_error "--night YYYY-MM-DD" ;; + --at) at="${2:-}"; shift 2 || caller_error "--at EPOCH" ;; + --sha) sha="${2:-}"; shift 2 || caller_error "--sha SHA" ;; + *) caller_error "unknown option '$1'" ;; + esac +done +if [ "$cmd" = night ]; then np_night "$at"; exit $?; fi +if [ -z "$night" ]; then + [ "$cmd" = pick ] || caller_error "--night YYYY-MM-DD" + night="$(np_night "$at")" || exit 3 +fi +case "$cmd" in + pick) [ -n "$sha" ] || caller_error "pick needs --sha SHA"; np_pick "$repo" "$remote" "$night" "$sha" ;; + resolve) np_resolve "$repo" "$remote" "$night" ;; + verify) [ -n "$sha" ] || caller_error "verify needs --sha SHA"; np_verify "$repo" "$remote" "$night" "$sha" ;; +esac +exit $? From 654155e212f07bf51b02d5817b94667866a0e5ba Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 13:51:47 +0200 Subject: [PATCH 02/34] fix(release): nightly pick review round 1, create-only push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain push is not create-only outside refs/heads and refs/tags: it fast-forwards an existing ref. main only moves forward, so a pick that lost a race to an older commit moved the night's C (proved: with the lease removed, the new race row goes RED). - push carries --force-with-lease=refs/nightly/: (expects the ref absent), so the first pick wins in every race. - merge-base exit 1 is RED (not on main); any other failure is NOT_MEASURED, never RED. - np_night fails by return 3 if date fails; lib header cites no ruling. - case table 28/28: race won by an ancestor, the ref left at the winner, an unknown commit is not_measured. Mutants 12/12 (m11 plain push, m12 unknown commit as RED; m01 now drops the lease for -f). ont-delta: none — contract text only, no new ontology class or predicate Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-pick-v1.yaml | 10 +++++----- scripts/lib/nightly_pick.sh | 17 ++++++++++++----- scripts/release/nightly_pick.sh | 27 +++++++++++++++++++-------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index 4bd03eb44b..15a489bba2 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -32,9 +32,9 @@ equations: domain: "n a night, s a 40-hex commit id, ref(n) = refs/nightly/ on the remote" codomain: "C = ref(n), printed as PICKED (this call created it) or KEPT (it existed)" invariants: - - "ref(n) is created once and never moved: no force push, and an existing ref is returned unchanged" - - "a commit not on main is refused and no ref is created" - - "a lost race returns the winner as KEPT, never PICKED" + - "ref(n) is created once and never moved: the push carries a lease that expects the ref absent (a plain push is not enough: outside refs/heads and refs/tags it fast-forwards an existing ref), and an existing ref is returned unchanged" + - "a commit not on main is refused and no ref is created; a commit whose ancestry cannot be read is not_measured, never refused as RED and never picked" + - "a lost race returns the winner as KEPT, never PICKED, also when the winner is an ancestor of the loser's commit" verify: formula: "verify(n, s) = OK if s = ref(n); RED if ref(n) exists and s ≠ ref(n); NOT_MEASURED if ref(n) is absent or unreadable" @@ -79,13 +79,13 @@ proof_obligations: falsification_tests: - id: FALSIFY-NP-001 rule: "The picker is right on every row of its case table, including the planted merge after the pick" - prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" + prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" test: "bash scripts/release/nightly_pick.sh --self-test" if_fails: "A merge after the pick can change what the night measured, so producers, the train and a release can disagree on the commit they judged" - id: FALSIFY-NP-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (a force push that moves the ref, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" test: "bash scripts/release/nightly_pick.sh --mutants" if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh index 1e9b472772..fd680e2d36 100644 --- a/scripts/lib/nightly_pick.sh +++ b/scripts/lib/nightly_pick.sh @@ -1,5 +1,5 @@ # shellcheck shell=bash -# nightly_pick.sh (lib) — one candidate C per night, published as an immutable ref (#C310-2, see the checker). +# nightly_pick.sh (lib) — one candidate C per night, published as an immutable ref (see scripts/release/nightly_pick.sh). # # SOURCED, so option-neutral: no shell options are changed here, nothing exits; every function fails by return status. # . scripts/lib/nightly_pick.sh || exit 1 @@ -21,7 +21,7 @@ np_night() { # [EPOCH] local e="${1:-}" [ -n "$e" ] || e="$(date -u +%s)" || return 3 case "$e" in '' | *[!0-9]*) printf 'NP caller error: epoch %s is not a number\n' "$e" >&2; return 3 ;; esac - date -u -d "@$((e - 43200))" +%F + date -u -d "@$((e - 43200))" +%F || return 3 } np__args() { # NIGHT [SHA] @@ -68,9 +68,16 @@ np_pick() { # REPO REMOTE NIGHT SHA [ "$rc" -eq 1 ] || { printf 'NP NOT_MEASURED: cannot read refs/nightly/%s; nothing picked\n' "$3"; return 2; } # C must be on main: a dispatch from a branch must never become the night git -C "$1" fetch -q "$2" main 2>/dev/null || { printf 'NP NOT_MEASURED: cannot fetch main; nothing picked\n'; return 2; } - git -C "$1" merge-base --is-ancestor "$4" FETCH_HEAD 2>/dev/null || { printf 'NP RED: %s is not on main; nothing picked\n' "${4:0:10}"; return 1; } - # create-only: no force, so a ref another picker created first is never overwritten - git -C "$1" push -q "$2" "$4:refs/nightly/$3" 2>/dev/null || rc=$? + rc=0; git -C "$1" merge-base --is-ancestor "$4" FETCH_HEAD 2>/dev/null || rc=$? + case "$rc" in + 0) ;; + 1) printf 'NP RED: %s is not on main; nothing picked\n' "${4:0:10}"; return 1 ;; + *) printf 'NP NOT_MEASURED: cannot tell whether %s is on main (merge-base rc=%s); nothing picked\n' "${4:0:10}" "$rc"; return 2 ;; + esac + # create-only: the lease with an empty expected value accepts the push only while the ref does not exist. A plain + # push is NOT create-only outside refs/heads and refs/tags: it fast-forwards an existing ref, and main only moves + # forward, so a racing later pick would move the night's C. + rc=0; git -C "$1" push -q --force-with-lease="refs/nightly/$3:" "$2" "$4:refs/nightly/$3" 2>/dev/null || rc=$? c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } if [ "$c" = "$4" ]; then printf 'C=%s PICKED night %s\n' "$c" "$3"; return 0; fi printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3" diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index d110217da0..35b0a3ec10 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -81,13 +81,22 @@ self_test() { row a_refused_pick_publishes_no_ref 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-06 c3="$(commit "$W" racer)" && git -C "$W" push -q origin main && git -C "$W" push -q origin "$c3:refs/nightly/2026-10-07" row a_pick_never_overwrites_an_existing_ref 0 "C=$c3 KEPT night 2026-10-07" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-07 --sha "$c2" - # a race: the ref is absent when read and created by another picker before the push lands - mkdir -p "$FX/bin" && printf '%s\n' '#!/usr/bin/env bash' \ - "case \" \$* \" in" \ - " *' ls-remote '*) [ -f '$FX/seen' ] || { : > '$FX/seen'; exit 0; } ;;" \ - " *' push '*) '$(command -v git)' -C '$FX/origin.git' update-ref refs/nightly/2026-10-10 '$c3' ;;" \ - "esac" "exec '$(command -v git)' \"\$@\"" > "$FX/bin/git" && chmod +x "$FX/bin/git" + # a race: the ref is absent when read and created by another picker before the push lands. race_shim DIR NIGHT + # WINNER puts a git on PATH whose first ls-remote sees nothing and whose push first creates the ref at WINNER. + race_shim() { + mkdir -p "$1" && printf '%s\n' '#!/usr/bin/env bash' \ + "case \" \$* \" in" \ + " *' ls-remote '*) [ -f '$1/seen' ] || { : > '$1/seen'; exit 0; } ;;" \ + " *' push '*) '$(command -v git)' -C '$FX/origin.git' update-ref refs/nightly/$2 '$3' ;;" \ + "esac" "exec '$(command -v git)' \"\$@\"" > "$1/git" && chmod +x "$1/git" + } + race_shim "$FX/bin" 2026-10-10 "$c3" row a_lost_race_keeps_the_winner 0 "C=$c3 KEPT night 2026-10-10 (another pick won the race)" "PICKED" -- env PATH="$FX/bin:$PATH" bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-10 --sha "$c2" + # the winner is an ANCESTOR of the loser's commit (main only moves forward), so a plain push would fast-forward it + race_shim "$FX/bin-ff" 2026-10-11 "$c1" + row a_lost_race_to_an_older_commit_keeps_the_winner 0 "C=$c1 KEPT night 2026-10-11 (another pick won the race)" "PICKED" -- env PATH="$FX/bin-ff:$PATH" bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-11 --sha "$c3" + row a_lost_race_leaves_the_ref_at_the_winner 0 "$c1" "$c3" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-11 + row an_unknown_commit_is_not_measured 2 "cannot tell whether" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-12 --sha 0123456789abcdef0123456789abcdef01234567 row unreachable_remote_is_not_measured 2 "NOT_MEASURED" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --remote "$FX/absent.git" --night 2026-10-08 --sha "$c2" row unreachable_remote_resolve_is_not_measured 2 "cannot read" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --remote "$FX/absent.git" --night "$n" row malformed_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-1-4 --sha "$c1" @@ -101,7 +110,7 @@ self_test() { # ------------------------------------------------------------------------------------ the mutants ---- # namesed expression on the lib. Each must change the lib and turn the case table RED. -MUTANTS='m01_pick_moves_the_ref s/push -q "\$2" "\$4:refs\/nightly\/\$3"/push -q -f "$2" "$4:refs\/nightly\/$3"/ +MUTANTS='m01_pick_moves_the_ref s/--force-with-lease="refs\/nightly\/\$3:"/-f/ m02_existing_ref_ignored s/ if \[ "\$rc" -eq 0 \]; then$/ if false; then/ m03_branch_commit_accepted s/merge-base --is-ancestor "\$4" FETCH_HEAD 2>\/dev\/null ||/true ||/ m04_verify_accepts_any_sha s/if \[ "\$c" = "\$4" \]; then printf .NP OK/if true; then printf '"'"'NP OK/ @@ -110,7 +119,9 @@ m06_unreadable_is_absent s/\[ "\$rc" -eq 0 \] || return 2$/[ "$rc" -eq 0 ] || re m07_night_is_the_fire_date s/\$((e - 43200))/$e/ m08_bad_night_accepted s/grep -qxE .\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}. || {/true || {/ m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ -m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"$/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/' +m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"$/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ +m11_plain_push_fast_forwards s/ --force-with-lease="refs\/nightly\/\$3:"// +m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/' mutants() { local tmp name expr killed=0 total=0 errors=0 out tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" From 8cb404c7df096dfcac5aeaefd8c998e3286f81c4 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 13:56:06 +0200 Subject: [PATCH 03/34] fix(release): nightly pick review round 2, ambiguous ref names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git matches an ls-remote pattern and a push destination by their tail, so a branch named refs/heads/refs/nightly/ was read as the night's C, and it swallowed the pick's push ("Everything up-to-date", no ref created). Any other remote ref ending in refs/nightly/ now makes the night ambiguous: NOT_MEASURED for pick and resolve, never a C. - np__read reads the exact ref name only and refuses an ambiguous one. - the race loser's KEPT path returns 0 explicitly. - workflow checks out full history so the on-main test is not shallow. - case table 30/30 (decoy branch rows), mutants 13/13 (m13 the decoy read as C; m10 re-anchored after the explicit return). ont-delta: none — contract text only, no new ontology class or predicate Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-pick.yml | 2 +- contracts/nightly-pick-v1.yaml | 5 +++-- scripts/lib/nightly_pick.sh | 11 +++++++++-- scripts/release/nightly_pick.sh | 9 +++++++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml index f53c90df6b..fdd081be42 100644 --- a/.github/workflows/nightly-pick.yml +++ b/.github/workflows/nightly-pick.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v7 with: ref: main - fetch-depth: 50 + fetch-depth: 0 - name: Picker case table run: bash scripts/release/nightly_pick.sh --self-test diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index 15a489bba2..e612c83c7d 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -43,6 +43,7 @@ equations: invariants: - "a producer handed main's head instead of C is RED" - "an unreadable remote is NOT_MEASURED, never OK" + - "another remote ref whose name ends in refs/nightly/ (git matches a pattern and a push destination by tail) makes the night ambiguous: NOT_MEASURED, never a C" proof_obligations: - id: NP-INV-001 @@ -79,13 +80,13 @@ proof_obligations: falsification_tests: - id: FALSIFY-NP-001 rule: "The picker is right on every row of its case table, including the planted merge after the pick" - prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" + prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; a decoy branch named refs/nightly/ makes the night ambiguous and not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" test: "bash scripts/release/nightly_pick.sh --self-test" if_fails: "A merge after the pick can change what the night measured, so producers, the train and a release can disagree on the commit they judged" - id: FALSIFY-NP-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" test: "bash scripts/release/nightly_pick.sh --mutants" if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh index fd680e2d36..96dc0387c1 100644 --- a/scripts/lib/nightly_pick.sh +++ b/scripts/lib/nightly_pick.sh @@ -33,8 +33,15 @@ np__read() { # REPO REMOTE NIGHT -> the ref's sha on stdout; 0 found, 1 absent, local out rc=0 out="$(git -C "$1" ls-remote --refs "$2" "refs/nightly/$3" 2>/dev/null)" || rc=$? [ "$rc" -eq 0 ] || return 2 + # exact name only. ls-remote matches a pattern by its tail, and so does a push destination, so another ref ending in + # refs/nightly/N (a branch refs/heads/refs/nightly/N) would be read as C, and would swallow the pick's push. + # Such a name makes the night ambiguous: unreadable (2), never a C. + if printf '%s\n' "$out" | awk -v r="refs/nightly/$3" 'NF && $2 != r { found = 1 } END { exit !found }'; then + printf 'NP: refs/nightly/%s is ambiguous on the remote (another ref ends in that name)\n' "$3" >&2; return 2 + fi + out="$(printf '%s\n' "$out" | awk -v r="refs/nightly/$3" '$2 == r { print $1; exit }')" [ -n "$out" ] || return 1 - printf '%s\n' "${out%%[[:space:]]*}" + printf '%s\n' "$out" } np_resolve() { # REPO REMOTE NIGHT @@ -80,5 +87,5 @@ np_pick() { # REPO REMOTE NIGHT SHA rc=0; git -C "$1" push -q --force-with-lease="refs/nightly/$3:" "$2" "$4:refs/nightly/$3" 2>/dev/null || rc=$? c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } if [ "$c" = "$4" ]; then printf 'C=%s PICKED night %s\n' "$c" "$3"; return 0; fi - printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3" + printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3"; return 0 } diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index 35b0a3ec10..1f7c537d83 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -97,6 +97,10 @@ self_test() { row a_lost_race_to_an_older_commit_keeps_the_winner 0 "C=$c1 KEPT night 2026-10-11 (another pick won the race)" "PICKED" -- env PATH="$FX/bin-ff:$PATH" bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-11 --sha "$c3" row a_lost_race_leaves_the_ref_at_the_winner 0 "$c1" "$c3" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-11 row an_unknown_commit_is_not_measured 2 "cannot tell whether" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-12 --sha 0123456789abcdef0123456789abcdef01234567 + # a decoy: a BRANCH named refs/nightly/ must not be read as the night's ref (ls-remote matches by tail) + git -C "$W" push -q origin "$br:refs/heads/refs/nightly/2026-10-13" || caller_error "fixture decoy push" + row a_decoy_branch_is_not_the_night 2 "is ambiguous" "C=" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-13 --sha "$c3" + row a_decoy_branch_does_not_resolve 2 "is ambiguous" "$br" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-13 row unreachable_remote_is_not_measured 2 "NOT_MEASURED" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --remote "$FX/absent.git" --night 2026-10-08 --sha "$c2" row unreachable_remote_resolve_is_not_measured 2 "cannot read" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --remote "$FX/absent.git" --night "$n" row malformed_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-1-4 --sha "$c1" @@ -119,9 +123,10 @@ m06_unreadable_is_absent s/\[ "\$rc" -eq 0 \] || return 2$/[ "$rc" -eq 0 ] || re m07_night_is_the_fire_date s/\$((e - 43200))/$e/ m08_bad_night_accepted s/grep -qxE .\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}. || {/true || {/ m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ -m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"$/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ +m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ m11_plain_push_fast_forwards s/ --force-with-lease="refs\/nightly\/\$3:"// -m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/' +m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ +m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/' mutants() { local tmp name expr killed=0 total=0 errors=0 out tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" From c1263b1e1f284ed6033429ab1029d29fff7c44f9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 14:20:59 +0200 Subject: [PATCH 04/34] fix(release): nightly pick ref is a branch, refs/heads/nightly/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repository rulesets cover only branches and tags, so the night's ref moves from refs/nightly/ to the branch refs/heads/nightly/, where a ruleset (update, deletion and non-fast-forward refused; creation allowed; no bypass) can hold it immutable on the server. The pick still creates it once with the expect-absent lease, pushed with the workflow's GITHUB_TOKEN, so no workflow fires on the new branch. Only that exact branch is C. New case rows: the rolling tag refs/tags/nightly is never read as a pick, a pick beside it creates the branch, and a branch named nightly (a directory/file clash) is not_measured. The ambiguity and decoy rows are re-planted under the new namespace. Receipts: --self-test 34/34 green; --mutants killed=13 total=13 errors=0; pv validate valid; bashrs 0 errors on both scripts. ont-delta: none — the contract's refs and invariant wording move to the branch namespace; no new term or relation Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-pick.yml | 8 ++++++-- contracts/nightly-pick-v1.yaml | 13 +++++++------ scripts/lib/nightly_pick.sh | 28 ++++++++++++++++------------ scripts/release/nightly_pick.sh | 24 ++++++++++++++++-------- 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml index fdd081be42..4fe39c0791 100644 --- a/.github/workflows/nightly-pick.yml +++ b/.github/workflows/nightly-pick.yml @@ -1,4 +1,4 @@ -# Nightly pick: ONE job picks the night's candidate C and publishes it as refs/nightly/. +# Nightly pick: ONE job picks the night's candidate C and publishes it as refs/heads/nightly/. # # Every producer measures that C, the nightly train prints its line for that C, and a release # promotes that C. The ref is created once and never moved (create-only push, never forced), so a @@ -13,7 +13,11 @@ # idempotent: it prints KEPT and leaves the ref as it is. The night is the UTC date of (now - 12h), # so a pick that fires late still lands on its own night. # -# REPORT-ONLY. Nothing reads refs/nightly/* yet; no job here gates anything. +# The ref is a BRANCH because repository rulesets cover only branches and tags; a ruleset on refs/heads/nightly/* +# (update, deletion and non-fast-forward refused; creation allowed; no bypass) keeps it immutable on the server. It is +# pushed with this workflow's GITHUB_TOKEN, so creating it fires no workflow. The rolling tag refs/tags/nightly is never C. +# +# REPORT-ONLY. Nothing reads refs/heads/nightly/* yet; no job here gates anything. name: Nightly pick on: diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index e612c83c7d..dfca694788 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "One job picks the night's candidate C once and publishes it as the ref refs/nightly/. Every producer measures that C, the nightly train prints its line for that C, and a release promotes that C. The ref is created once and never moved, so a merge to main after the pick cannot change the night's line. scripts/release/nightly_pick.sh picks, resolves and verifies; .github/workflows/nightly-pick.yml is the one job. Report-only: no producer, train or release reads the ref yet." + description: "One job picks the night's candidate C once and publishes it as the ref refs/heads/nightly/. Every producer measures that C, the nightly train prints its line for that C, and a release promotes that C. The ref is created once and never moved, so a merge to main after the pick cannot change the night's line. scripts/release/nightly_pick.sh picks, resolves and verifies; .github/workflows/nightly-pick.yml is the one job. Report-only: no producer, train or release reads the ref yet." kind: pattern references: - "scripts/release/nightly_pick.sh: the command line (night, pick, resolve, verify), the case table (--self-test) and the planted mutants (--mutants)" @@ -29,11 +29,12 @@ equations: pick: formula: "pick(n, s) = ref(n) if ref(n) exists; else create ref(n) := s when s is reachable from main, then return ref(n)" - domain: "n a night, s a 40-hex commit id, ref(n) = refs/nightly/ on the remote" + domain: "n a night, s a 40-hex commit id, ref(n) = refs/heads/nightly/ on the remote" codomain: "C = ref(n), printed as PICKED (this call created it) or KEPT (it existed)" invariants: - - "ref(n) is created once and never moved: the push carries a lease that expects the ref absent (a plain push is not enough: outside refs/heads and refs/tags it fast-forwards an existing ref), and an existing ref is returned unchanged" + - "ref(n) is created once and never moved: the push carries a lease that expects the ref absent (a plain push is not enough: it fast-forwards an existing branch). It is a branch because repository rulesets cover only branches and tags; a ruleset on refs/heads/nightly/* refusing update, deletion and non-fast-forward (creation allowed, no bypass) holds the same rule on the server, and an existing ref is returned unchanged" - "a commit not on main is refused and no ref is created; a commit whose ancestry cannot be read is not_measured, never refused as RED and never picked" + - "only the exact branch refs/heads/nightly/ is C: the rolling tag refs/tags/nightly is never read, and a branch named nightly (a directory/file clash) makes the pick not_measured" - "a lost race returns the winner as KEPT, never PICKED, also when the winner is an ancestor of the loser's commit" verify: @@ -43,7 +44,7 @@ equations: invariants: - "a producer handed main's head instead of C is RED" - "an unreadable remote is NOT_MEASURED, never OK" - - "another remote ref whose name ends in refs/nightly/ (git matches a pattern and a push destination by tail) makes the night ambiguous: NOT_MEASURED, never a C" + - "another remote ref whose name ends in refs/heads/nightly/ (git matches a pattern and a push destination by tail) makes the night ambiguous: NOT_MEASURED, never a C" proof_obligations: - id: NP-INV-001 @@ -80,7 +81,7 @@ proof_obligations: falsification_tests: - id: FALSIFY-NP-001 rule: "The picker is right on every row of its case table, including the planted merge after the pick" - prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; a decoy branch named refs/nightly/ makes the night ambiguous and not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" + prediction: "Against a fixture remote: the first pick publishes the ref; a merge to main then a second pick keeps the first C; the train-line stub is identical before and after the merge while a main-head reader's line changes (the counter-row that proves the plant can fail); a producer handed main's head is RED; a commit not on main is refused with no ref; an existing or racing ref is never overwritten, including a race won by an older commit that a plain push would fast-forward; a commit whose ancestry cannot be read is not_measured; a decoy branch whose name ends in refs/heads/nightly/ makes the night ambiguous and not_measured; the rolling tag is never C; a branch named nightly is not_measured; an unreadable remote and an unpicked night are not_measured; malformed input is a caller error; a late fire lands on its night" test: "bash scripts/release/nightly_pick.sh --self-test" if_fails: "A merge after the pick can change what the night measured, so producers, the train and a release can disagree on the commit they judged" @@ -93,7 +94,7 @@ falsification_tests: qa_gate: id: QA-NP-001 name: "One immutable C per night, picked from main and published as a ref" - description: "scripts/release/nightly_pick.sh creates refs/nightly/ once from a commit on main, never moves it, and verifies that a producer measured that commit. Report-only until the producers, the train and the release read it." + description: "scripts/release/nightly_pick.sh creates refs/heads/nightly/ once from a commit on main, never moves it, and verifies that a producer measured that commit. Report-only until the producers, the train and the release read it." checks: - "night" - "pick" diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh index 96dc0387c1..79d048d132 100644 --- a/scripts/lib/nightly_pick.sh +++ b/scripts/lib/nightly_pick.sh @@ -4,15 +4,19 @@ # SOURCED, so option-neutral: no shell options are changed here, nothing exits; every function fails by return status. # . scripts/lib/nightly_pick.sh || exit 1 # -# The night's ref is refs/nightly/, NIGHT = the UTC date of (pick time - 12 h): a night runs 12:00Z to 12:00Z, +# The night's ref is refs/heads/nightly/, NIGHT = the UTC date of (pick time - 12 h): a night runs 12:00Z to 12:00Z, # so a scheduled pick that fires hours late still lands on the night it was meant for. The ref is created once and # never moved: a second pick the same night keeps the first C, whatever main is by then. Every producer, the train # and the release read C from that ref, never from main's head. +# It is a BRANCH because repository rulesets cover only branches and tags: a ruleset on refs/heads/nightly/* (update, +# deletion and non-fast-forward refused, creation allowed, no bypass) makes it immutable on the server as well. The +# pick pushes with the workflow's GITHUB_TOKEN, so creating the branch fires no workflow. Only that exact name is C: +# the rolling tag refs/tags/nightly, or any other ref whose name merely ends in the night's name, is never read as C. # # Return codes: 0 ok; 1 refused (RED); 2 not_measured (could not read or write the remote); 3 caller error. # # np_night [EPOCH] print the night of EPOCH (default now) -# np_pick REPO REMOTE NIGHT SHA create refs/nightly/NIGHT -> SHA unless it exists; prints "C=" + PICKED|KEPT +# np_pick REPO REMOTE NIGHT SHA create refs/heads/nightly/NIGHT -> SHA unless it exists; prints "C=" + PICKED|KEPT # np_resolve REPO REMOTE NIGHT print the night's C; 2 when the night has no pick # np_verify REPO REMOTE NIGHT SHA # 0 when SHA is the night's C, 1 when it is not, 2 when the night has no pick @@ -31,15 +35,15 @@ np__args() { # NIGHT [SHA] np__read() { # REPO REMOTE NIGHT -> the ref's sha on stdout; 0 found, 1 absent, 2 unreadable local out rc=0 - out="$(git -C "$1" ls-remote --refs "$2" "refs/nightly/$3" 2>/dev/null)" || rc=$? + out="$(git -C "$1" ls-remote --refs "$2" "refs/heads/nightly/$3" 2>/dev/null)" || rc=$? [ "$rc" -eq 0 ] || return 2 # exact name only. ls-remote matches a pattern by its tail, and so does a push destination, so another ref ending in - # refs/nightly/N (a branch refs/heads/refs/nightly/N) would be read as C, and would swallow the pick's push. + # refs/heads/nightly/N (a branch refs/heads/refs/heads/nightly/N) would be read as C, and would swallow the pick's push. # Such a name makes the night ambiguous: unreadable (2), never a C. - if printf '%s\n' "$out" | awk -v r="refs/nightly/$3" 'NF && $2 != r { found = 1 } END { exit !found }'; then - printf 'NP: refs/nightly/%s is ambiguous on the remote (another ref ends in that name)\n' "$3" >&2; return 2 + if printf '%s\n' "$out" | awk -v r="refs/heads/nightly/$3" 'NF && $2 != r { found = 1 } END { exit !found }'; then + printf 'NP: refs/heads/nightly/%s is ambiguous on the remote (another ref ends in that name)\n' "$3" >&2; return 2 fi - out="$(printf '%s\n' "$out" | awk -v r="refs/nightly/$3" '$2 == r { print $1; exit }')" + out="$(printf '%s\n' "$out" | awk -v r="refs/heads/nightly/$3" '$2 == r { print $1; exit }')" [ -n "$out" ] || return 1 printf '%s\n' "$out" } @@ -51,7 +55,7 @@ np_resolve() { # REPO REMOTE NIGHT case "$rc" in 0) printf '%s\n' "$c" ;; 1) printf 'NP NOT_MEASURED: night %s has no pick\n' "$3" >&2; return 2 ;; - *) printf 'NP NOT_MEASURED: cannot read refs/nightly/%s\n' "$3" >&2; return 2 ;; + *) printf 'NP NOT_MEASURED: cannot read refs/heads/nightly/%s\n' "$3" >&2; return 2 ;; esac } @@ -72,7 +76,7 @@ np_pick() { # REPO REMOTE NIGHT SHA if [ "$rc" -eq 0 ]; then printf 'C=%s KEPT night %s (picked earlier; main now %s does not move it)\n' "$c" "$3" "${4:0:10}"; return 0 fi - [ "$rc" -eq 1 ] || { printf 'NP NOT_MEASURED: cannot read refs/nightly/%s; nothing picked\n' "$3"; return 2; } + [ "$rc" -eq 1 ] || { printf 'NP NOT_MEASURED: cannot read refs/heads/nightly/%s; nothing picked\n' "$3"; return 2; } # C must be on main: a dispatch from a branch must never become the night git -C "$1" fetch -q "$2" main 2>/dev/null || { printf 'NP NOT_MEASURED: cannot fetch main; nothing picked\n'; return 2; } rc=0; git -C "$1" merge-base --is-ancestor "$4" FETCH_HEAD 2>/dev/null || rc=$? @@ -82,10 +86,10 @@ np_pick() { # REPO REMOTE NIGHT SHA *) printf 'NP NOT_MEASURED: cannot tell whether %s is on main (merge-base rc=%s); nothing picked\n' "${4:0:10}" "$rc"; return 2 ;; esac # create-only: the lease with an empty expected value accepts the push only while the ref does not exist. A plain - # push is NOT create-only outside refs/heads and refs/tags: it fast-forwards an existing ref, and main only moves + # push is NOT create-only: it fast-forwards an existing branch, and main only moves # forward, so a racing later pick would move the night's C. - rc=0; git -C "$1" push -q --force-with-lease="refs/nightly/$3:" "$2" "$4:refs/nightly/$3" 2>/dev/null || rc=$? - c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } + rc=0; git -C "$1" push -q --force-with-lease="refs/heads/nightly/$3:" "$2" "$4:refs/heads/nightly/$3" 2>/dev/null || rc=$? + c="$(np__read "$1" "$2" "$3")" || { printf 'NP NOT_MEASURED: refs/heads/nightly/%s absent or unreadable after the push (push rc=%s)\n' "$3" "$rc"; return 2; } if [ "$c" = "$4" ]; then printf 'C=%s PICKED night %s\n' "$c" "$3"; return 0; fi printf 'C=%s KEPT night %s (another pick won the race)\n' "$c" "$3"; return 0 } diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index 1f7c537d83..82d7a4074f 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# nightly_pick.sh — pick the night's candidate C once and publish it as refs/nightly/. +# nightly_pick.sh — pick the night's candidate C once and publish it as refs/heads/nightly/. # # The rule (operator): one job picks C once a night and publishes it as a ref; every producer measures that C; the # train prints its line for that C; a release promotes that C. Planted falsifier: a merge to main after C is picked @@ -61,7 +61,7 @@ self_test() { row night_turns_at_noon 0 "2026-10-05" "" -- bash "$SCRIPT_PATH" night --at 1791201600 # 2026-10-05T12:00Z row unpicked_night_is_not_measured 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night "$n" row first_pick_publishes_the_ref 0 "C=$c1 PICKED night $n" "" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "$n" --sha "$c1" - row the_ref_is_on_the_remote 0 "$c1" "" -- git -C "$W" ls-remote "$FX/origin.git" "refs/nightly/$n" + row the_ref_is_on_the_remote 0 "$c1" "" -- git -C "$W" ls-remote "$FX/origin.git" "refs/heads/nightly/$n" l1="$(train_line "$W" "$n")"; m1="$(main_line "$W" "$FX/origin.git")" # THE PLANT: a merge to main after the pick c2="$(commit "$W" merged-after-the-pick)" && git -C "$W" push -q origin main || caller_error "fixture push" @@ -79,7 +79,7 @@ self_test() { git -C "$W" checkout -q -b side && br="$(commit "$W" branch-only)" && git -C "$W" push -q origin side && git -C "$W" checkout -q main row a_commit_not_on_main_is_refused 1 "is not on main" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-06 --sha "$br" row a_refused_pick_publishes_no_ref 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-06 - c3="$(commit "$W" racer)" && git -C "$W" push -q origin main && git -C "$W" push -q origin "$c3:refs/nightly/2026-10-07" + c3="$(commit "$W" racer)" && git -C "$W" push -q origin main && git -C "$W" push -q origin "$c3:refs/heads/nightly/2026-10-07" row a_pick_never_overwrites_an_existing_ref 0 "C=$c3 KEPT night 2026-10-07" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-07 --sha "$c2" # a race: the ref is absent when read and created by another picker before the push lands. race_shim DIR NIGHT # WINNER puts a git on PATH whose first ls-remote sees nothing and whose push first creates the ref at WINNER. @@ -87,7 +87,7 @@ self_test() { mkdir -p "$1" && printf '%s\n' '#!/usr/bin/env bash' \ "case \" \$* \" in" \ " *' ls-remote '*) [ -f '$1/seen' ] || { : > '$1/seen'; exit 0; } ;;" \ - " *' push '*) '$(command -v git)' -C '$FX/origin.git' update-ref refs/nightly/$2 '$3' ;;" \ + " *' push '*) '$(command -v git)' -C '$FX/origin.git' update-ref refs/heads/nightly/$2 '$3' ;;" \ "esac" "exec '$(command -v git)' \"\$@\"" > "$1/git" && chmod +x "$1/git" } race_shim "$FX/bin" 2026-10-10 "$c3" @@ -97,10 +97,18 @@ self_test() { row a_lost_race_to_an_older_commit_keeps_the_winner 0 "C=$c1 KEPT night 2026-10-11 (another pick won the race)" "PICKED" -- env PATH="$FX/bin-ff:$PATH" bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-11 --sha "$c3" row a_lost_race_leaves_the_ref_at_the_winner 0 "$c1" "$c3" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-11 row an_unknown_commit_is_not_measured 2 "cannot tell whether" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-12 --sha 0123456789abcdef0123456789abcdef01234567 - # a decoy: a BRANCH named refs/nightly/ must not be read as the night's ref (ls-remote matches by tail) - git -C "$W" push -q origin "$br:refs/heads/refs/nightly/2026-10-13" || caller_error "fixture decoy push" + # a decoy: a BRANCH named refs/heads/nightly/ must not be read as the night's ref (ls-remote matches by tail) + git -C "$W" push -q origin "$br:refs/heads/refs/heads/nightly/2026-10-13" || caller_error "fixture decoy push" row a_decoy_branch_is_not_the_night 2 "is ambiguous" "C=" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-13 --sha "$c3" row a_decoy_branch_does_not_resolve 2 "is ambiguous" "$br" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-13 + # the rolling tag refs/tags/nightly (as on the real remote) is never C: only the branch is + git -C "$W" push -q origin "$br:refs/tags/nightly" || caller_error "fixture tag push" + row the_rolling_tag_is_not_a_pick 2 "has no pick" "$br" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-14 + row a_pick_beside_the_rolling_tag_picks_the_branch 0 "C=$c3 PICKED night 2026-10-14" "KEPT" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-14 --sha "$c3" + row the_rolling_tag_never_resolves 0 "$c3" "$br" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night 2026-10-14 + # a branch named nightly blocks every nightly/ (a directory/file clash): not_measured, never a C + git init -q --bare -b main "$FX/df.git" && git -C "$W" push -q "$FX/df.git" main "$c3:refs/heads/nightly" || caller_error "fixture df remote" + row a_branch_named_nightly_is_not_measured 2 "NOT_MEASURED" "C=" -- bash "$SCRIPT_PATH" pick --repo "$W" --remote "$FX/df.git" --night 2026-10-15 --sha "$c3" row unreachable_remote_is_not_measured 2 "NOT_MEASURED" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --remote "$FX/absent.git" --night 2026-10-08 --sha "$c2" row unreachable_remote_resolve_is_not_measured 2 "cannot read" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --remote "$FX/absent.git" --night "$n" row malformed_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-1-4 --sha "$c1" @@ -114,7 +122,7 @@ self_test() { # ------------------------------------------------------------------------------------ the mutants ---- # namesed expression on the lib. Each must change the lib and turn the case table RED. -MUTANTS='m01_pick_moves_the_ref s/--force-with-lease="refs\/nightly\/\$3:"/-f/ +MUTANTS='m01_pick_moves_the_ref s/--force-with-lease="refs\/heads\/nightly\/\$3:"/-f/ m02_existing_ref_ignored s/ if \[ "\$rc" -eq 0 \]; then$/ if false; then/ m03_branch_commit_accepted s/merge-base --is-ancestor "\$4" FETCH_HEAD 2>\/dev\/null ||/true ||/ m04_verify_accepts_any_sha s/if \[ "\$c" = "\$4" \]; then printf .NP OK/if true; then printf '"'"'NP OK/ @@ -124,7 +132,7 @@ m07_night_is_the_fire_date s/\$((e - 43200))/$e/ m08_bad_night_accepted s/grep -qxE .\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}. || {/true || {/ m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ -m11_plain_push_fast_forwards s/ --force-with-lease="refs\/nightly\/\$3:"// +m11_plain_push_fast_forwards s/ --force-with-lease="refs\/heads\/nightly\/\$3:"// m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/' mutants() { From 92d49938732150e1468b93faf9cc18e40f9ed1b0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 14:24:30 +0200 Subject: [PATCH 05/34] fix(release): nightly pick review round 4 minors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rolling-tag rows held only as a regression guard: no mutant read the tag. Mutant m14 widens the read so refs/tags/nightly is seen, and the case table turns RED (17 rows). The workflow header no longer says "never forced": the push is create-only because its lease expects the ref absent. Receipts: --self-test 34/34 green; --mutants killed=14 total=14 errors=0; pv validate valid; bashrs 0 errors. ont-delta: none — one more mutant named in the falsifier's prediction; no new term or relation Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-pick.yml | 2 +- contracts/nightly-pick-v1.yaml | 2 +- scripts/release/nightly_pick.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml index 4fe39c0791..a75f0c94e0 100644 --- a/.github/workflows/nightly-pick.yml +++ b/.github/workflows/nightly-pick.yml @@ -1,7 +1,7 @@ # Nightly pick: ONE job picks the night's candidate C and publishes it as refs/heads/nightly/. # # Every producer measures that C, the nightly train prints its line for that C, and a release -# promotes that C. The ref is created once and never moved (create-only push, never forced), so a +# promotes that C. The ref is created once and never moved (a create-only push: its lease expects the ref absent), so a # merge to main after the pick cannot change the night's line. Contract: contracts/nightly-pick-v1.yaml. # # WHY A PICK, NOT CRON SPACING. Scheduled events here fire 1h43m to 5h48m late (14 nights measured), diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index dfca694788..6a847538e2 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -87,7 +87,7 @@ falsification_tests: - id: FALSIFY-NP-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, a read widened so the rolling tag refs/tags/nightly is seen, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" test: "bash scripts/release/nightly_pick.sh --mutants" if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index 82d7a4074f..ee9a3c6848 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -134,7 +134,8 @@ m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ m11_plain_push_fast_forwards s/ --force-with-lease="refs\/heads\/nightly\/\$3:"// m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ -m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/' +m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/ +m14_read_widened_to_the_rolling_tag s/ls-remote --refs "\$2" "refs\/heads\/nightly\/\$3"/ls-remote --refs "\$2" "nightly"/' mutants() { local tmp name expr killed=0 total=0 errors=0 out tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" From a5a72b908fb54a5cc022b045e1d4fd4e6af27e89 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 17:49:37 +0200 Subject: [PATCH 06/34] T44: a producer can switch its checkout to the night's C scripts/release/nightly_c_checkout.sh resolves the night from the triggering pick's start time, fetches nightly/, detaches onto C, re-verifies HEAD against the ref, and exports NIGHTLY_C / NIGHTLY_NIGHT. No pick is not_measured (2); a tree that cannot switch or a ref that moved is RED (1). --self-test 22/22 rows, --mutants 9/9 killed. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/nightly_c_checkout.sh | 206 ++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 scripts/release/nightly_c_checkout.sh diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh new file mode 100755 index 0000000000..6c721787cf --- /dev/null +++ b/scripts/release/nightly_c_checkout.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# nightly_c_checkout.sh — put a nightly producer's work tree on the night's C, the commit the pick published. +# +# A producer starts on the "Nightly pick" workflow_run, so actions/checkout hands it main's head at the moment the +# event fired, not C: a merge that lands while the pick runs, or a rerun of the pick, moves main away from the ref. +# This step reads refs/heads/nightly/ (scripts/lib/nightly_pick.sh), switches the tree to that commit when it +# is not already there, and then asks np_verify whether HEAD is C. A producer that measures any other commit is RED. +# +# The night comes from --at, the pick run's start (github.event.workflow_run.run_started_at), never from the clock +# here: a producer queued for hours still measures the night it was started for. +# +# Usage: +# nightly_c_checkout.sh [checkout] [--repo DIR] [--remote NAME] --at ISO8601|EPOCH +# nightly_c_checkout.sh --self-test | --mutants +# On success it appends NIGHTLY_C and NIGHTLY_NIGHT to $GITHUB_ENV and one line to $GITHUB_STEP_SUMMARY, when set. +# Exit: 0 HEAD is C; 1 RED (HEAD is not C, or the tree could not be switched); 2 not_measured (no pick, or the remote +# could not be read); 3 caller error. +# +# Everything runs inside functions, called on the last line: the checkout replaces this file under a running bash. +set -uo pipefail + +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_PATH="$HERE/$(basename -- "${BASH_SOURCE[0]}")" +LIB="$HERE/../lib/nightly_pick.sh" + +caller_error() { printf 'nightly_c_checkout: caller error: %s\n' "$1" >&2; exit 3; } + +to_epoch() { # ISO8601|EPOCH -> epoch + case "$1" in + '') return 3 ;; + *[!0-9]*) date -u -d "$1" +%s 2>/dev/null || return 3 ;; + *) printf '%s\n' "$1" ;; + esac +} + +checkout_c() { # REPO REMOTE AT + local repo="$1" remote="$2" epoch night c head rc=0 how + epoch="$(to_epoch "$3")" || { printf 'NC caller error: --at %s is not a time\n' "$3" >&2; return 3; } + night="$(np_night "$epoch")" || return 3 + c="$(np_resolve "$repo" "$remote" "$night")" || return $? + head="$(git -C "$repo" rev-parse HEAD 2>/dev/null)" || { printf 'NC RED: %s is not a work tree\n' "$repo"; return 1; } + if [ "$head" != "$c" ]; then + git -C "$repo" fetch -q --no-tags "$remote" "+refs/heads/nightly/$night:refs/remotes/$remote/nightly/$night" 2>/dev/null \ + || { printf 'NC NOT_MEASURED: cannot fetch refs/heads/nightly/%s\n' "$night"; return 2; } + git -C "$repo" -c advice.detachedHead=false checkout -q --detach "$c" 2>/dev/null \ + || { printf 'NC RED: cannot switch the tree from %s to C=%s\n' "${head:0:10}" "${c:0:10}"; return 1; } + how="switched from ${head:0:10}" + else + how="already C" + fi + np_verify "$repo" "$remote" "$night" "$(git -C "$repo" rev-parse HEAD)" || { rc=$?; return "$rc"; } + if [ -n "${GITHUB_ENV:-}" ]; then + printf 'NIGHTLY_C=%s\nNIGHTLY_NIGHT=%s\n' "$c" "$night" >> "$GITHUB_ENV" || { printf 'NC NOT_MEASURED: cannot write GITHUB_ENV\n'; return 2; } + fi + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf 'Measured C=`%s` of night %s (%s).\n' "$c" "$night" "$how" >> "$GITHUB_STEP_SUMMARY" || true + fi + printf 'NIGHTLY C=%s night=%s (%s)\n' "$c" "$night" "$how" +} + +# ------------------------------------------------------------------------------------ the case table ---- +CASES=0; FAILED=0 +row() { # NAME WANT-RC MUST MUSTNOT -- CMD... + local name="$1" want="$2" must="$3" mustnot="$4" out rc + shift 5 + out="$("$@" 2>&1)"; rc=$? + CASES=$((CASES + 1)) + if [ "$rc" != "$want" ]; then printf 'RED %-44s rc=%s want %s\n' "$name" "$rc" "$want"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$mustnot" ] && printf '%s\n' "$out" | grep -qF -e "$mustnot"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi + printf 'ok %s\n' "$name" +} +same() { # NAME A B : two values must be equal + CASES=$((CASES + 1)) + if [ -n "$2" ] && [ "$2" = "$3" ]; then printf 'ok %s\n' "$1"; else printf 'RED %-44s %s != %s\n' "$1" "$2" "$3"; FAILED=$((FAILED + 1)); fi +} +commit() { # REPO MSG -> new head sha; the file f holds MSG, so two commits differ in a tracked file + printf "%s\n" "$2" > "$1/f" && git -C "$1" add f && git -C "$1" -c core.hooksPath=/dev/null -c user.name=fixture -c user.email=fixture@example.invalid commit -q -m "$2" && git -C "$1" rev-parse HEAD +} + +self_test() { + local FX A W S c1 c2 n=2026-10-04 at=2026-10-04T20:31:07Z env + FX="$(mktemp -d "${TMPDIR:-/tmp}/nc-st.XXXXXX")" || caller_error "no temp dir" + A="$FX/author"; W="$FX/w"; S="$FX/shallow" + git init -q --bare -b main "$FX/origin.git" && git clone -q "$FX/origin.git" "$A" 2>/dev/null || caller_error "no fixture repo" + git -C "$A" checkout -q -b main 2>/dev/null + c1="$(commit "$A" one)" && git -C "$A" push -q origin main || caller_error "fixture push" + git -C "$A" push -q origin "$c1:refs/heads/nightly/$n" || caller_error "fixture pick" + c2="$(commit "$A" two)" && commit "$A" three >/dev/null && git -C "$A" push -q origin main || caller_error "fixture merge" + git clone -q "$FX/origin.git" "$W" 2>/dev/null && git clone -q --depth 1 "file://$FX/origin.git" "$S" 2>/dev/null || caller_error "fixture clones" + + # the defect: a producer that checked out main's head and measured it + row main_head_is_switched_to_c 0 "switched from" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + same the_tree_is_on_c_after_a_merge "$(git -C "$W" rev-parse HEAD)" "$c1" + row a_second_run_finds_c_already 0 "already C" "switched" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + row a_shallow_checkout_reaches_c 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" + same the_shallow_tree_is_on_c "$(git -C "$S" rev-parse HEAD)" "$c1" + # a late start still measures the night it was started for: 05:48Z the next day is still night N + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" + row a_late_start_is_still_the_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T05:48:00Z + # the night comes from --at, not from the clock: an epoch the day after has no pick + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" + row an_unpicked_night_is_not_measured 2 "has no pick" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791230400 + same not_measured_leaves_the_tree "$(git -C "$W" rev-parse HEAD)" "$c2" + row epoch_at_is_accepted 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791147600 + # a tree that cannot be switched is RED, never a measurement of the wrong commit + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" + printf 'local edit\n' > "$W/f" + row an_unswitchable_tree_is_red 1 "cannot switch" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + same the_unswitchable_tree_is_left_alone "$(git -C "$W" rev-parse HEAD)" "$c2" + git -C "$W" checkout -q -- f + # GITHUB_ENV carries C to every later step; it is written only once HEAD is C + env="$FX/github_env"; : > "$env" + row env_is_written 0 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" + row env_holds_c 0 "NIGHTLY_C=$c1" "" -- cat "$env" + row env_holds_the_night 0 "NIGHTLY_NIGHT=$n" "" -- cat "$env" + : > "$env" + row no_env_when_not_measured 2 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at 1791230400 + same not_measured_writes_no_env "$(wc -c < "$env" | tr -d ' ')x" "0x" + # a ref that moves while the step runs (a forced re-pick; the ruleset forbids it, the fixture plants it with a + # post-checkout hook): the commit just checked out is no longer C, so the step is RED and C is never exported + mkdir -p "$FX/hooks" + printf '#!/bin/sh\ngit push -q -f "%s" "%s:refs/heads/nightly/%s" 2>/dev/null\n' "$FX/origin.git" "$c2" "$n" > "$FX/hooks/post-checkout" + chmod +x "$FX/hooks/post-checkout" + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2"; : > "$env" + row a_ref_moved_mid_step_is_red 1 "is not the C" "NIGHTLY C=" -- env GITHUB_ENV="$env" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0="$FX/hooks" bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + same a_moved_ref_exports_nothing "$(wc -c < "$env" | tr -d ' ')x" "0x" + git -C "$A" push -q -f origin "$c1:refs/heads/nightly/$n" || caller_error "fixture restore" + row unreachable_remote_is_not_measured 2 "cannot read" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --remote "$FX/absent.git" --at "$at" + row missing_at_is_a_caller_error 3 "--at" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" + row bad_at_is_a_caller_error 3 "not a time" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at yesterday-ish + row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" measure + rm -rf -- "${FX:?}" + printf '%s %s/%s case rows green\n' "$([ "$FAILED" -eq 0 ] && echo SELF-TEST-GREEN || echo SELF-TEST-RED)" "$((CASES - FAILED))" "$CASES" + [ "$FAILED" -eq 0 ] +} + +# ------------------------------------------------------------------------------------ the mutants ---- +# namesed expression on the code above the case table (never on this table itself). Each must change that code +# and turn at least one case row RED; a mutant that breaks the script without a RED row is an error, not a kill. +MUTANTS='m01_main_head_measured s/ if \[ "\$head" != "\$c" \]; then$/ if false; then/ +m02_verify_skipped s/ np_verify "\$repo" "\$remote" "\$night" "\$(git -C "\$repo" rev-parse HEAD)" || { rc=\$?; return "\$rc"; }/ true/ +m03_env_not_written s/printf .NIGHTLY_C=%s\\nNIGHTLY_NIGHT=%s\\n. "\$c" "\$night" >> "\$GITHUB_ENV"/true/ +m04_no_pick_is_ok s/ c="\$(np_resolve "\$repo" "\$remote" "\$night")" || return \$?/ c="$(np_resolve "$repo" "$remote" "$night")" || return 0/ +m05_night_from_the_clock s/ night="\$(np_night "\$epoch")" || return 3/ night="$(np_night)" || return 3/ +m06_fetches_main s/"+refs\/heads\/nightly\/\$night:refs\/remotes\/\$remote\/nightly\/\$night"/main/ +m07_switch_failure_ignored s/|| { printf .NC RED: cannot switch/|| true || { printf '"'"'NC RED: cannot switch/ +m08_bad_at_accepted s/date -u -d "\$1" +%s 2>\/dev\/null || return 3/date -u +%s/ +m09_env_before_verify /^ np_verify "\$repo"/{h;d};/^ printf .NIGHTLY C=%s night=%s/{x;G}' +mutants() { + local tmp name expr killed=0 total=0 errors=0 out cut reds + cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" + [ -n "$cut" ] || caller_error "no case-table marker" + tmp="$(mktemp -d "${TMPDIR:-/tmp}/nc-mu.XXXXXX")" || caller_error "no temp dir" + mkdir -p "$tmp/release" "$tmp/lib" + cp "$LIB" "$tmp/lib/nightly_pick.sh" || caller_error "cannot copy the lib" + while IFS="$(printf '\t')" read -r name expr; do + [ -n "$name" ] || continue + total=$((total + 1)) + { head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/release/nightly_c_checkout.sh" + if cmp -s "$SCRIPT_PATH" "$tmp/release/nightly_c_checkout.sh"; then + printf 'ERROR %-32s the patch did not apply (an error, never a survivor)\n' "$name"; errors=$((errors + 1)); continue + fi + if out="$(bash "$tmp/release/nightly_c_checkout.sh" --self-test 2>&1)"; then + printf 'SURVIVED %s\n' "$name" + else + reds="$(printf '%s\n' "$out" | grep -c -e '^RED ')" + if [ "$reds" -eq 0 ]; then + printf 'ERROR %-32s failed with no RED row (a broken script, not a kill)\n' "$name"; errors=$((errors + 1)); continue + fi + killed=$((killed + 1)) + printf 'killed %-32s %s\n' "$name" "$reds" + fi + done <<< "$MUTANTS" + rm -rf -- "${tmp:?}" + printf 'MUTANTS killed=%s total=%s errors=%s\n' "$killed" "$total" "$errors" + [ "$killed" -eq "$total" ] && [ "$errors" -eq 0 ] +} + +# ------------------------------------------------------------------------------------ main ---- +main() { + local cmd repo=. remote=origin at='' + [ -f "$LIB" ] || caller_error "no library at $LIB" + # shellcheck source=../lib/nightly_pick.sh + . "$LIB" || caller_error "cannot source $LIB" + cmd="${1:-}"; [ "$#" -gt 0 ] && shift + case "$cmd" in + --self-test) self_test; return $? ;; + --mutants) mutants; return $? ;; + -h | --help) sed -n '2,20p' "$SCRIPT_PATH"; return 0 ;; + checkout) ;; + --repo | --remote | --at) set -- "$cmd" "$@" ;; + *) caller_error "unknown command '$cmd' (checkout|--self-test|--mutants)" ;; + esac + while [ "$#" -gt 0 ]; do + case "$1" in + --repo) repo="${2:-}"; shift 2 || caller_error "--repo DIR" ;; + --remote) remote="${2:-}"; shift 2 || caller_error "--remote NAME" ;; + --at) at="${2:-}"; shift 2 || caller_error "--at ISO8601|EPOCH" ;; + *) caller_error "unknown option '$1'" ;; + esac + done + [ -n "$at" ] || caller_error "--at ISO8601|EPOCH (the pick run's start) is required" + checkout_c "$repo" "$remote" "$at" +} +main "$@"; exit $? From 36dea70c34bbb8b11a279d35fc84f16718439471 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 17:55:36 +0200 Subject: [PATCH 07/34] T44: the nightly producers chain from "Nightly pick" and measure its C Each of the 15 producers loses its schedule and gains a workflow_run on "Nightly pick" (completed, main). After every checkout of this repo, a step runs scripts/release/nightly_c_checkout.sh --at , so the producer measures the night's C pinned at nightly/, never main's head at whatever hour GitHub created a scheduled run. A failed pick leaves no C, so the step exits 2 (not_measured) and the producer run is red. Gates that read the trigger accept the chain: - check_coverage_has_producers.sh R3: a producer may be scheduled OR chained (workflow_run, workflows: ["Nightly pick"], types: completed); 6 new rows, including another workflow, two workflows, a requested type, and a commented-out chain, which stay RED. - check_silicon_coverage.sh counts workflow_run runs next to schedule and workflow_dispatch; 3 new rows (chained, switch-over inside the lookback, real run chained) are RED on the old guard, green here. Known limits: mutants-nightly's base is the last success's head_sha (main at trigger), so the commits between C and that sha are skipped once; cuda-nightly's yield marker uses the pick's head_sha, which equals C unless the nightly ref already existed. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/beat-speed-nightly.yml | 23 ++++++------- .github/workflows/book-contracts.yml | 41 ++++++++++++++++++++++-- .github/workflows/book.yml | 16 +++++++-- .github/workflows/conleche-nightly.yml | 16 ++++++--- .github/workflows/coverage-nightly.yml | 23 ++++++------- .github/workflows/cuda-nightly.yml | 30 ++++++++++------- .github/workflows/examples-nightly.yml | 23 ++++++------- .github/workflows/guards-nightly.yml | 32 +++++++++--------- .github/workflows/install-script.yml | 16 +++++++-- .github/workflows/mutants-nightly.yml | 14 ++++++-- .github/workflows/nightly-bench.yml | 23 ++++++------- .github/workflows/nightly.yml | 38 +++++++++++++++------- .github/workflows/qwen-story-daily.yml | 24 +++++++------- .github/workflows/silicon-nightly.yml | 33 ++++++++++++------- .github/workflows/toolchain-ceiling.yml | 33 ++++++++++++------- scripts/check_coverage_has_producers.sh | 38 +++++++++++++++++++--- scripts/check_silicon_coverage.sh | 39 ++++++++++++++++++---- 17 files changed, 318 insertions(+), 144 deletions(-) diff --git a/.github/workflows/beat-speed-nightly.yml b/.github/workflows/beat-speed-nightly.yml index eb80311551..859dedfaa2 100644 --- a/.github/workflows/beat-speed-nightly.yml +++ b/.github/workflows/beat-speed-nightly.yml @@ -31,17 +31,13 @@ name: BEAT Speed (nightly) on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:45 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:45 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "45 0 * * *" # 00:45 UTC -> lands ~05:45 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: jobs: @@ -79,6 +75,11 @@ jobs: BEAT_LOG: ${{ github.workspace }}/beat-speed-measurements.log steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # PERF-031 measurement-environment receipt. # diff --git a/.github/workflows/book-contracts.yml b/.github/workflows/book-contracts.yml index b28d4404fe..c219687747 100644 --- a/.github/workflows/book-contracts.yml +++ b/.github/workflows/book-contracts.yml @@ -35,9 +35,14 @@ on: - ".github/workflows/book-contracts.yml" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '45 22 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] # #3676: no concurrency group meant every superseded PR push ran the ~18 min @@ -62,6 +67,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -106,6 +116,11 @@ jobs: needs: chapter-examples-compile steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -157,6 +172,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -189,6 +209,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Every chapter has a contract run: | FAIL=0 @@ -249,6 +274,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: No legacy imports in chapter examples run: | FAIL=0 @@ -290,6 +320,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: No dead links in SUMMARY.md run: | FAIL=0 diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index f695c6ed4e..fdfd85cb07 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -51,9 +51,14 @@ on: - "crates/apr-cli/src/tool_commands.rs" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '30 22 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] permissions: contents: read @@ -75,6 +80,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # Self-hosted (#3073): tools go into this run's private $RUNNER_TEMP, never a $HOME path that every # runner on the box execs from (the aprender#2353 race; check_cargo_install_private_root.sh enforces diff --git a/.github/workflows/conleche-nightly.yml b/.github/workflows/conleche-nightly.yml index 63285936a7..369d2a4dc4 100644 --- a/.github/workflows/conleche-nightly.yml +++ b/.github/workflows/conleche-nightly.yml @@ -21,10 +21,13 @@ name: conleche-nightly on: - schedule: - # 21:23 UTC -> lands ~02:23 UTC. GitHub dispatches schedules ~5h late on this account (#3292); the slot is - # free and clear of guards-nightly (21:47) and cuda-nightly (20:30). - - cron: '23 21 * * *' + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: concurrency: @@ -50,6 +53,11 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Verdict case table (the classifier this job trusts) run: bash crates/aprender-contracts-staging/lean/conleche.sh --self-test - name: Pinned elan in the job's own directory (sha256-checked, never a host install) diff --git a/.github/workflows/coverage-nightly.yml b/.github/workflows/coverage-nightly.yml index 939972122b..7159351c9e 100644 --- a/.github/workflows/coverage-nightly.yml +++ b/.github/workflows/coverage-nightly.yml @@ -21,17 +21,13 @@ name: Coverage Nightly # measurement ever worked. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 22 * * *' # 22:00 UTC -> lands ~03:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} # #3676: no v* tag trigger here. The release tag's coverage is owned by ci.yml # (`coverage_on: tag`, which sovereign-ci's gate requires on a tag push), so a tag @@ -109,6 +105,11 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # Provision cargo-llvm-cov WITHOUT touching ~/.cargo/bin (#2353). # diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index 53dafd909d..94280ccf1a 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -44,17 +44,13 @@ name: CUDA Nightly (GPU QLoRA falsifiers) # without deleting the workflow. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 01:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 01:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "30 20 * * *" # 20:30 UTC -> lands ~01:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: silicon: @@ -165,7 +161,7 @@ jobs: # NOT_RUN is not PASS. mkdir -p "$PERF041_OUT" printf '{"host":"%s","status":"NOT_RUN","reason":"yield to training (%s MiB, %s procs)","started_utc":"%s","commit":"%s","exit":null,"max_m":null,"cc":null,"sha256":null}\n' \ - "${{ matrix.name }}" "${used:-?}" "${procs:-?}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${{ github.sha }}" \ + "${{ matrix.name }}" "${used:-?}" "${procs:-?}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${{ github.event.workflow_run.head_sha || github.sha }}" \ > "$PERF041_OUT/marker.json" echo "proceed=false" >> "$GITHUB_OUTPUT"; exit 0 fi @@ -186,6 +182,11 @@ jobs: - name: Preflight — this box has the tools this job assumes if: steps.decide.outputs.proceed == 'true' run: bash scripts/ci_self_hosted_preflight.sh --cuda + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' && (steps.decide.outputs.proceed == 'true') + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # PP-26's release-phase marker gate (scripts/check_perf041_marker.sh, a # dogfood gate) may not be named in a REQUIRED workflow @@ -797,6 +798,11 @@ jobs: CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: flock -E 175 -w 600 /run/lock/fleet-gpu/gpu.lock steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Put the runner's Rust toolchain on PATH run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 2adaaed942..0db25b56d3 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -9,17 +9,13 @@ name: examples-nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '30 22 * * *' # 22:30 UTC -> lands ~03:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: filter: @@ -41,6 +37,11 @@ jobs: timeout-minutes: 300 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: dogfood_examples.sh case table (hermetic, scratch workspace) run: bash scripts/dogfood_examples.sh --selftest diff --git a/.github/workflows/guards-nightly.yml b/.github/workflows/guards-nightly.yml index 9e045048e1..4fd6895d67 100644 --- a/.github/workflows/guards-nightly.yml +++ b/.github/workflows/guards-nightly.yml @@ -37,20 +37,13 @@ name: guards-nightly on: - schedule: - # 02:47Z -- an off-minute, and a slot no other nightly holds: cuda-nightly - # 01:30 (10 min), silicon-nightly 03:30, qwen-story-daily 04:17. A run of - # this job measured ~27 min of steps, so it lands clear of all three. - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 02:47 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 02:47 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '47 21 * * *' # 21:47 UTC -> lands ~02:47 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: # One at a time. Two of these jobs would share $GUARD_CARGO_HOME for the same @@ -99,6 +92,11 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # THE MANIFEST IS THE JOIN KEY, AND IT IS CHECKED BEFORE ANYTHING RUNS. # # A step that leaves ci.yml and never arrives here is a guard that was @@ -218,7 +216,7 @@ jobs: # export-ignore, so an ignored source file would vanish from the # comparand and lower the measured count (review m2-prb-review, 4/5 lanes). comparand="$RUNNER_TEMP/comparand"; git worktree remove --force "$comparand" 2>/dev/null || rm -rf "$comparand" - git worktree add --detach "$comparand" origin/main + git worktree add --detach "$comparand" "${NIGHTLY_C:-origin/main}" trap 'git worktree remove --force "$comparand" 2>/dev/null || true' EXIT DR_ENV=(-e SATD_BASELINE=0) set +e @@ -227,7 +225,7 @@ jobs: n=$(printf '%s\n' "$out" | sed -n 's/.*BROKEN -- \([0-9][0-9]*\) markers in production source.*/\1/p' | head -1) if [ "$mrc" -eq 0 ] && [ -z "$n" ] && printf '%s\n' "$out" | grep -q '^test result: ok'; then n=0; fi case "$n" in ''|*[!0-9]*) printf '%s\n' "$out" | tail -20; echo "FAIL: the comparand measurement did not yield a count (rc=$mrc)"; exit 1 ;; esac - echo "SATD_BASELINE=$n measured on origin/main $(git rev-parse --short origin/main) by the HEAD-built suite" + echo "SATD_BASELINE=$n measured on ${NIGHTLY_C:+nightly C }$(git rev-parse --short "${NIGHTLY_C:-origin/main}") by the HEAD-built suite" # 3. the suites, judged against the measured ceiling DR_ENV=(-e "SATD_BASELINE=$n") dr "$GITHUB_WORKSPACE" cargo test -p aprender-core --features model-tests \ @@ -246,7 +244,7 @@ jobs: printf 'budget (PMAT-1098, row 67-E3). Manifest: `scripts/guards_nightly_manifest.txt`.\n\n' printf '| fact | value |\n|---|---|\n' printf '| ref | `%s` |\n' "${GITHUB_REF_NAME}" - printf '| commit | `%s` |\n' "${GITHUB_SHA}" + printf '| commit | `%s` |\n' "${NIGHTLY_C:-$GITHUB_SHA}" printf '| steps moved here | %s |\n' "$(grep -cvE '^[[:space:]]*(#|$)' scripts/guards_nightly_manifest.txt)" printf '| result | %s |\n' "${JOB_STATUS}" printf '\nA red run here is a defect that reached main: the PR path no longer\n' diff --git a/.github/workflows/install-script.yml b/.github/workflows/install-script.yml index f8daec95c5..05490e9559 100644 --- a/.github/workflows/install-script.yml +++ b/.github/workflows/install-script.yml @@ -33,9 +33,14 @@ on: - ".github/workflows/nightly.yml" # #3676 rule 3 (scripts/check_workflow_path_filters.sh): a path filter is a # claim that nothing outside these paths can break this gate; the nightly run - # re-checks the claim every day. Set ~5h early for the dispatch delay (#3292). - schedule: - - cron: '0 23 * * *' + # re-checks the claim every night, on the night's C. + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] # #3676: a newer push to the same PR cancels the superseded run @@ -52,6 +57,11 @@ jobs: runs-on: [self-hosted, Linux, X64, clean-room] steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # pmat/bashrs are pinned-fleet tools, never installed per-job (BSE-10a, # PMAT-1066) — see ci.yml's identical step for why. diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index a5d12a8c98..55dfdf6b97 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -7,8 +7,13 @@ name: Mutants Nightly # names itself. on: - schedule: - - cron: '0 1 * * *' # 01:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} permissions: @@ -39,6 +44,11 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 0 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Everything merged since the last nightly that JUDGED main, as a diff id: day # Not a rolling "24 hours ago" window: that has no memory, so one missed or failed run would drop a day of diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 71b75c67a2..2db179bc42 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -26,17 +26,13 @@ name: Nightly Bench on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: "30 0 * * *" # 00:30 UTC -> lands ~05:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -64,6 +60,11 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # The receipt validator's own case table must discriminate before any # number it blesses means anything. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1f34b0e156..837618d827 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -36,17 +36,13 @@ name: Nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 04:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 04:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 23 * * *' # 23:00 UTC -> lands ~04:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -78,6 +74,11 @@ jobs: uses: actions/checkout@v7 with: ref: main + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Case table run: python3 scripts/nightly_manifest.py --self-test @@ -119,6 +120,11 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ needs.gate.outputs.sha }} + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # A sibling container through the mounted socket, as in binary-release.yml's # CPU lanes: every bind-mount path must exist on the HOST (the work root is @@ -255,6 +261,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Build apr (host-native, pinned toolchain) run: | @@ -297,6 +308,11 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ needs.gate.outputs.sha }} + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # A runner that died uploaded nothing; merge records that arch red, so # an empty download must not stop the other arch from publishing. diff --git a/.github/workflows/qwen-story-daily.yml b/.github/workflows/qwen-story-daily.yml index 84c9561a07..8681b427c5 100644 --- a/.github/workflows/qwen-story-daily.yml +++ b/.github/workflows/qwen-story-daily.yml @@ -13,18 +13,13 @@ name: qwen-story-daily on: - schedule: - # 04:17 UTC daily — off-peak for the self-hosted fleet. - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 04:17 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 04:17 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '17 23 * * *' # 23:17 UTC -> lands ~04:17 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: pmat_hunt: @@ -79,6 +74,11 @@ jobs: - uses: actions/checkout@v7 with: fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Install apr from HEAD shell: bash diff --git a/.github/workflows/silicon-nightly.yml b/.github/workflows/silicon-nightly.yml index ae5ed579b9..2e955b291f 100644 --- a/.github/workflows/silicon-nightly.yml +++ b/.github/workflows/silicon-nightly.yml @@ -21,17 +21,13 @@ name: Silicon Nightly on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 03:30 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 03:30 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '30 22 * * *' # 22:30 UTC -> lands ~03:30 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: inputs: axis: @@ -56,6 +52,11 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" # Probe the instrument before believing it. The label matcher must classify # six committed cases — including a runner whose labels are a strict SUBSET @@ -110,6 +111,11 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH @@ -151,6 +157,11 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH diff --git a/.github/workflows/toolchain-ceiling.yml b/.github/workflows/toolchain-ceiling.yml index e400cd22f9..22c6528c42 100644 --- a/.github/workflows/toolchain-ceiling.yml +++ b/.github/workflows/toolchain-ceiling.yml @@ -23,17 +23,13 @@ name: Toolchain Ceiling # clippy component, broken version comparator) — see its header. on: - schedule: - # SHIFTED so it actually runs at night (#3292). GitHub dispatches - # schedule events 3h51m-6h49m late on this account (28 runs, 10 - # workflows, 3 nights, zero on time), so a 05:00 cron fired mid- - # morning and the nightly tier occupied 06:49-14:14 UTC — the - # working day, competing with PR traffic for the same runners. - # Cron is set ~5h EARLY so the delayed dispatch lands near 05:00 UTC. - # This is CALIBRATION AGAINST A MOVING TARGET: the delay varies by - # ~3h and is growing. The durable fix is a fleet-side timer calling - # `gh workflow run` (#3292); until then, re-measure before trusting. - - cron: '0 0 * * *' # 00:00 UTC -> lands ~05:00 UTC + # Chained, not scheduled (T44, #4686): this producer measures the night's + # C that "Nightly pick" pinned at nightly/, never main's head at + # whatever hour GitHub got round to creating a scheduled run. + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] workflow_dispatch: {} permissions: @@ -64,6 +60,11 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v7 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Workspace builds on its DECLARED MSRV run: setsid --wait bash scripts/check_msrv.sh @@ -78,6 +79,11 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Clippy on current stable (ceiling gate) run: | @@ -102,6 +108,11 @@ jobs: - uses: actions/checkout@v7 with: clean: false + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" - name: Self-test — an unreached feature is refused, and a planted cuda-only finding reddens only the cuda-reaching axes run: | export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" diff --git a/scripts/check_coverage_has_producers.sh b/scripts/check_coverage_has_producers.sh index daf1f06208..b4b2dbfd78 100644 --- a/scripts/check_coverage_has_producers.sh +++ b/scripts/check_coverage_has_producers.sh @@ -12,8 +12,11 @@ # pre-publish dogfood uses measures coverage itself, reads nothing from CI. # R2 scripts/dogfood.sh still runs `make ... coverage-check` -- the release # CONSUMER exists (without it the chain would be vacuously intact). -# R3 .github/workflows/coverage-nightly.yml triggers on `schedule:` and NOT on -# `v*` tags -- the nightly trend stays, and a tag is measured ONCE, by ci.yml. +# R3 .github/workflows/coverage-nightly.yml has a NIGHTLY trigger and NOT `v*` tags -- +# the nightly trend stays, and a tag is measured ONCE, by ci.yml. Nightly is +# `schedule:`, or (T44, #4686) `workflow_run:` on the "Nightly pick" workflow, +# `types: [completed]` -- the chain that measures the night's C. A workflow_run +# on any other workflow is not nightly. # R4 the Makefile defines a numeric COV_FLOOR -- the floor `make coverage` enforces. # R5 ci/sections.yml (#4433) hands sovereign-ci `coverage_on: tag` (no `skip_coverage: true`) # AND its own `on:` has `push: tags: ['v*']` -- the reusable's two required @@ -36,6 +39,17 @@ on_block() { } # live -> the file with comments stripped (a setting in a comment is not a setting) live() { sed 's/#.*$//' "$1"; } +# nightly_trigger -> 0 when the block has `schedule:`, or a `workflow_run:` whose +# `workflows:` names exactly "Nightly pick" and whose `types:` includes completed (T44). +nightly_trigger() { + grep -qE '^[[:space:]]+schedule:' <<<"$1" && return 0 + awk ' + /^[[:space:]]+workflow_run:/ { match($0, /^ */); wi = RLENGTH; f = 1; next } + f && /[^[:space:]]/ { match($0, /^ */); if (RLENGTH <= wi) f = 0 } + f && /^[[:space:]]+workflows:[[:space:]]*\[[[:space:]]*["\047]?Nightly pick["\047]?[[:space:]]*\]/ { w = 1 } + f && /^[[:space:]]+types:[[:space:]]*\[.*completed/ { t = 1 } + END { exit !(w && t) }' <<<"$1" +} V_TAGS_RE="^[[:space:]]+tags:[[:space:]]*\[[^]]*['\"]?v\*['\"]?" judge() { # judge -> 0 all links hold, 1 a link broke, 2 ENV @@ -67,11 +81,11 @@ judge() { # judge -> 0 all links hold, 1 a link broke, 2 ENV else printf 'FAIL R2 scripts/dogfood.sh no longer runs coverage-check -- nothing checks coverage at the release\n'; bad=1; fi blk=$(on_block "$wf") - if ! grep -qE '^[[:space:]]+schedule:' <<<"$blk"; then - printf 'FAIL R3 coverage-nightly.yml must trigger on schedule (on: block, comments ignored)\n'; bad=1 + if ! nightly_trigger "$blk"; then + printf 'FAIL R3 coverage-nightly.yml must trigger nightly: schedule, or workflow_run on "Nightly pick" completed (on: block, comments ignored)\n'; bad=1 elif grep -qE "$V_TAGS_RE" <<<"$blk"; then printf 'FAIL R3 coverage-nightly.yml also triggers on v* tags -- ci.yml owns tag coverage (R5); a tag would be measured twice\n'; bad=1 - else printf 'ok R3 coverage-nightly.yml triggers on schedule and not on v* tags\n'; fi + else printf 'ok R3 coverage-nightly.yml triggers nightly and not on v* tags\n'; fi if grep -qE '^COV_FLOOR[[:space:]]*:?=[[:space:]]*[0-9]+' "$mk"; then printf 'ok R4 the Makefile defines a numeric COV_FLOOR\n' @@ -114,6 +128,14 @@ if [ "${1:-}" = "--self-test" ]; then m_r2_comment() { printf '#!/usr/bin/env bash\n# gate coverage make -C x coverage-check\n' > scripts/dogfood.sh; } m_r3_nosched() { sed -i '/schedule:/d; /cron:/d' .github/workflows/coverage-nightly.yml; } m_r3_tagsback() { printf "on:\n schedule:\n - cron: '0 22 * * *'\n push:\n tags: ['v*']\njobs: {}\n" > .github/workflows/coverage-nightly.yml; } + # T44 (#4686): the chained form, as the producers carry it, and its near misses. + chain() { printf "on:\n workflow_run:\n workflows: [%s]\n types: [%s]\n branches: [main]\n workflow_dispatch: {}\njobs: {}\n" "$1" "$2" > .github/workflows/coverage-nightly.yml; } + m_r3_chain() { chain '"Nightly pick"' completed; } + m_r3_chain_sq() { chain "'Nightly pick'" completed; } + m_r3_chain_other(){ chain '"Nightly"' completed; } + m_r3_chain_two() { chain '"Nightly pick", "CI"' completed; } + m_r3_chain_req() { chain '"Nightly pick"' requested; } + m_r3_chain_cmt() { m_r3_chain; sed -i 's/^ workflows:/ # workflows:/' .github/workflows/coverage-nightly.yml; } m_r4() { sed -i '/^COV_FLOOR/d' Makefile; } m_r5_skip() { sed -i 's/coverage_on: tag/skip_coverage: true/' ci/sections.yml; } m_r5_both() { printf ' skip_coverage: true\n' >> ci/sections.yml; } @@ -138,6 +160,12 @@ if [ "${1:-}" = "--self-test" ]; then row 1 "R2: the call only in a COMMENT -> RED" m_r2_comment row 1 "R3: nightly schedule removed -> RED" m_r3_nosched row 1 "R3: the nightly's v* tag trigger back (a tag measured twice) -> RED" m_r3_tagsback + row 0 "R3: workflow_run on \"Nightly pick\" completed (T44 chain) -> PASS" m_r3_chain + row 0 "R3: the chain, single-quoted -> PASS" m_r3_chain_sq + row 1 "R3: workflow_run on another workflow -> RED" m_r3_chain_other + row 1 "R3: workflow_run on Nightly pick AND another -> RED" m_r3_chain_two + row 1 "R3: workflow_run on Nightly pick, types [requested] -> RED" m_r3_chain_req + row 1 "R3: the chain's workflows: only in a COMMENT -> RED" m_r3_chain_cmt row 1 "R4: COV_FLOOR removed -> RED" m_r4 row 1 "R5: skip_coverage: true instead of coverage_on (the #3688 red) -> RED" m_r5_skip row 1 "R5: skip_coverage: true alongside coverage_on -> RED" m_r5_both diff --git a/scripts/check_silicon_coverage.sh b/scripts/check_silicon_coverage.sh index e251eb03a5..51778faa05 100755 --- a/scripts/check_silicon_coverage.sh +++ b/scripts/check_silicon_coverage.sh @@ -376,7 +376,9 @@ build_jobs() { _bj_since="$(epoch_day "$_bj_cut")" || return 1 _bj_runs="$(mktemp)" || return 1 for _bj_wf in "$@"; do - for _bj_ev in schedule workflow_dispatch; do + # workflow_run: the nightlies chain from "Nightly pick" (T44, #4686); + # schedule stays for the runs from before the chain, inside the lookback. + for _bj_ev in schedule workflow_run workflow_dispatch; do _bj_one="$(mktemp)" || return 1 list_window "$_bj_wf" "$_bj_ev" "$_bj_one" || { rm -f "${_bj_one:?}" "${_bj_runs:?}"; return 1; } # CROSS-CHECK EVERY LISTING (PMAT-3337 §8, widened). The API can hand @@ -599,7 +601,7 @@ st_page() { st_finish() { _sf_since="$(epoch_day $(( ST_NOW - 30 * 86400 )))" _sf_since_e="$(date -u -d "$_sf_since" +%s)" # bashrs disable-line=DET002 - for _sf_ev in schedule workflow_dispatch; do + for _sf_ev in schedule workflow_run workflow_dispatch; do _sf_key="repos/$ST_REPO/actions/runs?event=${_sf_ev}&per_page=100" if [ "$ST_ERROR" = 1 ]; then st_route "$_sf_key" @ERROR; continue; fi awk -F'\t' -v ev="$_sf_ev" '$3 == ev && $6 == 1' "$ST/u/runs.tsv" \ @@ -608,7 +610,7 @@ st_finish() { st_route "$_sf_key" "pages/global-$_sf_ev.json" done for _sf_wf in sil.yml gpu.yml noise.yml silicon-nightly.yml; do - for _sf_ev in schedule workflow_dispatch; do + for _sf_ev in schedule workflow_run workflow_dispatch; do awk -F'\t' -v wf="$_sf_wf" -v ev="$_sf_ev" -v s="$_sf_since_e" \ '$2 == wf && $3 == ev && $5 >= s' "$ST/u/runs.tsv" \ | sort -t"$TAB" -k5,5nr > "$ST/u/l.tsv" @@ -708,11 +710,11 @@ st_real() { # st_real_nightlies : one Silicon Nightly a day, newest # first. The newest is run 36364433328 with its four jobs as the API returned # them (2026-09-28): names, conclusions and runs-on labels verbatim. -st_real_nightlies() { - _rn_h="$1"; _rn_i=0 +st_real_nightlies() { # args: hours before now, count, event (default schedule) + _rn_h="$1"; _rn_i=0; _rn_ev="${3:-schedule}" while [ "$_rn_i" -lt "$2" ]; do _rn_id=$(( 36364433328 - _rn_i )) - st_run "$_rn_id" silicon-nightly.yml schedule $(( _rn_h + 24 * _rn_i )) 1 \ + st_run "$_rn_id" silicon-nightly.yml "$_rn_ev" $(( _rn_h + 24 * _rn_i )) 1 \ "x86_64-cpu|success|self-hosted,clean-room,intel" \ "coverage|success|self-hosted,clean-room,intel" \ "aarch64-cuda-sm121|success|self-hosted,gpu,gx10,cuda,blackwell" \ @@ -818,6 +820,31 @@ selftest_rows() { st_check j-short-always 2 '^ NO-GO +x86_64-cpu +REQUIRED: inconsistent listing .*listed 4 of 4.*total 13' \ || _rows_bad=$((_rows_bad + 1)) + # (k) T44 (#4686): the producers run on workflow_run from "Nightly pick", no + # longer on schedule. Runs of the chain are evidence like scheduled runs; a + # guard that lists only schedule / workflow_dispatch reads every chained + # night as no run at all. + st_reset + st_run 9001 sil.yml workflow_run 12 1 "$CPU_JOB"; st_run 9101 gpu.yml workflow_run 20 1 "$GPU_JOB" + st_finish + _rows=$((_rows + 1)) + st_check k-chained 0 '^ ok +gpu-axis ' || _rows_bad=$((_rows_bad + 1)) + + # (l) the switch-over night: the last scheduled run is stale (8 days), the + # chained run is fresh. The newest run of EITHER event is the evidence. + st_reset + st_run 9001 sil.yml schedule 192 1 "$CPU_JOB"; st_run 9002 sil.yml workflow_run 12 1 "$CPU_JOB" + st_run 9101 gpu.yml schedule 200 1 "$GPU_JOB"; st_run 9102 gpu.yml workflow_run 20 1 "$GPU_JOB" + st_finish + _rows=$((_rows + 1)) + st_check l-switch-over 0 '^ ok +gpu-axis ' || _rows_bad=$((_rows_bad + 1)) + + # (m) the real silicon-nightly jobs, chained: both real axes GREEN. + st_reset; st_real || return 2; st_real_nightlies 10 13 workflow_run; st_finish + _rows=$((_rows + 1)) + st_check m-real-chained 0 '^ ok +aarch64-cuda-sm121 +success .*Silicon Nightly / aarch64-cuda-sm121' \ + || _rows_bad=$((_rows_bad + 1)) + # (e) the API errors: refuse, exactly as before. st_reset; ST_ERROR=1 st_run 9001 sil.yml schedule 12 1 "$CPU_JOB"; st_run 9101 gpu.yml schedule 20 1 "$GPU_JOB" From fc543ca11b95689836e2833f407bfc14a2ee1cf0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 18:03:10 +0200 Subject: [PATCH 08/34] T44: a guard that every nightly producer chains from the pick and measures C scripts/release/check_producers_chain.sh reads the workflow text and is RED when a listed producer still has a schedule (P1), lacks the workflow_run on the name nightly-pick.yml declares, completed, main (P2), checks this repo out without the C step next, after an optional Preflight (P3; the C step's if must be exactly the workflow_run test, or that test && (the checkout's own if)), names github.sha/GITHUB_SHA without NIGHTLY_C or workflow_run.head_sha (P4), or when the list and the chained set differ either way (P5). Self-test 28 rows on fixtures copied from the real producers; mutants 16/16 killed, 0 errors. The real tree PASSes. Not wired into CI here: a new gate is held for sign-off. Limits: it reads the workflow, not the scripts a step calls; a declared origin/main comparand (guards-nightly's SATD ceiling) is not judged. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/check_producers_chain.sh | 292 +++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 scripts/release/check_producers_chain.sh diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh new file mode 100644 index 0000000000..a4e64ae2ab --- /dev/null +++ b/scripts/release/check_producers_chain.sh @@ -0,0 +1,292 @@ +#!/usr/bin/env bash +# check_producers_chain.sh — every nightly producer chains from "Nightly pick" and measures the night's C (#4686). +# +# A producer that starts from its own cron runs whenever GitHub gets round to creating the run (2–6.5h late on +# 09-26..10-03), and measures main's head at that hour: two producers of one night can measure two commits. T44 chains +# them instead: each one starts on the pick's workflow_run and, after every checkout, switches the tree to the commit +# the pick pinned at nightly/ (scripts/release/nightly_c_checkout.sh). This guard reads the workflow text and +# is RED when a producer: +# P1 still has a `schedule:` trigger (a live line in its on: block); +# P2 lacks the chain: workflow_run with workflows: [], types: [completed], +# branches: [main]; +# P3 checks this repo out (actions/checkout without `repository:`) and the next step, after an optional +# "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"`, +# env NIGHTLY_PICK_AT from github.event.workflow_run.run_started_at, and if exactly +# `github.event_name == 'workflow_run'`, or `github.event_name == 'workflow_run' && ()`; +# P4 names github.sha or GITHUB_SHA on a live line that names neither NIGHTLY_C nor workflow_run.head_sha; +# P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways). +# Limits: it reads the workflow text, not the scripts a step calls; an origin/main a step fetches as a declared +# comparand (guards-nightly's SATD ceiling) is not a measurement and is not judged here. +# +# Usage: +# check_producers_chain.sh [--root DIR] judge the tree (default: the repo this file is in) +# check_producers_chain.sh --self-test | --mutants +# Exit: 0 every producer chained; 1 RED; 3 caller error. +set -uo pipefail + +HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT_PATH="$HERE/$(basename -- "${BASH_SOURCE[0]}")" + +# The nightly producers: the workflows whose results are read as "the night's" measurement. +PRODUCERS="beat-speed-nightly book book-contracts conleche-nightly coverage-nightly cuda-nightly examples-nightly +guards-nightly install-script mutants-nightly nightly nightly-bench qwen-story-daily silicon-nightly toolchain-ceiling" + +caller_error() { printf 'check_producers_chain: caller error: %s\n' "$1" >&2; exit 3; } + +# judge_file FILE PICK LISTED -> FAIL lines on stdout; "CHAINED" when the file chains from PICK +judge_file() { + awk -v pick="$2" -v listed="$3" -v f="$(basename -- "$1")" ' + function ind(s) { match(s, /^ */); return RLENGTH } + function live(s) { return s !~ /^[[:space:]]*(#|$)/ } + function bare(s) { sub(/^[^:]*:[[:space:]]*/, "", s); sub(/[[:space:]]+$/, "", s); return s } + function list1(s) { s = bare(s); if (s !~ /^\[.*\]$/) return "\001"; s = substr(s, 2, length(s) - 2) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); if (s ~ /^".*"$/ || s ~ /^\047.*\047$/) s = substr(s, 2, length(s) - 2) + return s } + function fail(m) { printf "FAIL %s: %s\n", f, m; bad = 1 } + function key(st, k, a, m, i, p) { + m = split(st, a, "\n"); p = sprintf("%" (ind(a[1]) + 2) "s", "") + if (a[1] ~ ("^ *- " k ":")) return bare(a[1]) + for (i = 2; i <= m; i++) if (index(a[i], p k ":") == 1) return bare(a[i]) + return "\001" + } + function hasline(st, want, a, m, i, s) { + m = split(st, a, "\n") + for (i = 1; i <= m; i++) { s = a[i]; gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); if (s == want) return 1 } + return 0 + } + function flush( k, j, cif, want, nm) { + if (!listed) { n = 0; ii = -1; return } # only a listed producer owes the C step + for (k = 1; k <= n; k++) { + if (key(S[k], "uses") !~ /^actions\/checkout@/ || S[k] ~ /\n[[:space:]]*repository:/) continue + checkouts++ + j = k + 1; nm = key(S[j], "name") + if (j <= n && nm ~ /^Preflight/) j++ + cif = key(S[k], "if") + want = (cif == "\001") ? "github.event_name == \047workflow_run\047" \ + : "github.event_name == \047workflow_run\047 && (" cif ")" + if (j > n) { fail("P3 checkout in job step " k " is not followed by the C step (end of steps)"); continue } + if (key(S[j], "run") != "bash scripts/release/nightly_c_checkout.sh --at \"$NIGHTLY_PICK_AT\"") { + fail("P3 checkout in job step " k " is not followed by the C step (step " j " runs something else)"); continue } + if (!hasline(S[j], "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}")) + fail("P3 the C step after job step " k " does not take NIGHTLY_PICK_AT from workflow_run.run_started_at") + if (key(S[j], "if") != want) + fail("P3 the C step after job step " k " has if: " key(S[j], "if") " -- want: " want) + } + n = 0; ii = -1 + } + { L = $0 } + # on: block + /^on:[[:space:]]*$/ { inon = 1; next } + inon && live(L) && ind(L) == 0 { inon = 0; wr = 0 } + inon && live(L) { + if (L ~ /^ schedule:/) sched = 1 + if (L ~ /^ workflow_run:/) { wr = 1; next } + if (ind(L) <= 2) wr = 0 + if (wr && L ~ /^ workflows:/ && list1(L) == pick) cw = 1 + if (wr && L ~ /^ types:/ && list1(L) == "completed") ct = 1 + if (wr && L ~ /^ branches:/ && list1(L) == "main") cb = 1 + } + # sha rule, every live line + live(L) && L ~ /github\.sha|GITHUB_SHA/ && L !~ /NIGHTLY_C|workflow_run\.head_sha/ { + shaline[++ns] = NR } + # steps + insteps && live(L) && ind(L) <= si { flush(); insteps = 0 } + insteps && live(L) { + if (L ~ /^ *- / && (ii < 0 || ind(L) == ii)) { ii = ind(L); S[++n] = L } + else if (n > 0) S[n] = S[n] "\n" L + next + } + /^ *steps:[[:space:]]*$/ && live(L) { insteps = 1; si = ind(L); n = 0; ii = -1; next } + END { + if (insteps) flush() + chained = (cw && ct && cb) + if (!listed) { if (cw) printf "FAIL %s: P5 chains from \"%s\" but is not in the producer list\n", f, pick; exit } + if (sched) fail("P1 still has a schedule: trigger -- a producer starts on the pick, never on its own cron") + if (!chained) fail("P2 no workflow_run on [\"" pick "\"], types [completed], branches [main]" \ + " (workflows=" cw " types=" ct " branches=" cb ")") + for (i = 1; i <= ns; i++) fail("P4 line " shaline[i] " names github.sha/GITHUB_SHA -- main at trigger, not C") + if (!bad) printf "ok %s: chained from \"%s\", %d checkout(s), each followed by the C step\n", f, pick, checkouts + }' "$1" +} + +judge() { # ROOT + local root="$1" wf pick p rc=0 out listed + wf="$root/.github/workflows" + [ -f "$wf/nightly-pick.yml" ] || { printf 'FAIL nightly-pick.yml is missing: there is no pick to chain from\n'; return 1; } + pick="$(sed -n -e 's/^name:[[:space:]]*//p' "$wf/nightly-pick.yml" | head -n 1 | sed -e 's/[[:space:]]*$//' -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/")" + [ -n "$pick" ] || { printf 'FAIL nightly-pick.yml declares no name:\n'; return 1; } + for p in $PRODUCERS; do + [ -f "$wf/$p.yml" ] || { printf 'FAIL %s.yml: P5 listed producer is missing\n' "$p"; rc=1; } + done + for f in "$wf"/*.yml; do + [ -f "$f" ] || continue + p="$(basename -- "$f" .yml)" + [ "$p" = nightly-pick ] && continue + listed=0 + case " $(printf '%s' "$PRODUCERS" | tr '\n' ' ') " in *" $p "*) listed=1 ;; esac + out="$(judge_file "$f" "$pick" "$listed")" + [ -n "$out" ] && printf '%s\n' "$out" + printf '%s\n' "$out" | grep -q -e '^FAIL' && rc=1 + done + if [ "$rc" -eq 0 ]; then printf 'PASS every nightly producer chains from "%s" and measures its C\n' "$pick" + else printf 'RED a nightly producer is not chained to the pick, or does not measure its C\n'; fi + return "$rc" +} + +# ------------------------------------------------------------------------------------ the case table ---- +CASES=0; FAILED=0 +REAL_ROOT="$(cd -- "$HERE/../.." && pwd)" +fixture() { # DIR: the real producers + nightly-pick.yml + rm -rf -- "${1:?}"; mkdir -p "$1/.github/workflows" || return 1 + local p + for p in $PRODUCERS nightly-pick; do cp "$REAL_ROOT/.github/workflows/$p.yml" "$1/.github/workflows/" || return 1; done +} +row() { # NAME WANT-RC MUST -- MUTATION-FN (run inside the fixture's workflows dir) + local name="$1" want="$2" must="$3" fx out rc + shift 4 + fx="$TMP_ST/fx" + fixture "$fx" || { printf 'RED %-40s fixture failed\n' "$name"; FAILED=$((FAILED + 1)); CASES=$((CASES + 1)); return; } + (cd "$fx/.github/workflows" && "$@") || { printf 'RED %-40s mutation did not apply\n' "$name"; FAILED=$((FAILED + 1)); CASES=$((CASES + 1)); return; } + out="$(judge "$fx" 2>&1)"; rc=$? + CASES=$((CASES + 1)) + if [ "$rc" != "$want" ]; then printf 'RED %-40s rc=%s want %s\n%s\n' "$name" "$rc" "$want" "$(printf '%s\n' "$out" | grep -e '^FAIL' | head -n 3)"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-40s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + printf 'ok %s\n' "$name" +} +changed() { # FILE SED-EXPR: apply, and fail when nothing changed (a no-op mutation is an error, never a pass) + local before; before="$(cat "$1")"; sed -i -e "$2" "$1" || return 1; [ "$before" != "$(cat "$1")" ] +} +CSTEP='bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"' +synth() { # FILE: a minimal chained producer, built by the caller-supplied steps on stdin + { printf 'name: Synth\non:\n workflow_run:\n workflows: ["Nightly pick"]\n types: [completed]\n branches: [main]\n workflow_dispatch:\n' + printf 'jobs:\n a:\n runs-on: x\n steps:\n'; cat; } > "$1" +} +cstep() { # [IF]: the C step text, with the checkout's if when given + local cif="github.event_name == 'workflow_run'"; [ -n "${1:-}" ] && cif="$cif && ($1)" + printf ' - name: Measure the night'"'"'s C, not main'"'"'s head (T44)\n if: %s\n env:\n NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}\n run: %s\n' "$cif" "$CSTEP" +} +m_none() { :; } +m_sched_back() { changed nightly.yml 's/^ workflow_run:$/ schedule:\n - cron: "0 3 * * *"\n workflow_run:/'; } +m_sched_comment() { changed nightly.yml 's/^ workflow_run:$/ # schedule:\n # - cron: "0 3 * * *"\n workflow_run:/'; } +m_chain_other() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly"]/'; } +m_chain_two() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly pick", "CI"]/'; } +m_chain_type() { changed book.yml '0,/types: \[completed\]/s//types: [requested]/'; } +m_chain_branch() { changed cuda-nightly.yml '/^ branches: \[main\]$/d'; } +m_chain_branch2() { changed cuda-nightly.yml 's/^ branches: \[main\]$/ branches: [release]/'; } +m_chain_cmt() { changed cuda-nightly.yml 's/^ workflows:/ # workflows:/'; } +m_pick_renamed() { changed nightly-pick.yml 's/^name: Nightly pick$/name: Nightly Pick/'; } +m_missing() { rm -f -- silicon-nightly.yml; } +m_unlisted() { synth extra.yml <<< ' - run: true'; } +m_no_cstep() { changed toolchain-ceiling.yml '/- name: Measure the night.s C, not main.s head (T44)/,/nightly_c_checkout.sh/d'; } +m_cstep_or() { changed install-script.yml "0,/if: github.event_name == 'workflow_run'\$/s//if: github.event_name == 'workflow_run' || true/"; } +m_cstep_ifdrop() { changed cuda-nightly.yml "s/if: github.event_name == 'workflow_run' && (.*)\$/if: github.event_name == 'workflow_run'/"; } +m_cstep_noat() { changed nightly-bench.yml '0,/nightly_c_checkout.sh --at "\$NIGHTLY_PICK_AT"/s//nightly_c_checkout.sh/'; } +m_cstep_wrongat() { changed coverage-nightly.yml '0,/workflow_run.run_started_at/s//workflow_run.head_commit.timestamp/'; } +m_step_between() { changed examples-nightly.yml '0,/- name: Measure the night.s C/s//- run: make bench\n - name: Measure the night'"'"'s C/'; } +m_repo_exempt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/other\n')"; } +m_preflight_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - name: Preflight — tools\n run: true\n'; cstep)"; } +m_flow_if_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep 'a || b')"; } +m_flow_if_bare() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep | sed -e "s/'workflow_run'\$/'workflow_run' \&\& a || b/")"; } +m_last_step() { synth nightly.yml <<< ' - uses: actions/checkout@v7'; } +m_sha_plain() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ github.sha }}"/'; } +m_sha_env() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: git diff "$GITHUB_SHA"/'; } +m_sha_comment() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n # measured at $GITHUB_SHA, before T44/'; } +m_sha_c() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: echo "${NIGHTLY_C:-$GITHUB_SHA}"/'; } +m_bystander() { printf 'name: Other\non:\n schedule:\n - cron: "0 1 * * *"\njobs:\n a:\n runs-on: x\n steps:\n - uses: actions/checkout@v7\n - run: echo "$GITHUB_SHA"\n' > other.yml; } + +self_test() { + TMP_ST="$(mktemp -d "${TMPDIR:-/tmp}/pc-st.XXXXXX")" || caller_error "no temp dir" + echo "=== nightly producers chain: each rule must turn RED when its link breaks ===" + row base_real_producers 0 'PASS' -- m_none + row p1_schedule_back 1 'P1 still has a schedule' -- m_sched_back + row p1_schedule_commented 0 'PASS' -- m_sched_comment + row p2_other_workflow 1 'book.yml: P2' -- m_chain_other + row p2_two_workflows 1 'book.yml: P2' -- m_chain_two + row p2_requested_type 1 'book.yml: P2' -- m_chain_type + row p2_no_branches 1 'cuda-nightly.yml: P2' -- m_chain_branch + row p2_other_branch 1 'cuda-nightly.yml: P2' -- m_chain_branch2 + row p2_commented_workflows 1 'cuda-nightly.yml: P2' -- m_chain_cmt + row p2_pick_renamed 1 'P2 no workflow_run on ["Nightly Pick"]' -- m_pick_renamed + row p5_missing_producer 1 'silicon-nightly.yml: P5 listed' -- m_missing + row p5_unlisted_chain 1 'extra.yml: P5 chains' -- m_unlisted + row p3_no_cstep 1 'toolchain-ceiling.yml: P3' -- m_no_cstep + row p3_if_or_true 1 'install-script.yml: P3' -- m_cstep_or + row p3_if_drops_checkout_if 1 'cuda-nightly.yml: P3' -- m_cstep_ifdrop + row p3_no_at 1 'nightly-bench.yml: P3' -- m_cstep_noat + row p3_at_from_head_commit 1 'coverage-nightly.yml: P3 the C step' -- m_cstep_wrongat + row p3_step_between 1 'examples-nightly.yml: P3' -- m_step_between + row p3_last_step_checkout 1 'end of steps' -- m_last_step + row p3_other_repo_exempt 0 'PASS' -- m_repo_exempt + row p3_preflight_between 0 'nightly.yml: chained' -- m_preflight_ok + row p3_checkout_if_with_or 0 'nightly.yml: chained' -- m_flow_if_ok + row p3_unparenthesised_or 1 'nightly.yml: P3' -- m_flow_if_bare + row p4_github_sha 1 'qwen-story-daily.yml: P4' -- m_sha_plain + row p4_GITHUB_SHA 1 'beat-speed-nightly.yml: P4' -- m_sha_env + row p4_comment_ignored 0 'PASS' -- m_sha_comment + row p4_nightly_c_fallback 0 'PASS' -- m_sha_c + row p5_unlisted_scheduled_ok 0 'PASS' -- m_bystander + rm -rf -- "${TMP_ST:?}" + if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi + printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 +} + +MUTANTS='m01_no_schedule_rule s/if (sched) fail/if (0) fail/ +m02_any_branch s/list1(L) == "main") cb = 1/1) cb = 1/ +m03_any_type s/list1(L) == "completed") ct = 1/1) ct = 1/ +m04_any_workflow s/list1(L) == pick) cw = 1/1) cw = 1/ +m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ +m06_no_repo_exempt s/ || S\[k\] ~ \/\\n\[\[:space:\]\]\*repository:\/) continue/) continue/ +m07_any_if s/if (key(S\[j\], "if") != want)/if (0)/ +m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ +m09_no_sha_rule s/&& L !~ \/NIGHTLY_C|workflow_run\\.head_sha\/ {/\&\& 0 {/ +m10_sha_no_exemption s/&& L !~ \/NIGHTLY_C|workflow_run\\.head_sha\/ {/{/ +m11_no_missing_rule /P5 listed producer is missing/s/; rc=1; }/; }/ +m12_no_unlisted_rule s/if (cw) printf "FAIL/if (0) printf "FAIL/ +m13_comments_live s/function live(s) { return s !~ \/^\[\[:space:\]\]\*(#|\$)\/ }/function live(s) { return s !~ \/^[[:space:]]*$\/ }/ +m14_end_of_steps_ok s/if (j > n) { fail(/if (0) { fail(/ +m15_any_run s/if (key(S\[j\], "run") != /if (0 \&\& key(S[j], "run") != / +m16_judge_bystanders /only a listed producer owes the C step/d' +mutants() { + local tmp name expr killed=0 total=0 errors=0 out cut reds + cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" + [ -n "$cut" ] || caller_error "no case-table marker" + tmp="$(mktemp -d "${TMPDIR:-/tmp}/pc-mu.XXXXXX")" || caller_error "no temp dir" + mkdir -p "$tmp/scripts/release" "$tmp/.github/workflows" + cp "$REAL_ROOT"/.github/workflows/*.yml "$tmp/.github/workflows/" || caller_error "cannot copy the workflows" + while IFS="$(printf '\t')" read -r name expr; do + [ -n "$name" ] || continue + total=$((total + 1)) + { head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/scripts/release/check_producers_chain.sh" + if cmp -s "$SCRIPT_PATH" "$tmp/scripts/release/check_producers_chain.sh"; then + printf 'ERROR %-28s the patch did not apply (an error, never a survivor)\n' "$name"; errors=$((errors + 1)); continue + fi + if out="$(bash "$tmp/scripts/release/check_producers_chain.sh" --self-test 2>&1)"; then + printf 'SURVIVED %s\n' "$name" + else + reds="$(printf '%s\n' "$out" | grep -c -e '^RED ')" + if [ "$reds" -eq 0 ]; then + printf 'ERROR %-28s failed with no RED row (a broken script, not a kill)\n' "$name"; errors=$((errors + 1)); continue + fi + killed=$((killed + 1)) + printf 'killed %-28s %s\n' "$name" "$reds" + fi + done <<< "$MUTANTS" + rm -rf -- "${tmp:?}" + printf 'MUTANTS killed=%s total=%s errors=%s\n' "$killed" "$total" "$errors" + [ "$killed" -eq "$total" ] && [ "$errors" -eq 0 ] +} + +# ------------------------------------------------------------------------------------ main ---- +main() { + local root="$REAL_ROOT" + case "${1:-}" in + --self-test) self_test; return $? ;; + --mutants) mutants; return $? ;; + -h | --help) sed -n '2,26p' "$SCRIPT_PATH"; return 0 ;; + --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; + '') ;; + *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; + esac + judge "$root" +} +main "$@"; exit $? From a6fa93bbf9f50b8ddc6f6f1caea35d69e83af809 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 18:08:21 +0200 Subject: [PATCH 09/34] T44: rule 3 of the path-filter check moves into awk and accepts the chain Rule 3 (a path-filtered workflow must re-check itself every night) is now judged by scripts/lib/workflow_runs_nightly.awk, not by the SCHEDULE line of workflow_path_filters.py. That line is still printed and nothing reads it. It is deleted in a follow-up, because the pre-commit complexity gate rejects any commit that touches the .py (cognitive 44 at base, 40 with the deletion, limit 25). A nightly trigger is a schedule with a non-empty cron, or a workflow_run on "Nightly pick" with types naming completed. Without the second form, the producers that chain from the pick would turn the check RED. Evidence: - scripts/tests/workflow_runs_nightly_test.sh: 22-row table PASS, mutants killed 9/9, errors 0. - check_workflow_path_filters.sh self-test gains wf7 (chain, passes), wf8 (chain from another workflow, fails) and wf9 (commented chain, fails). - scripts/tests/workflow_path_filters_rule3_parity.sh runs the old rule 3 (from --base) and the new one over the same fixtures plus every head and base workflow. The only allowed difference is old RED -> new green on the chained workflows. - not_measured locally: both sides of the parity run and the check's self-test need python3 with yaml. They are CI-only and not wired into CI here. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/check_workflow_path_filters.sh | 41 ++++++--- scripts/lib/workflow_runs_nightly.awk | 33 ++++++++ .../workflow_path_filters_rule3_parity.sh | 80 ++++++++++++++++++ scripts/tests/workflow_runs_nightly_test.sh | 84 +++++++++++++++++++ 4 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 scripts/lib/workflow_runs_nightly.awk create mode 100755 scripts/tests/workflow_path_filters_rule3_parity.sh create mode 100755 scripts/tests/workflow_runs_nightly_test.sh diff --git a/scripts/check_workflow_path_filters.sh b/scripts/check_workflow_path_filters.sh index 7b47f2fd6a..dbd38ae61b 100644 --- a/scripts/check_workflow_path_filters.sh +++ b/scripts/check_workflow_path_filters.sh @@ -36,7 +36,7 @@ # five wrong guard regexes in this repo were caught by tables, none by review. # # bash scripts/check_workflow_path_filters.sh # check -# bash scripts/check_workflow_path_filters.sh --self-test # 4-case table +# bash scripts/check_workflow_path_filters.sh --self-test # 9-row table set -uo pipefail @@ -44,6 +44,7 @@ SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0] REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WF_DIR="${REPO_ROOT}/.github/workflows" FILTER_DUMP="${REPO_ROOT}/scripts/lib/workflow_path_filters.py" +NIGHTLY_AWK="${REPO_ROOT}/scripts/lib/workflow_runs_nightly.awk" # Workflows that RUN code from a crate must watch that crate's source. # Format: | @@ -69,22 +70,23 @@ check_workflow() { rm -f /tmp/wfpf_err.$$ [ -z "$out" ] && return 0 - local push pr sched + local push pr push="$(printf '%s\n' "$out" | awk -F'\t' '$1=="PUSH"{print $2}' | sort -u)" pr="$(printf '%s\n' "$out" | awk -F'\t' '$1=="PR"{print $2}' | sort -u)" - sched="$(printf '%s\n' "$out" | awk -F'\t' '$1=="SCHEDULE"{print $2}')" - # Rule 3 (#3676): a path-FILTERED workflow must also run on `schedule:`. The + # Rule 3 (#3676): a path-FILTERED workflow must also run nightly: a `schedule:` + # cron, or the chain from "Nightly pick" the nightly producers start on (T44, + # #4686; scripts/lib/workflow_runs_nightly.awk). The # filter is a claim that nothing outside its paths can break the gate; the # nightly run is what turns a wrong claim into a red within a day instead of # the three months book.yml sat dark. Applies when either event is filtered. local filtered=0 if [ -n "$push" ] && [ "$push" != '' ]; then filtered=1; fi if [ -n "$pr" ] && [ "$pr" != '' ]; then filtered=1; fi - if [ "$filtered" = 1 ] && [ "$sched" != yes ]; then - printf '\nFAIL %s: path-filtered, but no `schedule:` trigger.\n' "$name" + if [ "$filtered" = 1 ] && ! awk -f "$NIGHTLY_AWK" "$wf"; then + printf '\nFAIL %s: path-filtered, but no nightly trigger (a `schedule:` cron, or workflow_run on "Nightly pick").\n' "$name" printf ' A change outside the filter that breaks this gate is never seen;\n' - printf ' add a nightly cron so the claim is re-checked every day.\n' + printf ' add a nightly cron, or chain from "Nightly pick", so the claim is re-checked every day.\n' return 1 fi @@ -188,8 +190,25 @@ on: paths: ["book/**", "scripts/book-gate.sh"] YML + # 7 - symmetric, chained from "Nightly pick" instead of a cron (T44, #4686). MUST pass. + cat > "$TD/wf7.yml" <<'YML' +on: + push: + paths: ["book/**"] + pull_request: + paths: ["book/**"] + workflow_run: + workflows: ["Nightly pick"] + types: [completed] + branches: [main] +YML + # 8 - chained from some other workflow: not the nightly (rule 3). MUST fail. + sed 's/"Nightly pick"/"CI"/' "$TD/wf7.yml" > "$TD/wf8.yml" + # 9 - the chain commented out is no trigger (rule 3). MUST fail. + sed 's/^ workflow_run:/ # workflow_run:/; s/^ workflows:/ # workflows:/' "$TD/wf7.yml" > "$TD/wf9.yml" + fails=0 - for c in 1 4 5 6; do + for c in 1 4 5 6 8 9; do if check_workflow "$TD/wf${c}.yml" >/dev/null 2>&1; then printf 'FAIL row %s NOT flagged - the guard is blind to a real defect shape\n' "$c" fails=$((fails + 1)) @@ -197,7 +216,7 @@ YML printf 'ok row %s flagged (must turn RED)\n' "$c" fi done - for c in 2 3; do + for c in 2 3 7; do if check_workflow "$TD/wf${c}.yml" >/dev/null 2>&1; then printf 'ok row %s clean (must stay GREEN)\n' "$c" else @@ -207,9 +226,9 @@ YML done if [ "$fails" -ne 0 ]; then - printf '\nSELF-TEST FAILED (%s/6 wrong)\n' "$fails"; exit 1 + printf '\nSELF-TEST FAILED (%s/9 wrong)\n' "$fails"; exit 1 fi - printf '\nSELF-TEST PASSED (6/6)\n' + printf '\nSELF-TEST PASSED (9/9)\n' exit 0 fi diff --git a/scripts/lib/workflow_runs_nightly.awk b/scripts/lib/workflow_runs_nightly.awk new file mode 100644 index 0000000000..117e6a1efd --- /dev/null +++ b/scripts/lib/workflow_runs_nightly.awk @@ -0,0 +1,33 @@ +# workflow_runs_nightly.awk — does a workflow re-check itself every night? Rule 3 of +# scripts/check_workflow_path_filters.sh (#3676), moved here from workflow_path_filters.py (T44, #4686). +# +# Exit 0 when the workflow's on: block has either +# - a `schedule:` with at least one cron entry that has a value, or +# - the chain: `workflow_run:` with `workflows: ["Nightly pick"]` (that one workflow) and `types:` naming +# `completed`; the nightly producers start on the pick's run instead of their own cron (T44). +# Exit 1 otherwise: no on: block, `schedule: []`, a schedule with no cron, a chain from another workflow. +# Comment lines are ignored, so a commented-out trigger is no trigger. A cron is accepted in the forms the YAML +# reader of the old rule accepted: `- cron: X`, `- {cron: X}` and `schedule: [{cron: X}]`; an empty or quoted-empty +# value is no cron. +# +# awk -f scripts/lib/workflow_runs_nightly.awk WORKFLOW.yml +function ind(s) { match(s, /^ */); return RLENGTH } +function list1(s) { + sub(/^[^:]*:[[:space:]]*/, "", s); sub(/[[:space:]]+$/, "", s) + if (s !~ /^\[.*\]$/) return "\001" + s = substr(s, 2, length(s) - 2); gsub(/^[[:space:]]+|[[:space:]]+$/, "", s) + if (s ~ /^".*"$/ || s ~ /^\047.*\047$/) s = substr(s, 2, length(s) - 2) + return s +} +function hascron(s) { return s ~ /cron:[[:space:]]*["\047]?[^[:space:],}"\047#]/ } +/^on:[[:space:]]*$/ { on = 1; next } +/^[[:space:]]*(#|$)/ { next } +on && ind($0) == 0 { on = 0 } +!on { next } +ind($0) == 2 { k = $0; sub(/^ +/, "", k); sub(/:.*/, "", k); sc = (k == "schedule"); wr = (k == "workflow_run") + if (sc && hascron($0)) cron = 1 + next } +sc && /^[[:space:]]+- [{]?[[:space:]]*cron:/ && hascron($0) { cron = 1 } +wr && ind($0) == 4 && /^ +workflows:/ && list1($0) == "Nightly pick" { w = 1 } +wr && ind($0) == 4 && /^ +types:[[:space:]]*\[/ && $0 ~ /[[,[:space:]]completed[],[:space:]]/ { t = 1 } +END { exit !(cron || (w && t)) } diff --git a/scripts/tests/workflow_path_filters_rule3_parity.sh b/scripts/tests/workflow_path_filters_rule3_parity.sh new file mode 100755 index 0000000000..d11bb257a1 --- /dev/null +++ b/scripts/tests/workflow_path_filters_rule3_parity.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# workflow_path_filters_rule3_parity.sh — rule 3 of check_workflow_path_filters.sh moved from the .py (PyYAML) into +# scripts/lib/workflow_runs_nightly.awk (T44, #4686). This runs the OLD check (both files read from --base) and the +# NEW one (this tree) over the SAME workflows: fixture shapes, every workflow of this tree, every workflow at +# --base. The rule-3 verdicts must agree on every workflow except the ones EXPECTED below, and those must move +# one way only: RED under the old rule, green under the new, because they chain from "Nightly pick". +# +# bash scripts/tests/workflow_path_filters_rule3_parity.sh [--base REF] (default origin/main) +# Exit: 0 parity holds; 1 RED; 2 not_measured (no python3 with yaml: both sides dump filters with it); 3 caller error. +set -uo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +BASE=origin/main +case "${1:-}" in + --base) BASE="${2:-}"; [ -n "$BASE" ] || { echo "caller error: --base REF" >&2; exit 3; } ;; + '') ;; + *) echo "caller error: unknown argument '$1' (--base REF)" >&2; exit 3 ;; +esac +# The difference the move is for, and nothing else. +EXPECTED="fx-chain fx-chain_sq head-book head-book-contracts head-install-script" + +if ! python3 -c 'import yaml' > /dev/null 2>&1; then + echo "not_measured: no python3 with yaml here; both sides of the check need it to dump the filters"; exit 2 +fi +git -C "$ROOT" rev-parse --verify -q "$BASE^{commit}" > /dev/null || { echo "caller error: no commit $BASE" >&2; exit 3; } + +T="$(mktemp -d "${TMPDIR:-/tmp}/r3-parity.XXXXXX")" || { echo "caller error: no temp dir" >&2; exit 3; } +trap 'rm -rf -- "${T:?}"' EXIT +for s in old new; do mkdir -p "$T/$s/scripts/lib" "$T/$s/.github/workflows"; done +git -C "$ROOT" show "$BASE:scripts/check_workflow_path_filters.sh" > "$T/old/scripts/check_workflow_path_filters.sh" || exit 3 +git -C "$ROOT" show "$BASE:scripts/lib/workflow_path_filters.py" > "$T/old/scripts/lib/workflow_path_filters.py" || exit 3 +cp "$ROOT/scripts/check_workflow_path_filters.sh" "$T/new/scripts/" || exit 3 +cp "$ROOT/scripts/lib/workflow_path_filters.py" "$ROOT/scripts/lib/workflow_runs_nightly.awk" "$T/new/scripts/lib/" || exit 3 + +W="$T/wf"; mkdir -p "$W" +P=' push:\n paths: ["book/**"]\n pull_request:\n paths: ["book/**"]\n' +fx() { printf 'on:\n%b%b' "$P" "$2" > "$W/fx-$1.yml"; } # NAME TRIGGERS +fx sched " schedule:\n - cron: '30 22 * * *'\n" +fx none "" +fx empty " schedule: []\n" +fx chain ' workflow_run:\n workflows: ["Nightly pick"]\n types: [completed]\n branches: [main]\n' +fx chain_sq " workflow_run:\n workflows: ['Nightly pick']\n types: [ completed ]\n" +fx chain_other ' workflow_run:\n workflows: ["CI"]\n types: [completed]\n' +fx chain_two ' workflow_run:\n workflows: ["Nightly pick", "CI"]\n types: [completed]\n' +fx chain_req ' workflow_run:\n workflows: ["Nightly pick"]\n types: [requested]\n' +fx chain_cmt ' # workflow_run:\n # workflows: ["Nightly pick"]\n # types: [completed]\n' +fx cron_cmt " schedule:\n # - cron: '30 22 * * *'\n" +fx cron_empty " schedule:\n - cron:\n" +fx cron_qempty " schedule:\n - cron: ''\n" +fx cron_outside_on " workflow_dispatch:\njobs:\n a:\n schedule:\n - cron: '1 1 * * *'\n" +fx flow_list " schedule: [{cron: '1 1 * * *'}]\n" +fx flow_item " schedule:\n - {cron: \"1 1 * * *\"}\n" +fx flow_empty " schedule: [{cron: ''}]\n" +for f in "$ROOT"/.github/workflows/*.yml; do cp "$f" "$W/head-$(basename -- "$f")"; done +git -C "$ROOT" ls-tree --name-only "$BASE" .github/workflows/ | grep -e '\.yml$' | while IFS= read -r p; do + git -C "$ROOT" show "$BASE:$p" > "$W/base-$(basename -- "$p")" +done +for s in old new; do cp "$W"/*.yml "$T/$s/.github/workflows/"; done + +# rule-3 RED set per side: the workflows whose FAIL line is rule 3's +r3() { # SIDE PATTERN + bash "$T/$1/scripts/check_workflow_path_filters.sh" > "$T/$1.out" 2>&1 + grep -e "^FAIL .*$2" "$T/$1.out" | sed -e 's/^FAIL \([^:]*\):.*/\1/' | sed -e 's/\.yml$//' | sort -u +} +r3 old 'no `schedule:` trigger' > "$T/old.r3" +r3 new 'no nightly trigger' > "$T/new.r3" +n="$(find "$W" -name '*.yml' | wc -l)" +grep -q -e '^scanned ' "$T/old.out" && grep -q -e '^scanned ' "$T/new.out" || { echo "RED: a side did not finish its scan"; tail -n 5 "$T/old.out" "$T/new.out"; exit 1; } + +rc=0 +only_old="$(comm -23 "$T/old.r3" "$T/new.r3")" +only_new="$(comm -13 "$T/old.r3" "$T/new.r3")" +want="$(printf '%s\n' $EXPECTED | sort -u)" +printf 'rule 3 over %s workflows: old RED %s, new RED %s\n' "$n" "$(wc -l < "$T/old.r3")" "$(wc -l < "$T/new.r3")" +if [ -n "$only_new" ]; then printf 'RED new rule 3 is RED where the old was not:\n%s\n' "$only_new"; rc=1; fi +if [ "$only_old" != "$want" ]; then + printf 'RED old-RED/new-green set differs from EXPECTED\n got: %s\n want: %s\n' "$(printf "%s\n" "$only_old" | tr "\n" " ")" "$(printf "%s\n" "$want" | tr "\n" " ")"; rc=1 +fi +[ "$rc" -eq 0 ] && printf 'PASS parity: the verdicts agree on every workflow but %s, each old RED -> new green (the chain)\n' "$(printf "%s\n" "$want" | tr "\n" " ")" +exit "$rc" diff --git a/scripts/tests/workflow_runs_nightly_test.sh b/scripts/tests/workflow_runs_nightly_test.sh new file mode 100755 index 0000000000..dafb3063a9 --- /dev/null +++ b/scripts/tests/workflow_runs_nightly_test.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# workflow_runs_nightly_test.sh — case table for scripts/lib/workflow_runs_nightly.awk, rule 3 of +# check_workflow_path_filters.sh (T44, #4686). No python: the awk is judged on its own, and each mutant of it must +# turn a row RED. The old-vs-new parity run is scripts/tests/workflow_path_filters_rule3_parity.sh. +# +# bash scripts/tests/workflow_runs_nightly_test.sh [--mutants] +# Exit: 0 every row as specified (and, with --mutants, every mutant killed); 1 RED; 3 caller error. +set -uo pipefail + +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +AWK_SRC="$ROOT/scripts/lib/workflow_runs_nightly.awk" +[ -f "$AWK_SRC" ] || { echo "caller error: no $AWK_SRC" >&2; exit 3; } +P=' push:\n paths: ["book/**"]\n pull_request:\n paths: ["book/**"]\n' + +# NAME WANT(0 nightly, 1 not) TRIGGERS +ROWS="sched 0 schedule:\n - cron: '30 22 * * *'\n +flow_list 0 schedule: [{cron: '1 1 * * *'}]\n +flow_item 0 schedule:\n - {cron: \"1 1 * * *\"}\n +chain 0 workflow_run:\n workflows: [\"Nightly pick\"]\n types: [completed]\n branches: [main]\n +chain_sq 0 workflow_run:\n workflows: ['Nightly pick']\n types: [ completed ]\n +chain_two_types 0 workflow_run:\n workflows: [\"Nightly pick\"]\n types: [requested, completed]\n +none 1 +empty 1 schedule: []\n +cron_cmt 1 schedule:\n # - cron: '30 22 * * *'\n +cron_empty 1 schedule:\n - cron:\n +cron_qempty 1 schedule:\n - cron: ''\n +flow_empty 1 schedule: [{cron: ''}]\n +cron_outside_on 1 workflow_dispatch:\njobs:\n a:\n schedule:\n - cron: '1 1 * * *'\n +chain_other 1 workflow_run:\n workflows: [\"CI\"]\n types: [completed]\n +chain_two 1 workflow_run:\n workflows: [\"Nightly pick\", \"CI\"]\n types: [completed]\n +chain_req 1 workflow_run:\n workflows: [\"Nightly pick\"]\n types: [requested]\n +chain_no_types 1 workflow_run:\n workflows: [\"Nightly pick\"]\n +chain_cmt 1 # workflow_run:\n # workflows: [\"Nightly pick\"]\n # types: [completed]\n +chain_under_other_key 1 workflow_dispatch:\n workflows: [\"Nightly pick\"]\n types: [completed]\n +comment_col0_in_on 0 # a note at column 0 inside on:\n schedule:\n - cron: '1 1 * * *'\n +schedule_after_on 1 workflow_dispatch:\njobs:\n schedule:\n - cron: '1 1 * * *'\n +chain_nested_deeper 1 workflow_run:\n types: [completed]\n x:\n workflows: [\"Nightly pick\"]\n" + +table() { # AWK-FILE -> prints rows, returns 1 when any is wrong + local awkf="$1" t name want trig rc bad=0 n=0 + t="$(mktemp -d "${TMPDIR:-/tmp}/wrn.XXXXXX")" || exit 3 + while IFS="$(printf '\t')" read -r name want trig; do + [ -n "$name" ] || continue + n=$((n + 1)) + printf 'on:\n%b%b' "$P" "$trig" > "$t/$name.yml" + awk -f "$awkf" "$t/$name.yml" > /dev/null 2>&1; rc=$? + if [ "$rc" = "$want" ]; then printf 'ok %s\n' "$name"; else printf 'RED %-24s rc=%s want %s\n' "$name" "$rc" "$want"; bad=1; fi + done <<< "$ROWS" + rm -rf -- "${t:?}" + [ "$bad" -eq 0 ] && printf 'TABLE PASSED: %s rows\n' "$n" + return "$bad" +} + +MUTANTS='m01_comments_live s/^\/\^\[\[:space:\]\]\*(#|\$)\/ { next }$/\/^[[:space:]]*$\/ { next }/ +m02_any_workflow s/list1(\$0) == "Nightly pick"/1/ +m03_any_type s/ && \$0 ~ \/\[\[,\[:space:\]\]completed\[\],\[:space:\]\]\/ { t = 1 }/ { t = 1 }/ +m04_chain_alone s/exit !(cron || (w && t))/exit !(cron || w)/ +m05_on_never_ends /^on && ind(\$0) == 0 { on = 0 }$/d +m06_empty_cron s/\["\\047\]?\[^\[:space:\],}"\\047#\]/./ +m07_no_flow_list /^ if (sc && hascron(\$0)) cron = 1$/d +m08_no_flow_item s/- \[{\]?\[\[:space:\]\]\*cron:/- cron:/ +m09_any_indent s/wr && ind(\$0) == 4 && \/^ +workflows:/wr \&\& \/workflows:/' + +mutants() { + local t name expr killed=0 total=0 errors=0 + t="$(mktemp -d "${TMPDIR:-/tmp}/wrn-mu.XXXXXX")" || exit 3 + while IFS="$(printf '\t')" read -r name expr; do + [ -n "$name" ] || continue + total=$((total + 1)) + sed -e "$expr" "$AWK_SRC" > "$t/m.awk" + if cmp -s "$AWK_SRC" "$t/m.awk"; then printf 'ERROR %-24s the patch did not apply\n' "$name"; errors=$((errors + 1)); continue; fi + if table "$t/m.awk" > "$t/out" 2>&1; then printf 'SURVIVED %s\n' "$name" + else killed=$((killed + 1)); printf 'killed %-24s %s\n' "$name" "$(grep -c -e '^RED ' "$t/out")"; fi + done <<< "$MUTANTS" + rm -rf -- "${t:?}" + printf 'MUTANTS killed=%s total=%s errors=%s\n' "$killed" "$total" "$errors" + [ "$killed" -eq "$total" ] && [ "$errors" -eq 0 ] +} + +case "${1:-}" in + '') table "$AWK_SRC" ;; + --mutants) table "$AWK_SRC" > /dev/null || { echo "RED: the table fails unmutated" >&2; exit 1; }; mutants ;; + *) echo "caller error: unknown argument '$1' (--mutants)" >&2; exit 3 ;; +esac From f24e4724876bdb895c85fd64d64182ceac4cb947 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 18:26:04 +0200 Subject: [PATCH 10/34] T44: close the round-1 review findings with a row and a mutant each check_producers_chain.sh (rows 28 -> 37, mutants 16 -> 25, all killed): - P3: the C step may carry only name, if, env NIGHTLY_PICK_AT and run. A continue-on-error, or an env line such as GIT_DIR, would swallow a failed switch to C and leave the job measuring main's head while green. - P3: a quoted `uses: "actions/checkout@..."` counts as a checkout. A listed producer with zero checkouts of this repo is RED, and so is a job-level `uses:` (a reusable workflow whose checkout this file cannot see). - P3: a checkout `if:` with `${{ }}` or a block scalar is RED, because the C step cannot copy it into `... && ()` (`a && (${{ x }})` is always true). - P4: github.workflow_sha and GITHUB_WORKFLOW_SHA are named too, and a NIGHTLY_C in a trailing comment no longer exempts the line. - The repo-exempt row now also has a checkout of this repo (a producer with only an other-repo checkout is RED by the rule above). workflow_runs_nightly.awk: a `- cron:` item at the key's own indent (` schedule:` then ` - cron: X`, valid YAML the old rule accepted) is a cron, not a new key. 24 rows, mutants 10/10. The parity fixtures gain the same shape. workflow_path_filters_rule3_parity.sh: the default --base is 316dee2cd4, main before T44. Once T44 is on main, origin/main holds no old rule 3. Mutation harnesses (this guard, nightly_c_checkout.sh, the awk test): a sed that rejects the patch is now an ERROR. Before, it produced an empty script whose RED rows counted as a kill. A broken patch injected into the awk test reports ERROR. Declined: requiring a checkout in every job of a producer. Jobs that only post results do not touch the tree. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/lib/workflow_runs_nightly.awk | 4 +- scripts/release/check_producers_chain.sh | 82 +++++++++++++++---- scripts/release/nightly_c_checkout.sh | 5 +- .../workflow_path_filters_rule3_parity.sh | 5 +- scripts/tests/workflow_runs_nightly_test.sh | 9 +- 5 files changed, 83 insertions(+), 22 deletions(-) diff --git a/scripts/lib/workflow_runs_nightly.awk b/scripts/lib/workflow_runs_nightly.awk index 117e6a1efd..1a072a60a2 100644 --- a/scripts/lib/workflow_runs_nightly.awk +++ b/scripts/lib/workflow_runs_nightly.awk @@ -8,7 +8,7 @@ # Exit 1 otherwise: no on: block, `schedule: []`, a schedule with no cron, a chain from another workflow. # Comment lines are ignored, so a commented-out trigger is no trigger. A cron is accepted in the forms the YAML # reader of the old rule accepted: `- cron: X`, `- {cron: X}` and `schedule: [{cron: X}]`; an empty or quoted-empty -# value is no cron. +# value is no cron. A sequence item may sit at the key's own indent (` schedule:` then ` - cron: X`). # # awk -f scripts/lib/workflow_runs_nightly.awk WORKFLOW.yml function ind(s) { match(s, /^ */); return RLENGTH } @@ -24,7 +24,7 @@ function hascron(s) { return s ~ /cron:[[:space:]]*["\047]?[^[:space:],}"\047#]/ /^[[:space:]]*(#|$)/ { next } on && ind($0) == 0 { on = 0 } !on { next } -ind($0) == 2 { k = $0; sub(/^ +/, "", k); sub(/:.*/, "", k); sc = (k == "schedule"); wr = (k == "workflow_run") +ind($0) == 2 && !/^ - / { k = $0; sub(/^ +/, "", k); sub(/:.*/, "", k); sc = (k == "schedule"); wr = (k == "workflow_run") if (sc && hascron($0)) cron = 1 next } sc && /^[[:space:]]+- [{]?[[:space:]]*cron:/ && hascron($0) { cron = 1 } diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index a4e64ae2ab..5aa8158146 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -13,7 +13,11 @@ # "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"`, # env NIGHTLY_PICK_AT from github.event.workflow_run.run_started_at, and if exactly # `github.event_name == 'workflow_run'`, or `github.event_name == 'workflow_run' && ()`; -# P4 names github.sha or GITHUB_SHA on a live line that names neither NIGHTLY_C nor workflow_run.head_sha; +# the C step carries nothing else (no continue-on-error, no other env), a checkout's if must be a bare +# expression the C step can copy, a listed producer has at least one checkout of this repo, and no job of it +# calls a reusable workflow (`uses:` at job level), whose checkout this file cannot see; +# P4 names github.sha, GITHUB_SHA, github.workflow_sha or GITHUB_WORKFLOW_SHA on a live line that names neither +# NIGHTLY_C nor workflow_run.head_sha outside a trailing comment; # P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways). # Limits: it reads the workflow text, not the scripts a step calls; an origin/main a step fetches as a declared # comparand (guards-nightly's SATD ceiling) is not a measurement and is not judged here. @@ -49,6 +53,19 @@ judge_file() { for (i = 2; i <= m; i++) if (index(a[i], p k ":") == 1) return bare(a[i]) return "\001" } + function unq(s) { if (s ~ /^".*"$/ || s ~ /^\047.*\047$/) s = substr(s, 2, length(s) - 2); return s } + function stray(st, a, m, i, p, k, s) { # the first part of the C step that is not name/if/run or the env line + m = split(st, a, "\n"); p = ind(a[1]) + 2 + for (i = 1; i <= m; i++) { + s = a[i]; if (!live(s)) continue + if (i == 1) { k = s; sub(/^ *- /, "", k) } + else if (ind(s) == p) k = substr(s, p + 1) + else { gsub(/^[[:space:]]+|[[:space:]]+$/, "", s) + if (s == "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}") continue; return s } + sub(/:.*/, "", k); if (k !~ /^(name|if|env|run)$/) return k + } + return "" + } function hasline(st, want, a, m, i, s) { m = split(st, a, "\n") for (i = 1; i <= m; i++) { s = a[i]; gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); if (s == want) return 1 } @@ -57,11 +74,13 @@ judge_file() { function flush( k, j, cif, want, nm) { if (!listed) { n = 0; ii = -1; return } # only a listed producer owes the C step for (k = 1; k <= n; k++) { - if (key(S[k], "uses") !~ /^actions\/checkout@/ || S[k] ~ /\n[[:space:]]*repository:/) continue + if (unq(key(S[k], "uses")) !~ /^actions\/checkout@/ || S[k] ~ /\n[[:space:]]*repository:/) continue checkouts++ j = k + 1; nm = key(S[j], "name") if (j <= n && nm ~ /^Preflight/) j++ - cif = key(S[k], "if") + cif = unq(key(S[k], "if")) + if (cif ~ /\$\{\{|^[>|]/) { + fail("P3 checkout in job step " k " has an if: the C step cannot copy (${{ }} or a block): " cif); continue } want = (cif == "\001") ? "github.event_name == \047workflow_run\047" \ : "github.event_name == \047workflow_run\047 && (" cif ")" if (j > n) { fail("P3 checkout in job step " k " is not followed by the C step (end of steps)"); continue } @@ -69,8 +88,10 @@ judge_file() { fail("P3 checkout in job step " k " is not followed by the C step (step " j " runs something else)"); continue } if (!hasline(S[j], "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}")) fail("P3 the C step after job step " k " does not take NIGHTLY_PICK_AT from workflow_run.run_started_at") - if (key(S[j], "if") != want) + if (unq(key(S[j], "if")) != want) fail("P3 the C step after job step " k " has if: " key(S[j], "if") " -- want: " want) + if ((sx = stray(S[j])) != "") + fail("P3 the C step after job step " k " carries " sx " -- only name, if, env NIGHTLY_PICK_AT, run") } n = 0; ii = -1 } @@ -87,8 +108,10 @@ judge_file() { if (wr && L ~ /^ branches:/ && list1(L) == "main") cb = 1 } # sha rule, every live line - live(L) && L ~ /github\.sha|GITHUB_SHA/ && L !~ /NIGHTLY_C|workflow_run\.head_sha/ { - shaline[++ns] = NR } + live(L) { c = L; sub(/[[:space:]]#.*/, "", c) # an exemption in a trailing comment exempts nothing + if (c ~ /github\.(workflow_)?sha|GITHUB_(WORKFLOW_)?SHA/ && c !~ /NIGHTLY_C|workflow_run\.head_sha/) shaline[++ns] = NR } + # a job that calls a reusable workflow: its checkout is not in this file + listed && live(L) && L ~ /^ uses:/ { fail("P3 line " NR " is a job-level uses: -- its checkout cannot be judged here") } # steps insteps && live(L) && ind(L) <= si { flush(); insteps = 0 } insteps && live(L) { @@ -102,6 +125,7 @@ judge_file() { chained = (cw && ct && cb) if (!listed) { if (cw) printf "FAIL %s: P5 chains from \"%s\" but is not in the producer list\n", f, pick; exit } if (sched) fail("P1 still has a schedule: trigger -- a producer starts on the pick, never on its own cron") + if (!checkouts) fail("P3 no actions/checkout of this repo found -- nothing switches the tree to C") if (!chained) fail("P2 no workflow_run on [\"" pick "\"], types [completed], branches [main]" \ " (workflows=" cw " types=" ct " branches=" cb ")") for (i = 1; i <= ns; i++) fail("P4 line " shaline[i] " names github.sha/GITHUB_SHA -- main at trigger, not C") @@ -183,7 +207,7 @@ m_cstep_ifdrop() { changed cuda-nightly.yml "s/if: github.event_name == 'workfl m_cstep_noat() { changed nightly-bench.yml '0,/nightly_c_checkout.sh --at "\$NIGHTLY_PICK_AT"/s//nightly_c_checkout.sh/'; } m_cstep_wrongat() { changed coverage-nightly.yml '0,/workflow_run.run_started_at/s//workflow_run.head_commit.timestamp/'; } m_step_between() { changed examples-nightly.yml '0,/- name: Measure the night.s C/s//- run: make bench\n - name: Measure the night'"'"'s C/'; } -m_repo_exempt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/other\n')"; } +m_repo_exempt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/other\n')"; } m_preflight_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - name: Preflight — tools\n run: true\n'; cstep)"; } m_flow_if_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep 'a || b')"; } m_flow_if_bare() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep | sed -e "s/'workflow_run'\$/'workflow_run' \&\& a || b/")"; } @@ -192,6 +216,15 @@ m_sha_plain() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\ m_sha_env() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: git diff "$GITHUB_SHA"/'; } m_sha_comment() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n # measured at $GITHUB_SHA, before T44/'; } m_sha_c() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: echo "${NIGHTLY_C:-$GITHUB_SHA}"/'; } +m_cstep_coe() { changed install-script.yml '0,/nightly_c_checkout.sh --at "\$NIGHTLY_PICK_AT"$/s//&\n continue-on-error: true/'; } +m_cstep_envx() { changed nightly-bench.yml '0,/NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}$/s//&\n GIT_DIR: \/elsewhere/'; } +m_cstep_qif_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep | sed -e "s/if: \(.*\)\$/if: \"\1\"/")"; } +m_quoted_uses() { synth nightly.yml <<< "$(printf ' - uses: "actions/checkout@v7"\n - run: make bench\n')"; } +m_no_checkout() { synth nightly.yml <<< ' - run: make bench'; } +m_job_uses() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' b:\n uses: ./.github/workflows/bench.yml\n')"; } +m_checkout_ifexp(){ synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: ${{ inputs.x }}\n'; cstep '${{ inputs.x }}')"; } +m_sha_workflow() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ github.workflow_sha }}"/'; } +m_sha_cmt_exempt(){ changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: git diff "${{ github.sha }}" # NIGHTLY_C/'; } m_bystander() { printf 'name: Other\non:\n schedule:\n - cron: "0 1 * * *"\njobs:\n a:\n runs-on: x\n steps:\n - uses: actions/checkout@v7\n - run: echo "$GITHUB_SHA"\n' > other.yml; } self_test() { @@ -213,7 +246,7 @@ self_test() { row p3_if_or_true 1 'install-script.yml: P3' -- m_cstep_or row p3_if_drops_checkout_if 1 'cuda-nightly.yml: P3' -- m_cstep_ifdrop row p3_no_at 1 'nightly-bench.yml: P3' -- m_cstep_noat - row p3_at_from_head_commit 1 'coverage-nightly.yml: P3 the C step' -- m_cstep_wrongat + row p3_at_from_head_commit 1 'does not take NIGHTLY_PICK_AT' -- m_cstep_wrongat row p3_step_between 1 'examples-nightly.yml: P3' -- m_step_between row p3_last_step_checkout 1 'end of steps' -- m_last_step row p3_other_repo_exempt 0 'PASS' -- m_repo_exempt @@ -225,6 +258,15 @@ self_test() { row p4_comment_ignored 0 'PASS' -- m_sha_comment row p4_nightly_c_fallback 0 'PASS' -- m_sha_c row p5_unlisted_scheduled_ok 0 'PASS' -- m_bystander + row p3_cstep_continue_on_error 1 'carries continue-on-error' -- m_cstep_coe + row p3_cstep_extra_env 1 'carries GIT_DIR' -- m_cstep_envx + row p3_cstep_quoted_if 0 'nightly.yml: chained' -- m_cstep_qif_ok + row p3_quoted_checkout_uses 1 'is not followed by the C step' -- m_quoted_uses + row p3_no_checkout 1 'no actions/checkout of this repo' -- m_no_checkout + row p3_job_level_uses 1 'job-level uses' -- m_job_uses + row p3_checkout_if_expression 1 'cannot copy' -- m_checkout_ifexp + row p4_workflow_sha 1 'qwen-story-daily.yml: P4' -- m_sha_workflow + row p4_exemption_in_comment 1 'qwen-story-daily.yml: P4' -- m_sha_cmt_exempt rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -236,16 +278,25 @@ m03_any_type s/list1(L) == "completed") ct = 1/1) ct = 1/ m04_any_workflow s/list1(L) == pick) cw = 1/1) cw = 1/ m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ m06_no_repo_exempt s/ || S\[k\] ~ \/\\n\[\[:space:\]\]\*repository:\/) continue/) continue/ -m07_any_if s/if (key(S\[j\], "if") != want)/if (0)/ +m07_any_if s/if (unq(key(S\[j\], "if")) != want)/if (0)/ m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ -m09_no_sha_rule s/&& L !~ \/NIGHTLY_C|workflow_run\\.head_sha\/ {/\&\& 0 {/ -m10_sha_no_exemption s/&& L !~ \/NIGHTLY_C|workflow_run\\.head_sha\/ {/{/ +m09_no_sha_rule s|if (c ~ /github|if (0 \&\& c ~ /github| +m10_sha_no_exemption s| \&\& c !~ /NIGHTLY_C\|workflow_run\\.head_sha/)|)| m11_no_missing_rule /P5 listed producer is missing/s/; rc=1; }/; }/ m12_no_unlisted_rule s/if (cw) printf "FAIL/if (0) printf "FAIL/ m13_comments_live s/function live(s) { return s !~ \/^\[\[:space:\]\]\*(#|\$)\/ }/function live(s) { return s !~ \/^[[:space:]]*$\/ }/ m14_end_of_steps_ok s/if (j > n) { fail(/if (0) { fail(/ m15_any_run s/if (key(S\[j\], "run") != /if (0 \&\& key(S[j], "run") != / -m16_judge_bystanders /only a listed producer owes the C step/d' +m16_judge_bystanders /only a listed producer owes the C step/d +m17_cstep_any_key s/if ((sx = stray(S\[j\])) != "")/if (0)/ +m18_quoted_uses_unseen s/if (unq(key(S\[k\], "uses")) !~/if (key(S[k], "uses") !~/ +m19_zero_checkouts_ok s/if (!checkouts) fail/if (0) fail/ +m20_job_uses_ok s/listed \&\& live(L) \&\& L ~ \/^ uses:\//0 \&\& \/^ uses:\// +m21_checkout_if_any s/if (cif ~ \/\\\$\\{\\{|^\[>|\]\/) {/if (0) {/ +m22_comment_exempts s/c = L; sub(\/\[\[:space:\]\]#\.\*\/, "", c)/c = L/ +m23_no_workflow_sha s/github\\\.(workflow_)?sha/github\\.sha/ +m24_env_any_line s/if (s == "NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}") continue; return s }/continue }/ +m25_cstep_if_quoted_no s/if (unq(key(S\[j\], "if")) != want)/if (key(S[j], "if") != want)/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -256,7 +307,10 @@ mutants() { while IFS="$(printf '\t')" read -r name expr; do [ -n "$name" ] || continue total=$((total + 1)) - { head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/scripts/release/check_producers_chain.sh" + if ! head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr" > "$tmp/top"; then + printf 'ERROR %-28s sed rejected the patch (an error, never a kill)\n' "$name"; errors=$((errors + 1)); continue + fi + { cat "$tmp/top"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/scripts/release/check_producers_chain.sh" if cmp -s "$SCRIPT_PATH" "$tmp/scripts/release/check_producers_chain.sh"; then printf 'ERROR %-28s the patch did not apply (an error, never a survivor)\n' "$name"; errors=$((errors + 1)); continue fi @@ -282,7 +336,7 @@ main() { case "${1:-}" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,26p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,28p' "$SCRIPT_PATH"; return 0 ;; --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; '') ;; *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh index 6c721787cf..e5a663dddb 100755 --- a/scripts/release/nightly_c_checkout.sh +++ b/scripts/release/nightly_c_checkout.sh @@ -157,7 +157,10 @@ mutants() { while IFS="$(printf '\t')" read -r name expr; do [ -n "$name" ] || continue total=$((total + 1)) - { head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/release/nightly_c_checkout.sh" + if ! head -n "$((cut - 1))" "$SCRIPT_PATH" | sed -e "$expr" > "$tmp/top"; then + printf 'ERROR %-32s sed rejected the patch (an error, never a kill)\n' "$name"; errors=$((errors + 1)); continue + fi + { cat "$tmp/top"; tail -n "+$cut" "$SCRIPT_PATH"; } > "$tmp/release/nightly_c_checkout.sh" if cmp -s "$SCRIPT_PATH" "$tmp/release/nightly_c_checkout.sh"; then printf 'ERROR %-32s the patch did not apply (an error, never a survivor)\n' "$name"; errors=$((errors + 1)); continue fi diff --git a/scripts/tests/workflow_path_filters_rule3_parity.sh b/scripts/tests/workflow_path_filters_rule3_parity.sh index d11bb257a1..840ae82876 100755 --- a/scripts/tests/workflow_path_filters_rule3_parity.sh +++ b/scripts/tests/workflow_path_filters_rule3_parity.sh @@ -5,12 +5,12 @@ # --base. The rule-3 verdicts must agree on every workflow except the ones EXPECTED below, and those must move # one way only: RED under the old rule, green under the new, because they chain from "Nightly pick". # -# bash scripts/tests/workflow_path_filters_rule3_parity.sh [--base REF] (default origin/main) +# bash scripts/tests/workflow_path_filters_rule3_parity.sh [--base REF] (default 316dee2cd4, main before T44) # Exit: 0 parity holds; 1 RED; 2 not_measured (no python3 with yaml: both sides dump filters with it); 3 caller error. set -uo pipefail ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" -BASE=origin/main +BASE=316dee2cd4 # main before T44: the old rule 3 lives there. Once T44 is on main, origin/main has no old side case "${1:-}" in --base) BASE="${2:-}"; [ -n "$BASE" ] || { echo "caller error: --base REF" >&2; exit 3; } ;; '') ;; @@ -51,6 +51,7 @@ fx cron_outside_on " workflow_dispatch:\njobs:\n a:\n schedule:\n - cr fx flow_list " schedule: [{cron: '1 1 * * *'}]\n" fx flow_item " schedule:\n - {cron: \"1 1 * * *\"}\n" fx flow_empty " schedule: [{cron: ''}]\n" +fx cron_dash0 " schedule:\n - cron: '0 1 * * *'\n" for f in "$ROOT"/.github/workflows/*.yml; do cp "$f" "$W/head-$(basename -- "$f")"; done git -C "$ROOT" ls-tree --name-only "$BASE" .github/workflows/ | grep -e '\.yml$' | while IFS= read -r p; do git -C "$ROOT" show "$BASE:$p" > "$W/base-$(basename -- "$p")" diff --git a/scripts/tests/workflow_runs_nightly_test.sh b/scripts/tests/workflow_runs_nightly_test.sh index dafb3063a9..89f248eed9 100755 --- a/scripts/tests/workflow_runs_nightly_test.sh +++ b/scripts/tests/workflow_runs_nightly_test.sh @@ -34,7 +34,9 @@ chain_cmt 1 # workflow_run:\n # workflows: [\"Nightly pick\"]\n # types: chain_under_other_key 1 workflow_dispatch:\n workflows: [\"Nightly pick\"]\n types: [completed]\n comment_col0_in_on 0 # a note at column 0 inside on:\n schedule:\n - cron: '1 1 * * *'\n schedule_after_on 1 workflow_dispatch:\njobs:\n schedule:\n - cron: '1 1 * * *'\n -chain_nested_deeper 1 workflow_run:\n types: [completed]\n x:\n workflows: [\"Nightly pick\"]\n" +chain_nested_deeper 1 workflow_run:\n types: [completed]\n x:\n workflows: [\"Nightly pick\"]\n +cron_dash_at_key_indent 0 schedule:\n - cron: '0 1 * * *'\n +cron_dash_at_key_indent_empty 1 schedule:\n - cron: ''\n" table() { # AWK-FILE -> prints rows, returns 1 when any is wrong local awkf="$1" t name want trig rc bad=0 n=0 @@ -59,7 +61,8 @@ m05_on_never_ends /^on && ind(\$0) == 0 { on = 0 }$/d m06_empty_cron s/\["\\047\]?\[^\[:space:\],}"\\047#\]/./ m07_no_flow_list /^ if (sc && hascron(\$0)) cron = 1$/d m08_no_flow_item s/- \[{\]?\[\[:space:\]\]\*cron:/- cron:/ -m09_any_indent s/wr && ind(\$0) == 4 && \/^ +workflows:/wr \&\& \/workflows:/' +m09_any_indent s/wr && ind(\$0) == 4 && \/^ +workflows:/wr \&\& \/workflows:/ +m10_dash_is_a_key s/^ind(\$0) == 2 && !\/^ - \/ {/ind(\$0) == 2 {/' mutants() { local t name expr killed=0 total=0 errors=0 @@ -67,7 +70,7 @@ mutants() { while IFS="$(printf '\t')" read -r name expr; do [ -n "$name" ] || continue total=$((total + 1)) - sed -e "$expr" "$AWK_SRC" > "$t/m.awk" + if ! sed -e "$expr" "$AWK_SRC" > "$t/m.awk"; then printf 'ERROR %-24s sed rejected the patch\n' "$name"; errors=$((errors + 1)); continue; fi if cmp -s "$AWK_SRC" "$t/m.awk"; then printf 'ERROR %-24s the patch did not apply\n' "$name"; errors=$((errors + 1)); continue; fi if table "$t/m.awk" > "$t/out" 2>&1; then printf 'SURVIVED %s\n' "$name" else killed=$((killed + 1)); printf 'killed %-24s %s\n' "$name" "$(grep -c -e '^RED ' "$t/out")"; fi From 92322d549800fb6d007a191888a1246099867192 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 18:39:48 +0200 Subject: [PATCH 11/34] T44: close the round-2 review findings with a row and a mutant each check_producers_chain.sh (rows 37 -> 41, mutants 25 -> 29, all killed): - A step list whose dash sits at the `steps:` indent (`steps:` then `- uses: ...` at the same column) is parsed as steps. Before, it flushed on the first item, and its checkout went unjudged. - `repository:` exempts a checkout only when it names another repo literally. `${{ github.repository }}`, or this repo's name in any case, is a checkout of this repo and owes the C step. - A step that uses a local action (`uses: ./...`) is RED in a listed producer, because a checkout inside it cannot be judged from this file. - The stated limits now name a later run step that fetches main again (text only), and an `on:` that is not a block mapping at indent 2 (fails P2, closed). Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/check_producers_chain.sh | 38 +++++++++++++++++++----- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 5aa8158146..4fc202eed2 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -9,7 +9,7 @@ # P1 still has a `schedule:` trigger (a live line in its on: block); # P2 lacks the chain: workflow_run with workflows: [], types: [completed], # branches: [main]; -# P3 checks this repo out (actions/checkout without `repository:`) and the next step, after an optional +# P3 checks this repo out (actions/checkout, unless `repository:` names another repo literally) and the next step, after an optional # "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"`, # env NIGHTLY_PICK_AT from github.event.workflow_run.run_started_at, and if exactly # `github.event_name == 'workflow_run'`, or `github.event_name == 'workflow_run' && ()`; @@ -19,8 +19,10 @@ # P4 names github.sha, GITHUB_SHA, github.workflow_sha or GITHUB_WORKFLOW_SHA on a live line that names neither # NIGHTLY_C nor workflow_run.head_sha outside a trailing comment; # P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways). -# Limits: it reads the workflow text, not the scripts a step calls; an origin/main a step fetches as a declared -# comparand (guards-nightly's SATD ceiling) is not a measurement and is not judged here. +# Limits: it reads the workflow text, not the scripts a step calls, so a later run step that fetches or checks out +# main again is not seen; an origin/main a step fetches as a declared comparand (guards-nightly's SATD ceiling) is +# not a measurement and is not judged here. It reads `on:` as a block mapping with keys at indent 2; any other +# layout hides the chain and fails P2 (closed). # # Usage: # check_producers_chain.sh [--root DIR] judge the tree (default: the repo this file is in) @@ -66,6 +68,12 @@ judge_file() { } return "" } + function other(st, a, m, i, r) { # a checkout of ANOTHER repo: a literal owner/name that is not this one + m = split(st, a, "\n") + for (i = 2; i <= m; i++) if (a[i] ~ /^[[:space:]]*repository:/) { + r = unq(bare(a[i])); return (r !~ /\$\{\{/ && tolower(r) != "paiml/aprender") } + return 0 + } function hasline(st, want, a, m, i, s) { m = split(st, a, "\n") for (i = 1; i <= m; i++) { s = a[i]; gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); if (s == want) return 1 } @@ -74,7 +82,9 @@ judge_file() { function flush( k, j, cif, want, nm) { if (!listed) { n = 0; ii = -1; return } # only a listed producer owes the C step for (k = 1; k <= n; k++) { - if (unq(key(S[k], "uses")) !~ /^actions\/checkout@/ || S[k] ~ /\n[[:space:]]*repository:/) continue + if (unq(key(S[k], "uses")) ~ /^\.\//) { + fail("P3 job step " k " uses a local action -- a checkout inside it cannot be judged here"); continue } + if (unq(key(S[k], "uses")) !~ /^actions\/checkout@/ || other(S[k])) continue checkouts++ j = k + 1; nm = key(S[j], "name") if (j <= n && nm ~ /^Preflight/) j++ @@ -113,7 +123,7 @@ judge_file() { # a job that calls a reusable workflow: its checkout is not in this file listed && live(L) && L ~ /^ uses:/ { fail("P3 line " NR " is a job-level uses: -- its checkout cannot be judged here") } # steps - insteps && live(L) && ind(L) <= si { flush(); insteps = 0 } + insteps && live(L) && (ind(L) < si || (ind(L) == si && L !~ /^ *- /)) { flush(); insteps = 0 } insteps && live(L) { if (L ~ /^ *- / && (ii < 0 || ind(L) == ii)) { ii = ind(L); S[++n] = L } else if (n > 0) S[n] = S[n] "\n" L @@ -225,6 +235,10 @@ m_job_uses() { synth nightly.yml <<< "$(printf ' - uses: actions/check m_checkout_ifexp(){ synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: ${{ inputs.x }}\n'; cstep '${{ inputs.x }}')"; } m_sha_workflow() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ github.workflow_sha }}"/'; } m_sha_cmt_exempt(){ changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: git diff "${{ github.sha }}" # NIGHTLY_C/'; } +m_dash_at_steps() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - run: git checkout origin/main\n')"; } +m_repo_expr() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: ${{ github.repository }}\n ref: main\n')"; } +m_repo_self() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: Paiml/Aprender\n')"; } +m_local_action() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: ./.github/actions/bench\n')"; } m_bystander() { printf 'name: Other\non:\n schedule:\n - cron: "0 1 * * *"\njobs:\n a:\n runs-on: x\n steps:\n - uses: actions/checkout@v7\n - run: echo "$GITHUB_SHA"\n' > other.yml; } self_test() { @@ -267,6 +281,10 @@ self_test() { row p3_checkout_if_expression 1 'cannot copy' -- m_checkout_ifexp row p4_workflow_sha 1 'qwen-story-daily.yml: P4' -- m_sha_workflow row p4_exemption_in_comment 1 'qwen-story-daily.yml: P4' -- m_sha_cmt_exempt + row p3_dash_at_steps_indent 1 'is not followed by the C step' -- m_dash_at_steps + row p3_repository_expression 1 'is not followed by the C step' -- m_repo_expr + row p3_repository_this_repo 1 'is not followed by the C step' -- m_repo_self + row p3_local_action 1 'uses a local action' -- m_local_action rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -277,7 +295,7 @@ m02_any_branch s/list1(L) == "main") cb = 1/1) cb = 1/ m03_any_type s/list1(L) == "completed") ct = 1/1) ct = 1/ m04_any_workflow s/list1(L) == pick) cw = 1/1) cw = 1/ m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ -m06_no_repo_exempt s/ || S\[k\] ~ \/\\n\[\[:space:\]\]\*repository:\/) continue/) continue/ +m06_no_repo_exempt s/ || other(S\[k\])) continue/) continue/ m07_any_if s/if (unq(key(S\[j\], "if")) != want)/if (0)/ m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ m09_no_sha_rule s|if (c ~ /github|if (0 \&\& c ~ /github| @@ -296,7 +314,11 @@ m21_checkout_if_any s/if (cif ~ \/\\\$\\{\\{|^\[>|\]\/) {/if (0) {/ m22_comment_exempts s/c = L; sub(\/\[\[:space:\]\]#\.\*\/, "", c)/c = L/ m23_no_workflow_sha s/github\\\.(workflow_)?sha/github\\.sha/ m24_env_any_line s/if (s == "NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}") continue; return s }/continue }/ -m25_cstep_if_quoted_no s/if (unq(key(S\[j\], "if")) != want)/if (key(S[j], "if") != want)/' +m25_cstep_if_quoted_no s/if (unq(key(S\[j\], "if")) != want)/if (key(S[j], "if") != want)/ +m26_dash_at_si_ends s/(ind(L) < si || (ind(L) == si && L !~ \/^ \*- \/))/ind(L) <= si/ +m27_any_repo_exempt s/ || other(S\[k\])) continue/ || S[k] ~ \/repository:\/) continue/ +m28_literal_self_exempt s/ && tolower(r) != "paiml\/aprender")/)/ +m29_local_action_ok s/if (unq(key(S\[k\], "uses")) ~ \/^\\.\\\/\/) {/if (0) {/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -336,7 +358,7 @@ main() { case "${1:-}" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,28p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,30p' "$SCRIPT_PATH"; return 0 ;; --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; '') ;; *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; From f3e51b30ca4008e94fd7494184b33acdd643734a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 18:49:01 +0200 Subject: [PATCH 12/34] T44: close the round-3 review findings with a row and a mutant each check_producers_chain.sh (rows 41 -> 44, mutants 29 -> 32, all killed): - `repository: paiml/aprender # this repo` no longer exempts a checkout of this repo, because a trailing comment is stripped before the comparison. - P4 matches the SHA names case-insensitively and in index form: `github['sha']`, `GitHub.SHA`, `github_workflow_sha`. - The stated limits name a flow-style step (`- {uses: ...}`), which is not read as a checkout. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/check_producers_chain.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 4fc202eed2..1279d94ec1 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -22,7 +22,8 @@ # Limits: it reads the workflow text, not the scripts a step calls, so a later run step that fetches or checks out # main again is not seen; an origin/main a step fetches as a declared comparand (guards-nightly's SATD ceiling) is # not a measurement and is not judged here. It reads `on:` as a block mapping with keys at indent 2; any other -# layout hides the chain and fails P2 (closed). +# layout hides the chain and fails P2 (closed). A flow-style step (`- {uses: actions/checkout@v4}`) is not read as a +# checkout. # # Usage: # check_producers_chain.sh [--root DIR] judge the tree (default: the repo this file is in) @@ -71,7 +72,7 @@ judge_file() { function other(st, a, m, i, r) { # a checkout of ANOTHER repo: a literal owner/name that is not this one m = split(st, a, "\n") for (i = 2; i <= m; i++) if (a[i] ~ /^[[:space:]]*repository:/) { - r = unq(bare(a[i])); return (r !~ /\$\{\{/ && tolower(r) != "paiml/aprender") } + r = bare(a[i]); sub(/[[:space:]]+#.*/, "", r); r = unq(r); return (r !~ /\$\{\{/ && tolower(r) != "paiml/aprender") } return 0 } function hasline(st, want, a, m, i, s) { @@ -119,7 +120,7 @@ judge_file() { } # sha rule, every live line live(L) { c = L; sub(/[[:space:]]#.*/, "", c) # an exemption in a trailing comment exempts nothing - if (c ~ /github\.(workflow_)?sha|GITHUB_(WORKFLOW_)?SHA/ && c !~ /NIGHTLY_C|workflow_run\.head_sha/) shaline[++ns] = NR } + if (tolower(c) ~ /github(\.|\[["\047])(workflow_)?sha|github_(workflow_)?sha/ && c !~ /NIGHTLY_C|workflow_run\.head_sha/) shaline[++ns] = NR } # a job that calls a reusable workflow: its checkout is not in this file listed && live(L) && L ~ /^ uses:/ { fail("P3 line " NR " is a job-level uses: -- its checkout cannot be judged here") } # steps @@ -239,6 +240,9 @@ m_dash_at_steps() { synth nightly.yml <<< "$(printf ' - uses: actions/checkou m_repo_expr() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: ${{ github.repository }}\n ref: main\n')"; } m_repo_self() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: Paiml/Aprender\n')"; } m_local_action() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: ./.github/actions/bench\n')"; } +m_repo_self_cmt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/aprender # this repo\n')"; } +m_sha_bracket() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ github['"'"'sha'"'"'] }}"/'; } +m_sha_case() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ GitHub.SHA }}"/'; } m_bystander() { printf 'name: Other\non:\n schedule:\n - cron: "0 1 * * *"\njobs:\n a:\n runs-on: x\n steps:\n - uses: actions/checkout@v7\n - run: echo "$GITHUB_SHA"\n' > other.yml; } self_test() { @@ -285,6 +289,9 @@ self_test() { row p3_repository_expression 1 'is not followed by the C step' -- m_repo_expr row p3_repository_this_repo 1 'is not followed by the C step' -- m_repo_self row p3_local_action 1 'uses a local action' -- m_local_action + row p3_repository_this_repo_cmt 1 'is not followed by the C step' -- m_repo_self_cmt + row p4_github_sha_bracket 1 'qwen-story-daily.yml: P4' -- m_sha_bracket + row p4_github_sha_case 1 'qwen-story-daily.yml: P4' -- m_sha_case rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -298,7 +305,7 @@ m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ m06_no_repo_exempt s/ || other(S\[k\])) continue/) continue/ m07_any_if s/if (unq(key(S\[j\], "if")) != want)/if (0)/ m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ -m09_no_sha_rule s|if (c ~ /github|if (0 \&\& c ~ /github| +m09_no_sha_rule s|if (tolower(c) ~|if (0 \&\& tolower(c) ~| m10_sha_no_exemption s| \&\& c !~ /NIGHTLY_C\|workflow_run\\.head_sha/)|)| m11_no_missing_rule /P5 listed producer is missing/s/; rc=1; }/; }/ m12_no_unlisted_rule s/if (cw) printf "FAIL/if (0) printf "FAIL/ @@ -312,13 +319,16 @@ m19_zero_checkouts_ok s/if (!checkouts) fail/if (0) fail/ m20_job_uses_ok s/listed \&\& live(L) \&\& L ~ \/^ uses:\//0 \&\& \/^ uses:\// m21_checkout_if_any s/if (cif ~ \/\\\$\\{\\{|^\[>|\]\/) {/if (0) {/ m22_comment_exempts s/c = L; sub(\/\[\[:space:\]\]#\.\*\/, "", c)/c = L/ -m23_no_workflow_sha s/github\\\.(workflow_)?sha/github\\.sha/ +m23_no_workflow_sha s/(workflow_)?sha|github_/sha|github_/ m24_env_any_line s/if (s == "NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}") continue; return s }/continue }/ m25_cstep_if_quoted_no s/if (unq(key(S\[j\], "if")) != want)/if (key(S[j], "if") != want)/ m26_dash_at_si_ends s/(ind(L) < si || (ind(L) == si && L !~ \/^ \*- \/))/ind(L) <= si/ m27_any_repo_exempt s/ || other(S\[k\])) continue/ || S[k] ~ \/repository:\/) continue/ m28_literal_self_exempt s/ && tolower(r) != "paiml\/aprender")/)/ -m29_local_action_ok s/if (unq(key(S\[k\], "uses")) ~ \/^\\.\\\/\/) {/if (0) {/' +m29_local_action_ok s/if (unq(key(S\[k\], "uses")) ~ \/^\\.\\\/\/) {/if (0) {/ +m30_repo_comment_kept s/r = bare(a\[i\]); sub(\/\[\[:space:\]\]+#\.\*\/, "", r); r = unq(r)/r = unq(bare(a[i]))/ +m31_case_sensitive s/if (tolower(c) ~/if (c ~/ +m32_no_bracket s/github(\\.|\\\[\["\\047\])/github\\./' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -358,7 +368,7 @@ main() { case "${1:-}" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,30p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,31p' "$SCRIPT_PATH"; return 0 ;; --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; '') ;; *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; From 5da18cd6b3e38a0855d670bde6af3278cd21eacc Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 20:53:05 +0200 Subject: [PATCH 13/34] chore(contracts): regenerate the generated set for nightly-pick-v1 The serial contract lane's turn for this branch: one new contract file, so the README CONTRACT_COUNT blocks go 1893 -> 1894, contracts/contracts.nt gains the five nightly-pick-v1 triples, and pv-sat rewrites the witness (census id-set hash and reasoner sha). Fixed points asserted: pv extract contracts --check, readme_sync.sh --check, and pv lint gates ont-consistency, refines and bindings, all 0. Agent: aprender-a7f Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- contracts/contracts.nt | 5 +++++ ...1afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cfb1ab2c8a..f917dce40b 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1893** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1894** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -348,7 +348,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1893 contracts across inference, training, quantization, attention, FFN, +The tree carries 1894 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/contracts.nt b/contracts/contracts.nt index 867942481c..09e7e06e23 100644 --- a/contracts/contracts.nt +++ b/contracts/contracts.nt @@ -12246,6 +12246,11 @@ "contracts/nightly-greens-v1.yaml"^^ . "nightly-greens-v1"^^ . "pattern"^^ . + . + . + "contracts/nightly-pick-v1.yaml"^^ . + "nightly-pick-v1"^^ . + "pattern"^^ . . . "contracts/nightly-train-v1.yaml"^^ . diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index cf13cc312a..8f10905d79 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "13631c57f19bc4e519fbbed1bb3283c96d4e2695335129223d27222b4a921454", + "census_id_set_sha256": "76a6f73053b3d1619c77fc2745bcada4bfa00afa536ced586c8c54e82476a2bd", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "9d1ddc0e1dc272d80a0ff9cdcf0fbab013ce2570", + "reasoner_git_sha": "7874b1f07e53da51f9fe8a414cede064f75bbe6c", "checkable_n": 84, "result": { "kind": "model", @@ -10,5 +10,5 @@ } }, "pc_reasoner": "fired", - "cpu_ms": 113 + "cpu_ms": 1 } From 6e434b257fa975f55e9d77114b1acd2348fac88c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 21:03:47 +0200 Subject: [PATCH 14/34] T44: the train reads a chained producer by its workflow_run runs, and the guard keeps LANES in step After PR-A (#4731) the train's LANES table read every nightly producer by `^schedule$`. T44 removes those crons: the 15 producers now start on the "Nightly pick" workflow_run. Left alone, every one of those lanes would read not_measured for good. This lands atomically with the trigger change. - nightly_train.sh: the 15 chained lanes read `^workflow_run$`; fleet-toolset (still on its own cron) stays `^schedule$`. Two rows: a chained lane is green on its workflow_run run and not_measured on a schedule run. Mutant m43 (any event counted). The header states the limit: a chained run is filed under main's head when the pick finished while its tree is the pick's commit; they differ only when a merge lands while the pick runs. - check_producers_chain.sh P6: a listed producer whose lane is not exactly `^workflow_run$`, a lane reading workflow_run runs of an unlisted workflow, a missing train or no LANES table is RED. 6 rows, 5 mutants. The mutation harness now copies the train into its directory and first runs the unmutated copy there: without that, every row failed in the mutant dir and all 37 kills were vacuous. - contracts/nightly-train-v1.yaml: description, FALSIFY-NT-001 and -002 predictions name the chain. Receipts: - bash scripts/release/nightly_train.sh --self-test: 59/59 rows - bash scripts/release/nightly_train.sh --mutants: 43/43 killed - bash scripts/release/check_producers_chain.sh: PASS on this tree - bash scripts/release/check_producers_chain.sh --self-test: 50 rows; --mutants: 37/37 killed, errors 0 - pv validate contracts/nightly-train-v1.yaml: 0 errors, 0 warnings - pv lint contracts --gate sigma: Pass, formal_prose 1463 - bashrs lint: 0 errors on both scripts; +1 / +7 warnings, all parser false positives inside awk text or fixture paths, of kinds the files already carry ont-delta: none (prose fields of one contract; no shape or type change) Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-train-v1.yaml | 6 +-- scripts/release/check_producers_chain.sh | 54 ++++++++++++++++++++++-- scripts/release/nightly_train.sh | 46 ++++++++++++-------- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index 6c3213ef20..49279e8643 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' scheduled producers recorded for C, and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' nightly producers recorded for C (a producer chained from the nightly pick is read by its workflow_run runs only), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" @@ -120,13 +120,13 @@ proof_obligations: falsification_tests: - id: FALSIFY-NT-001 rule: "The judge is right on every row of its case table" - prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox" + prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run" test: "bash scripts/release/nightly_train.sh --self-test" if_fails: "The train can print RELEASABLE for a commit some release-day check never measured, or stay silent when it fails" - id: FALSIFY-NT-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit) turns at least one row RED, after the unmutated copy is shown green in the same directory" + prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit, a run of any event counted) turns at least one row RED, after the unmutated copy is shown green in the same directory" test: "bash scripts/release/nightly_train.sh --mutants" if_fails: "The case table is theater: it would pass a judge that calls main releasable on evidence it never read" diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 1279d94ec1..d4d7e63597 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -18,7 +18,9 @@ # calls a reusable workflow (`uses:` at job level), whose checkout this file cannot see; # P4 names github.sha, GITHUB_SHA, github.workflow_sha or GITHUB_WORKFLOW_SHA on a live line that names neither # NIGHTLY_C nor workflow_run.head_sha outside a trailing comment; -# P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways). +# P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways); +# P6 is read by a lane of scripts/release/nightly_train.sh's LANES table whose event pattern is not exactly +# ^workflow_run$ (its scheduled runs are gone), or a lane reads workflow_run runs of a workflow not listed. # Limits: it reads the workflow text, not the scripts a step calls, so a later run step that fetches or checks out # main again is not seen; an origin/main a step fetches as a declared comparand (guards-nightly's SATD ceiling) is # not a measurement and is not judged here. It reads `on:` as a block mapping with keys at indent 2; any other @@ -144,6 +146,26 @@ judge_file() { }' "$1" } +p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by its workflow_run runs, and only those + local t="$1/scripts/release/nightly_train.sh" + [ -f "$t" ] || { printf 'FAIL nightly_train.sh: P6 missing, so its LANES table cannot be read\n'; return 1; } + awk -F ';' -v prods=" $(printf '%s' "$PRODUCERS" | tr '\n' ' ') " ' + /^LANES=\047/ { inl = 1; sub(/^LANES=\047/, "") } + inl { + last = ($0 ~ /\047$/); line = $0; sub(/\047$/, "", line); split(line, a, ";") + if (a[4] ~ /^\.github\/workflows\/[^\/]+\.yml$/) { + w = a[4]; sub(/^.*\//, "", w); sub(/\.yml$/, "", w); lanes++ + if (index(prods, " " w " ") && a[5] != "^workflow_run$") { + printf "FAIL nightly_train.sh: P6 lane %s reads %s.yml by event %s; it chains from the pick, so ^workflow_run$\n", a[1], w, a[5]; bad = 1 } + if (!index(prods, " " w " ") && a[5] ~ /workflow_run/) { + printf "FAIL nightly_train.sh: P6 lane %s reads workflow_run runs of %s.yml, which is not a listed producer\n", a[1], w; bad = 1 } + } + if (last) { inl = 0; done = 1 } + } + END { if (!done || !lanes) { print "FAIL nightly_train.sh: P6 no LANES table read"; bad = 1 } + exit bad }' "$t" +} + judge() { # ROOT local root="$1" wf pick p rc=0 out listed wf="$root/.github/workflows" @@ -163,6 +185,7 @@ judge() { # ROOT [ -n "$out" ] && printf '%s\n' "$out" printf '%s\n' "$out" | grep -q -e '^FAIL' && rc=1 done + p6 "$root" || rc=1 if [ "$rc" -eq 0 ]; then printf 'PASS every nightly producer chains from "%s" and measures its C\n' "$pick" else printf 'RED a nightly producer is not chained to the pick, or does not measure its C\n'; fi return "$rc" @@ -175,6 +198,7 @@ fixture() { # DIR: the real producers + nightly-pick.yml rm -rf -- "${1:?}"; mkdir -p "$1/.github/workflows" || return 1 local p for p in $PRODUCERS nightly-pick; do cp "$REAL_ROOT/.github/workflows/$p.yml" "$1/.github/workflows/" || return 1; done + mkdir -p "$1/scripts/release" && cp "$REAL_ROOT/scripts/release/nightly_train.sh" "$1/scripts/release/" || return 1 } row() { # NAME WANT-RC MUST -- MUTATION-FN (run inside the fixture's workflows dir) local name="$1" want="$2" must="$3" fx out rc @@ -211,6 +235,13 @@ m_chain_branch2() { changed cuda-nightly.yml 's/^ branches: \[main\]$/ bra m_chain_cmt() { changed cuda-nightly.yml 's/^ workflows:/ # workflows:/'; } m_pick_renamed() { changed nightly-pick.yml 's/^name: Nightly pick$/name: Nightly Pick/'; } m_missing() { rm -f -- silicon-nightly.yml; } +NT=../../scripts/release/nightly_train.sh +m_lane_sched() { changed "$NT" '/^book;/s/;\^workflow_run\$;/;^schedule$;/'; } +m_lane_both() { changed "$NT" '/^cuda;/s/;\^workflow_run\$;/;^(schedule|workflow_run)$;/'; } +m_lane_unlisted() { changed "$NT" '/^fleet-toolset;/s/;\^schedule\$;/;^workflow_run$;/'; } +m_train_pre_t44() { changed "$NT" 's/;\^workflow_run\$;/;^schedule$;/g'; } +m_train_gone() { rm -f -- "$NT"; } +m_lanes_renamed() { changed "$NT" 's/^LANES=\x27/LANEZ=\x27/'; } m_unlisted() { synth extra.yml <<< ' - run: true'; } m_no_cstep() { changed toolchain-ceiling.yml '/- name: Measure the night.s C, not main.s head (T44)/,/nightly_c_checkout.sh/d'; } m_cstep_or() { changed install-script.yml "0,/if: github.event_name == 'workflow_run'\$/s//if: github.event_name == 'workflow_run' || true/"; } @@ -292,6 +323,12 @@ self_test() { row p3_repository_this_repo_cmt 1 'is not followed by the C step' -- m_repo_self_cmt row p4_github_sha_bracket 1 'qwen-story-daily.yml: P4' -- m_sha_bracket row p4_github_sha_case 1 'qwen-story-daily.yml: P4' -- m_sha_case + row p6_lane_still_schedule 1 'P6 lane book reads book.yml' -- m_lane_sched + row p6_lane_both_events 1 'P6 lane cuda reads cuda-nightly.yml' -- m_lane_both + row p6_unlisted_reads_chain 1 'P6 lane fleet-toolset reads workflow_run' -- m_lane_unlisted + row p6_train_before_t44 1 'P6 lane deep-examples reads' -- m_train_pre_t44 + row p6_train_missing 1 'P6 missing' -- m_train_gone + row p6_no_lanes_table 1 'P6 no LANES table read' -- m_lanes_renamed rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -328,7 +365,12 @@ m28_literal_self_exempt s/ && tolower(r) != "paiml\/aprender")/)/ m29_local_action_ok s/if (unq(key(S\[k\], "uses")) ~ \/^\\.\\\/\/) {/if (0) {/ m30_repo_comment_kept s/r = bare(a\[i\]); sub(\/\[\[:space:\]\]+#\.\*\/, "", r); r = unq(r)/r = unq(bare(a[i]))/ m31_case_sensitive s/if (tolower(c) ~/if (c ~/ -m32_no_bracket s/github(\\.|\\\[\["\\047\])/github\\./' +m32_no_bracket s/github(\\.|\\\[\["\\047\])/github\\./ +m33_p6_not_called s/^ p6 "\$root" || rc=1$/ :/ +m34_listed_any_event s/ && a\[5\] != "^workflow_run\$")/ \&\& 0)/ +m35_unlisted_any_event s/ && a\[5\] ~ \/workflow_run\/)/ \&\& 0)/ +m36_missing_train_read s/\[ -f "\$t" \] ||/[ 1 ] ||/ +m37_no_table_ok s/if (!done || !lanes) {/if (0) {/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -336,6 +378,12 @@ mutants() { tmp="$(mktemp -d "${TMPDIR:-/tmp}/pc-mu.XXXXXX")" || caller_error "no temp dir" mkdir -p "$tmp/scripts/release" "$tmp/.github/workflows" cp "$REAL_ROOT"/.github/workflows/*.yml "$tmp/.github/workflows/" || caller_error "cannot copy the workflows" + cp "$REAL_ROOT/scripts/release/nightly_train.sh" "$tmp/scripts/release/" || caller_error "cannot copy nightly_train.sh" + # baseline: the unmutated copy must pass here, or every kill below is the mutant dir's fault, not the mutant's + cp "$SCRIPT_PATH" "$tmp/scripts/release/check_producers_chain.sh" + if ! bash "$tmp/scripts/release/check_producers_chain.sh" --self-test > "$tmp/base" 2>&1; then + printf 'ERROR baseline: the unmutated copy fails in the mutant dir\n'; grep -e '^RED' "$tmp/base" | head -n 3; rm -rf -- "${tmp:?}"; return 1 + fi while IFS="$(printf '\t')" read -r name expr; do [ -n "$name" ] || continue total=$((total + 1)) @@ -368,7 +416,7 @@ main() { case "${1:-}" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,31p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,33p' "$SCRIPT_PATH"; return 0 ;; --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; '') ;; *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index 9fd3e91d3b..905134ca25 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -10,8 +10,12 @@ # and every line ends with [C= pin=..]: the commit judged and the shas of the three # scripts that judged it. H is the commit id of C, main's head at the moment of the read. # -# READ, DON'T RE-RUN. Night 1 runs nothing. Each lane takes the result its existing scheduled producer (a workflow on -# main and, optionally, a job-name pattern) recorded for C. A lane is +# READ, DON'T RE-RUN. Night 1 runs nothing. Each lane takes the result its existing nightly producer (a workflow on +# main and, optionally, a job-name pattern) recorded for C. A producer that chains from "Nightly pick" is read by +# its workflow_run runs only (scripts/release/check_producers_chain.sh P6 keeps this table in step with the +# triggers). Such a run is filed under main's head when the pick finished, while its tree is the pick's commit +# (nightly_c_checkout.sh); the two differ only when a merge lands while the pick runs, and the lane is then +# credited with the pick's commit's result. A lane is # green the newest completed run of its producer on C ran every matched job to success, at attempt 1; # red a matched job ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 # or later (a retried green is not a green: FLOW-003 G4, CI retries 2 -> 0); @@ -70,8 +74,8 @@ UNIT="aprender-nightly-train" LANES='ci-main;verdict;ci / gate + workspace-test (merge, autopilot wait);.github/workflows/ci.yml;^push$;^(ci / gate|workspace-test)$;2 deep-doctests;verdict;autopilot deep: cargo test --doc --workspace;-;-;- deep-nodefault;verdict;autopilot deep: cargo check --workspace --no-default-features;-;-;- -deep-examples;verdict;autopilot deep: cargo build --workspace --examples;.github/workflows/examples-nightly.yml;^schedule$;^examples$ -deep-bins-build;verdict;autopilot deep: all-bins cargo build --locked --release;.github/workflows/nightly.yml;^schedule$;-unknown-linux-gnu on +deep-examples;verdict;autopilot deep: cargo build --workspace --examples;.github/workflows/examples-nightly.yml;^workflow_run$;^examples$ +deep-bins-build;verdict;autopilot deep: all-bins cargo build --locked --release;.github/workflows/nightly.yml;^workflow_run$;-unknown-linux-gnu on deep-bins-smoke;verdict;autopilot deep: nightly_manifest.py smoke;-;-;- dogfood;verdict;dogfood.sh --phase pre-publish + preflight R5;-;-;- models;verdict;models_t1.sh GPU-host ladder legs, preflight R7;-;-;- @@ -82,19 +86,19 @@ cleanroom-gpu;verdict;b2-gpu.yml on the tag;-;-;- assets;verdict;binary-release.yml + check_release_assets.sh;-;-;- preflight;verdict;check_publish_preflight.sh R1-R8;-;-;- publish-dryrun;verdict;rc_publish_gate.sh --verify + cascade-publish.sh --check;-;-;- -coverage;info;tag_coverage_gate.sh (C291.1: not gating);.github/workflows/coverage-nightly.yml;^schedule$;^coverage$ -guards;info;guards-nightly;.github/workflows/guards-nightly.yml;^schedule$; -mutants;info;mutants-nightly;.github/workflows/mutants-nightly.yml;^schedule$; -toolchain-ceiling;info;toolchain-ceiling;.github/workflows/toolchain-ceiling.yml;^schedule$; -cuda;info;cuda-nightly;.github/workflows/cuda-nightly.yml;^schedule$; -silicon;info;silicon-nightly;.github/workflows/silicon-nightly.yml;^schedule$; -qwen-story;info;qwen-story-daily;.github/workflows/qwen-story-daily.yml;^schedule$; -beat-speed;info;beat-speed-nightly;.github/workflows/beat-speed-nightly.yml;^schedule$; -conleche;info;conleche-nightly;.github/workflows/conleche-nightly.yml;^schedule$; -bench;info;nightly-bench;.github/workflows/nightly-bench.yml;^schedule$; -book;info;book;.github/workflows/book.yml;^schedule$; -book-contracts;info;book-contracts;.github/workflows/book-contracts.yml;^schedule$; -install-script;info;install-script;.github/workflows/install-script.yml;^schedule$; +coverage;info;tag_coverage_gate.sh (C291.1: not gating);.github/workflows/coverage-nightly.yml;^workflow_run$;^coverage$ +guards;info;guards-nightly;.github/workflows/guards-nightly.yml;^workflow_run$; +mutants;info;mutants-nightly;.github/workflows/mutants-nightly.yml;^workflow_run$; +toolchain-ceiling;info;toolchain-ceiling;.github/workflows/toolchain-ceiling.yml;^workflow_run$; +cuda;info;cuda-nightly;.github/workflows/cuda-nightly.yml;^workflow_run$; +silicon;info;silicon-nightly;.github/workflows/silicon-nightly.yml;^workflow_run$; +qwen-story;info;qwen-story-daily;.github/workflows/qwen-story-daily.yml;^workflow_run$; +beat-speed;info;beat-speed-nightly;.github/workflows/beat-speed-nightly.yml;^workflow_run$; +conleche;info;conleche-nightly;.github/workflows/conleche-nightly.yml;^workflow_run$; +bench;info;nightly-bench;.github/workflows/nightly-bench.yml;^workflow_run$; +book;info;book;.github/workflows/book.yml;^workflow_run$; +book-contracts;info;book-contracts;.github/workflows/book-contracts.yml;^workflow_run$; +install-script;info;install-script;.github/workflows/install-script.yml;^workflow_run$; fleet-toolset;info;fleet-toolset;.github/workflows/fleet-toolset.yml;^schedule$;' caller_error() { printf 'NOT RELEASABLE: nightly-train, caller error: %s\n' "$*"; exit 3; } # ---------------------------------------------------------------- judgement (pure: files in, files out) ---------- @@ -556,6 +560,11 @@ self_test() { row all_verdict_lanes_green_is_releasable 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" d="$tmp/removed"; fixture "$d"; sed -i '/\t201\t/d' "$d/runs.tsv" row one_lane_removed_names_it 0 "NOT RELEASABLE: v-b, not_measured" "RELEASABLE H=" -- st_decide "$d" + st_chain() { decide "$(printf '%s\n' "$ST_LANES" | sed -e '/^v-b;/s/\^schedule\$/^workflow_run$/')" "$1" 2026-10-04T06:00:00Z; cat "$1/line"; } + d="$tmp/chain"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv" + row a_chained_lane_reads_its_workflow_run_run 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_chain "$d" + d="$tmp/chainsched"; fixture "$d" + row a_chained_lane_ignores_a_schedule_run 0 "NOT RELEASABLE: v-b, not_measured" "RELEASABLE H=" -- st_chain "$d" d="$tmp/other"; fixture "$d"; sed -i "/\t101\t/s/$ST_C/$ST_X/" "$d/runs.tsv" row run_on_another_commit_is_not_measured 0 "newest run 101 on bbbbbbbbbb" "RELEASABLE H=" -- st_decide "$d" d="$tmp/running"; fixture "$d"; sed -i '/\t101\t/s/\tCOMPLETED\tSUCCESS\tjob-a/\tIN_PROGRESS\t\tjob-a/' "$d/runs.tsv" @@ -767,7 +776,8 @@ m38_verdict_rc_always_red s/"RELEASABLE "\*) return 0 ;;/"RELEASABLE "*) return m39_exit_verdict_always_red s/^ \[ -z "\$EXIT_VERDICT" \] || verdict_rc "\$line" || exit 1$/ [ -z "$EXIT_VERDICT" ] || exit 1/ m40_config_token_ignored s/^ if grep -qsE / if false \&\& grep -qsE / m41_home_cargo_unchecked s/ "\$HOME\/.cargo"; do$/; do/ -m42_floor_ignores_the_limit s/fl=\$((lim \/ 5))/fl=$RATE_FLOOR/' +m42_floor_ignores_the_limit s/fl=\$((lim \/ 5))/fl=$RATE_FLOOR/ +m43_any_event_counts s/ && REV\[r\] ~ EV\[i\]//' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 3ea04cf28452d1e849a6b724e799508e68845c3c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 21:45:17 +0200 Subject: [PATCH 15/34] T44: a chained run counts only when its night picked C A chained producer's run is filed under main's head at the time the pick finished, but its tree is the night's pick. A pick dispatched again later keeps the older C and still fires with the newer head, so the train could credit main's head with evidence measured on an older commit. nightly_train.sh (rows 59 -> 70, mutants 43 -> 54, all killed): - The one GraphQL query also reads refs/heads/nightly/*. A chained lane counts a run only when the pick for the night that holds the run's creation time, [N 12:00Z, N+1 12:00Z), is C. An unread pick reads not_measured. - A run created 12:00Z-17:59Z also needs the night before to be C, because the producer takes its night from the pick's start. The scheduled pick fires at 20:30Z. The header states the remaining limit. - --from carries picks.tsv, and only dated ref names count as nights. check_producers_chain.sh P6 (rows 50 -> 52, mutants 37 -> 39): a listed producer with no lane is RED, and so is a second LANES assignment. Refs #4749 Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-train-v1.yaml | 6 +- scripts/release/check_producers_chain.sh | 19 ++++-- scripts/release/nightly_train.sh | 79 +++++++++++++++++++++--- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index 49279e8643..e94ce960b1 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' nightly producers recorded for C (a producer chained from the nightly pick is read by its workflow_run runs only), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' nightly producers recorded for C (a producer chained from the nightly pick is read by its workflow_run runs only, and only when the pick of the night the run was created in, and of the night before for a run created 12:00Z-17:59Z, is C), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" @@ -120,13 +120,13 @@ proof_obligations: falsification_tests: - id: FALSIFY-NT-001 rule: "The judge is right on every row of its case table" - prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run" + prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run, on a run whose night picked another commit or whose pick was not read, and on a run created 12:00Z-17:59Z whose night before picked another commit; a schedule lane ignores the picks; the query reads refs/heads/nightly/ and only dated night names count" test: "bash scripts/release/nightly_train.sh --self-test" if_fails: "The train can print RELEASABLE for a commit some release-day check never measured, or stay silent when it fails" - id: FALSIFY-NT-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit, a run of any event counted) turns at least one row RED, after the unmutated copy is shown green in the same directory" + prediction: "Each planted mutant (not_measured counted green, a run on any commit, an unfinished run counted, a retried green counted, a failed read counted, an unread attempt counted, no trap, info lanes voting, ranking reversed, total counted as streak, coverage made a verdict lane, a verdict lane dropped, the rollup dropped, a re-run job not counted, a missing producer not refused, the required-job count ignored, a cut job list read, a void run picked, an edited bundle run, a producerless lane green, a second history row after the first, the caller's inbox inherited by the self-test, a drill writing the named inbox, one leg hiding another, a cut rollup read, the day set after the pin step, --pinned not requiring PIN, unhashed PIN accepted, the rate floor ignored, the call budget ignored, GraphQL errors read, an inbox mismatch exiting 0, an inferred info attempt claimed as 1, the token guard dropped, the credentials file ignored, a named registry token ignored, --exit-verdict ignored, a RELEASABLE verdict exiting 1, every verdict exiting 1 under the flag, a config token ignored, ~/.cargo left unchecked, the floor ignoring the limit, a run of any event counted, a chained run counted without its night's pick, any night's pick counted, a night window shifted at either end, the night-before check dropped or shortened or read from the same night, every lane checking the picks, the picks not carried by --from, any ref name read as a night, the picks dropped from the query) turns at least one row RED, after the unmutated copy is shown green in the same directory" test: "bash scripts/release/nightly_train.sh --mutants" if_fails: "The case table is theater: it would pass a judge that calls main releasable on evidence it never read" diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index d4d7e63597..93c8d27c81 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -20,7 +20,8 @@ # NIGHTLY_C nor workflow_run.head_sha outside a trailing comment; # P5 is listed below and missing, or chains from the pick and is not listed (the list is complete both ways); # P6 is read by a lane of scripts/release/nightly_train.sh's LANES table whose event pattern is not exactly -# ^workflow_run$ (its scheduled runs are gone), or a lane reads workflow_run runs of a workflow not listed. +# ^workflow_run$ (its scheduled runs are gone), or a lane reads workflow_run runs of a workflow not listed, +# or a listed producer has no lane, or LANES is assigned more than once. # Limits: it reads the workflow text, not the scripts a step calls, so a later run step that fetches or checks out # main again is not seen; an origin/main a step fetches as a declared comparand (guards-nightly's SATD ceiling) is # not a measurement and is not judged here. It reads `on:` as a block mapping with keys at indent 2; any other @@ -150,11 +151,12 @@ p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by it local t="$1/scripts/release/nightly_train.sh" [ -f "$t" ] || { printf 'FAIL nightly_train.sh: P6 missing, so its LANES table cannot be read\n'; return 1; } awk -F ';' -v prods=" $(printf '%s' "$PRODUCERS" | tr '\n' ' ') " ' - /^LANES=\047/ { inl = 1; sub(/^LANES=\047/, "") } + /^LANES=\047/ { assigned++; inl = 1; sub(/^LANES=\047/, "") } inl { last = ($0 ~ /\047$/); line = $0; sub(/\047$/, "", line); split(line, a, ";") if (a[4] ~ /^\.github\/workflows\/[^\/]+\.yml$/) { w = a[4]; sub(/^.*\//, "", w); sub(/\.yml$/, "", w); lanes++ + if (index(prods, " " w " ")) seen[w] = 1 if (index(prods, " " w " ") && a[5] != "^workflow_run$") { printf "FAIL nightly_train.sh: P6 lane %s reads %s.yml by event %s; it chains from the pick, so ^workflow_run$\n", a[1], w, a[5]; bad = 1 } if (!index(prods, " " w " ") && a[5] ~ /workflow_run/) { @@ -163,6 +165,9 @@ p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by it if (last) { inl = 0; done = 1 } } END { if (!done || !lanes) { print "FAIL nightly_train.sh: P6 no LANES table read"; bad = 1 } + if (assigned > 1) { print "FAIL nightly_train.sh: P6 LANES is assigned more than once, so the table read may not be the live one"; bad = 1 } + np = split(prods, pp, " "); for (k = 1; k <= np; k++) if (!(pp[k] in seen)) { + printf "FAIL nightly_train.sh: P6 listed producer %s.yml has no lane, so the train never reads it\n", pp[k]; bad = 1 } exit bad }' "$t" } @@ -242,6 +247,8 @@ m_lane_unlisted() { changed "$NT" '/^fleet-toolset;/s/;\^schedule\$;/;^workflow_ m_train_pre_t44() { changed "$NT" 's/;\^workflow_run\$;/;^schedule$;/g'; } m_train_gone() { rm -f -- "$NT"; } m_lanes_renamed() { changed "$NT" 's/^LANES=\x27/LANEZ=\x27/'; } +m_lane_dropped() { changed "$NT" '/^book;info;book;/d'; } +m_lanes_twice() { printf '%s\n' "LANES='book;info;book;.github/workflows/book.yml;^workflow_run$;'" >> "$NT"; } m_unlisted() { synth extra.yml <<< ' - run: true'; } m_no_cstep() { changed toolchain-ceiling.yml '/- name: Measure the night.s C, not main.s head (T44)/,/nightly_c_checkout.sh/d'; } m_cstep_or() { changed install-script.yml "0,/if: github.event_name == 'workflow_run'\$/s//if: github.event_name == 'workflow_run' || true/"; } @@ -329,6 +336,8 @@ self_test() { row p6_train_before_t44 1 'P6 lane deep-examples reads' -- m_train_pre_t44 row p6_train_missing 1 'P6 missing' -- m_train_gone row p6_no_lanes_table 1 'P6 no LANES table read' -- m_lanes_renamed + row p6_listed_without_a_lane 1 'P6 listed producer book.yml has no lane' -- m_lane_dropped + row p6_lanes_assigned_twice 1 'P6 LANES is assigned more than once' -- m_lanes_twice rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -370,7 +379,9 @@ m33_p6_not_called s/^ p6 "\$root" || rc=1$/ :/ m34_listed_any_event s/ && a\[5\] != "^workflow_run\$")/ \&\& 0)/ m35_unlisted_any_event s/ && a\[5\] ~ \/workflow_run\/)/ \&\& 0)/ m36_missing_train_read s/\[ -f "\$t" \] ||/[ 1 ] ||/ -m37_no_table_ok s/if (!done || !lanes) {/if (0) {/' +m37_no_table_ok s/if (!done || !lanes) {/if (0) {/ +m38_a_listed_lane_may_be_missing s/if (!(pp\[k\] in seen)) {/if (0) {/ +m39_a_second_table_ok s/if (assigned > 1) {/if (0) {/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -416,7 +427,7 @@ main() { case "${1:-}" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,33p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,34p' "$SCRIPT_PATH"; return 0 ;; --root) root="${2:-}"; [ -d "$root" ] || caller_error "--root DIR" ;; '') ;; *) caller_error "unknown argument '$1' (--root DIR|--self-test|--mutants)" ;; diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index 905134ca25..f665059cb0 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -13,9 +13,13 @@ # READ, DON'T RE-RUN. Night 1 runs nothing. Each lane takes the result its existing nightly producer (a workflow on # main and, optionally, a job-name pattern) recorded for C. A producer that chains from "Nightly pick" is read by # its workflow_run runs only (scripts/release/check_producers_chain.sh P6 keeps this table in step with the -# triggers). Such a run is filed under main's head when the pick finished, while its tree is the pick's commit -# (nightly_c_checkout.sh); the two differ only when a merge lands while the pick runs, and the lane is then -# credited with the pick's commit's result. A lane is +# triggers). Such a run is filed under main's head when the pick finished, while its tree is the night's pick +# (nightly_c_checkout.sh), and the two differ when a merge lands while the pick runs or a pick is dispatched again +# later (it keeps the older C). So the run counts only when the pick of its night, refs/heads/nightly/ for the +# window [N 12:00Z, N+1 12:00Z) that holds the run's creation time, is C; an unread pick is not_measured. The +# producer takes its night from the pick's start, this check from the run's creation, so a run created 12:00Z-17:59Z +# also needs the night before to be C (the scheduled pick fires at 20:30Z). Limit: a pick that starts before noon +# and finishes after 18:00Z is judged by the later night. A lane is # green the newest completed run of its producer on C ran every matched job to success, at attempt 1; # red a matched job ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 # or later (a retried green is not a green: FLOW-003 G4, CI retries 2 -> 0); @@ -109,9 +113,18 @@ evaluate() { c="$(cat "$2/C" 2>/dev/null)"; rd="$(cat "$2/read" 2>/dev/null)" [ -f "$2/attempts.tsv" ] || : > "$2/attempts.tsv" [ -f "$2/runs.tsv" ] || : > "$2/runs.tsv" + [ -f "$2/picks.tsv" ] || : > "$2/picks.tsv" + # each picked night as the window of run creation times it owns, [N 12:00Z, N+1 12:00Z), and its C + local n o e + while IFS=$'\t' read -r n o; do + case "$n" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) continue ;; esac + e="$(date -u -d "$n + 1 day" +%F 2>/dev/null)" || continue + printf '%sT12:00:00Z\t%sT12:00:00Z\t%s\n' "$n" "$e" "$o" + done < "$2/picks.tsv" > "$2/pickwin.tsv" awk -F '\t' -v OFS='\t' -v C="$c" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" ' FILENAME == ARGV[1] { split($0, a, ";"); n++; L[n] = a[1]; K[n] = a[2]; CK[n] = a[3]; WF[n] = a[4]; EV[n] = a[5]; RE[n] = a[6]; NEED[n] = (a[7] ~ /^[0-9]+$/ ? a[7] + 0 : 1); next } FILENAME == ARGV[2] { AT[$1] = $2; next } + FILENAME == ARGV[3] { np++; PS[np] = $1; PE[np] = $2; PO[np] = $3; next } { key = $3 SUBSEP $10 SUBSEP $13 if (key in seen) next @@ -119,6 +132,11 @@ evaluate() { if (!(r in RW)) { RW[r] = $2; REV[r] = $4; RBR[r] = $5; RH[r] = $6; RC[r] = $7; RST[r] = toupper($8); RCO[r] = toupper($9); nr++; RID[nr] = r } if ($10 != "") { nj[r]++; k = nj[r]; JN[r, k] = $10; JS[r, k] = toupper($11); JC[r, k] = toupper($12); JB[r, k] = $13; JE[r, k] = $14; JD[r, k] = $15 } } + # nightk(t) -> the picked night whose window holds creation time t (0: no pick read for it); nightc(t) -> its C; + # prevc(t) -> the C of the night before it + function nightk(t, k) { for (k = 1; k <= np; k++) if (t >= PS[k] && t < PE[k]) return k; return 0 } + function nightc(t, k) { k = nightk(t); return (k ? PO[k] : "") } + function prevc(t, k, j) { k = nightk(t); if (!k) return ""; for (j = 1; j <= np; j++) if (PE[j] == PS[k]) return PO[j]; return "" } # lane state of run r for lane i: green | red | pending | void; sets WHY, ST, EN, DUP function lst(i, r, k, b, fb, nm, pend, bad, ok, tot) { split("", b); split("", fb); ST = ""; EN = ""; DUP = 0; WHY = "" @@ -168,6 +186,11 @@ evaluate() { print L[i], r, RC[r], "main", REV[r], hc, ha > HIST } if (RH[r] != C) { if (other == "") other = r " on " substr(RH[r], 1, 10); continue } + # a chained run is filed under the head of main when the pick finished, but measured the pick of its night: count it + # only when that pick is C (a re-dispatched pick keeps an older C and still fires with the newer head) + if (EV[i] == "^workflow_run$" && (pc = nightc(RC[r])) != C) { if (other == "") other = r " measured the pick " (pc == "" ? "unread" : substr(pc, 1, 10)); continue } + # a run created 12:00Z-17:59Z may come from a pick that started before noon: the night before must be C too + if (EV[i] == "^workflow_run$" && substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && (pc = prevc(RC[r])) != C) { if (other == "") other = r " may have measured the pick " (pc == "" ? "unread" : substr(pc, 1, 10)); continue } if (pick == "" && s != "void") { pick = r; pst = s; pwhy = WHY; pS = ST; pE = EN; pD = DUP } else if (pick == "" && pwhy == "") pwhy = WHY } @@ -195,7 +218,7 @@ evaluate() { if (att > 1) out(i, "red", pick, att, "success only at attempt " att ": a retried green is not a green") else out(i, "green", pick, att, "") } - }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/runs.tsv" + }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/pickwin.tsv" "$2/runs.tsv" } # hours LANESOUT REDAGE_OUTPUT_FILE -> lanehours lost (2 decimals) for every lane red_age measured hours() { @@ -255,9 +278,11 @@ gql_query() { | ($wf | first | .workflows | map(select(.path as $x | $want | index($x))) ) as $hit | if ($hit | length) != ($want | length) then error("producer workflow missing from the list") else . end | ($repo | split("/")) as $own - | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } } } " + | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } } " + ([$hit | to_entries[] | "w\(.key): node(id: \"\(.value.node_id)\") { ... on Workflow { id runs(first: 12) { nodes { databaseId createdAt event checkSuite { status conclusion branch { name } commit { oid } checkRuns(first: 100, filterBy: {checkType: ALL}) { pageInfo { hasNextPage } nodes { name status conclusion startedAt completedAt } } } } } } }"] | join(" ")) + " }"' 2>/dev/null } +# picks_of GQLJSON -> nightC for each refs/heads/nightly/ the query returned; other names are dropped +picks_of() { jq -r '(.data.repository.nightly.nodes // [])[] | select(.name | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) | [.name, .target.oid] | @tsv' "$1"; } normalize() { jq -r --slurpfile wf "$2" ' ($wf | first | .workflows | map({key: .node_id, value: .path}) | from_entries) as $p @@ -293,7 +318,7 @@ normalize() { # is recorded in RAW/read and makes every producer lane not_measured fetch() { local raw="$1" cache="$2" lim rem fl hdr st q a ids r - printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv" + printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv"; : > "$raw/picks.tsv" read -r lim rem <<< "$(gh api rate_limit --jq '"\(.resources.core.limit) \(.resources.core.remaining)"' 2>/dev/null)" case "$lim:$rem" in *[!0-9:]*|:*|*:) printf 'failed: rate_limit unreadable\n' > "$raw/read"; return 0 ;; esac # the floor is a fifth of the token's own hourly limit, capped at RATE_FLOOR: the fleet PAT (5000) keeps its 1000 @@ -320,6 +345,8 @@ fetch() { if jq -e '(.errors // []) | length > 0' "$raw/graphql.json" > /dev/null 2>&1; then printf 'failed: GraphQL errors\n' > "$raw/read"; return 0; fi jq -r '.data.repository.defaultBranchRef | select(.name == "main") | .target.oid // empty' "$raw/graphql.json" > "$raw/C" 2>/dev/null jq -r '.data.repository.defaultBranchRef.target.tree.oid // empty' "$raw/graphql.json" > "$raw/tree" 2>/dev/null + # the night's picks (refs/heads/nightly/): a chained lane counts a run only when its night picked C + picks_of "$raw/graphql.json" > "$raw/picks.tsv" 2>/dev/null grep -q -E '^[0-9a-f]{40}$' "$raw/C" || { printf 'failed: no head of main in the response\n' > "$raw/read"; return 0; } normalize "$raw/graphql.json" "$raw/workflows.json" > "$raw/runs.tsv" 2>/dev/null || { printf 'failed: response did not normalize\n' > "$raw/read"; return 0; } printf 'ok\n' > "$raw/read" @@ -409,7 +436,7 @@ run_train() { mkdir -p "$raw" || exit 1 step read if [ -n "$from" ]; then - for f in C read runs.tsv attempts.tsv tree; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done + for f in C read runs.tsv attempts.tsv tree picks.tsv; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done else fetch "$raw" "$OUTDIR/cache"; fi CSHA="$(cat "$raw/C" 2>/dev/null)" step judge @@ -537,6 +564,7 @@ fixture() { printf 'wf\t.github/workflows/d.yml\t401\tschedule\tmain\t%s\t2026-10-04T03:00:00Z\tCOMPLETED\tSUCCESS\td\tCOMPLETED\tSUCCESS\t2026-10-04T03:01:00Z\t2026-10-04T03:20:00Z\t1\n' "$ST_C" } > "$1/runs.tsv" printf '101\t1\n201\t1\n301\t1\n' > "$1/attempts.tsv" + printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_C" "$ST_X" > "$1/picks.tsv" # run 201 (02:00Z on the 4th) is night 2026-10-03 } # st_decide DIR -> the line, the lanes and the reds, as one text st_decide() { decide "$ST_LANES" "$1" 2026-10-04T06:00:00Z; cat "$1/line" "$1/lanes.tsv" "$1/reds.tsv"; } @@ -565,6 +593,29 @@ self_test() { row a_chained_lane_reads_its_workflow_run_run 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_chain "$d" d="$tmp/chainsched"; fixture "$d" row a_chained_lane_ignores_a_schedule_run 0 "NOT RELEASABLE: v-b, not_measured" "RELEASABLE H=" -- st_chain "$d" + st_chainl() { st_chain "$1"; cat "$1/lanes.tsv"; } + d="$tmp/repick"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; sed -i "1s/$ST_C/$ST_X/" "$d/picks.tsv" + row a_repicked_night_never_credits_main_head 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" + d="$tmp/nopick"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; : > "$d/picks.tsv" + row a_chained_run_without_a_read_pick_is_not_measured 0 "201 measured the pick unread" "RELEASABLE H=" -- st_chainl "$d" + d="$tmp/ownnight"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T13:00:00Z\t/' "$d/runs.tsv" + row a_chained_run_is_judged_by_its_own_night 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" + d="$tmp/atnoon"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T12:00:00Z\t/' "$d/runs.tsv" + row a_run_at_noon_belongs_to_the_new_night 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" + d="$tmp/opennight"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-03T12:00:00Z\t/' "$d/runs.tsv"; printf '2026-10-02\t%s\n' "$ST_C" >> "$d/picks.tsv" + row a_run_at_noon_opens_its_night 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_chain "$d" + d="$tmp/straddle"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T17:59:00Z\t/' "$d/runs.tsv" + printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_X" "$ST_C" > "$d/picks.tsv" + row a_run_after_noon_needs_the_night_before_too 0 "201 may have measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" + sed -i 's/\t2026-10-04T17:59:00Z\t/\t2026-10-04T18:00:00Z\t/' "$d/runs.tsv" + row a_run_from_six_pm_needs_its_own_night_only 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_chain "$d" + d="$tmp/schedpick"; fixture "$d"; sed -i "s/$ST_C/$ST_X/" "$d/picks.tsv" + row a_schedule_lane_ignores_the_picks 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" + st_picks() { printf '%s' '{"data":{"repository":{"nightly":{"nodes":[{"name":"2026-10-03","target":{"oid":"c3"}},{"name":"probe","target":{"oid":"p"}}]}}}}' > "$tmp/gqlp.json"; picks_of "$tmp/gqlp.json"; } + row picks_of_reads_dated_nights_only 0 "$(printf '2026-10-03\tc3')" "probe" -- st_picks + d="$tmp/fromchain"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv" + st_a_from_run_reads_the_picks() { (LANES="$(printf '%s\n' "$ST_LANES" | sed -e '/^v-b;/s/\^schedule\$/^workflow_run$/')"; NOW=2026-10-04T06:00:00Z; OUTDIR="$tmp/ofc"; INBOXF=""; run_train "$d"); } + row a_from_run_reads_the_picks 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_a_from_run_reads_the_picks d="$tmp/other"; fixture "$d"; sed -i "/\t101\t/s/$ST_C/$ST_X/" "$d/runs.tsv" row run_on_another_commit_is_not_measured 0 "newest run 101 on bbbbbbbbbb" "RELEASABLE H=" -- st_decide "$d" d="$tmp/running"; fixture "$d"; sed -i '/\t101\t/s/\tCOMPLETED\tSUCCESS\tjob-a/\tIN_PROGRESS\t\tjob-a/' "$d/runs.tsv" @@ -608,6 +659,7 @@ self_test() { row normalize_counts_a_rerun_job 0 "$(printf 'job-a\tCOMPLETED\tSUCCESS\ts1\te1\t2')" "" -- st_norm row gql_query_names_every_producer 0 'w1: node(id: "WC")' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" row gql_query_refuses_a_missing_producer 5 "" "query" -- gql_query "$ST_LANES" "$tmp/wf.json" + row gql_query_reads_the_nightly_picks 0 'nightly: refs(refPrefix: "refs/heads/nightly/"' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" d="$tmp/voidnew"; fixture "$d" printf 'wf\t.github/workflows/a.yml\t102\tschedule\tmain\t%s\t2026-10-04T05:00:00Z\tCOMPLETED\tCANCELLED\tjob-a\tCOMPLETED\tCANCELLED\t\t\t1\n' "$ST_C" >> "$d/runs.tsv" row a_void_newer_run_falls_to_the_older_green 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" @@ -777,7 +829,18 @@ m39_exit_verdict_always_red s/^ \[ -z "\$EXIT_VERDICT" \] || verdict_rc "\$li m40_config_token_ignored s/^ if grep -qsE / if false \&\& grep -qsE / m41_home_cargo_unchecked s/ "\$HOME\/.cargo"; do$/; do/ m42_floor_ignores_the_limit s/fl=\$((lim \/ 5))/fl=$RATE_FLOOR/ -m43_any_event_counts s/ && REV\[r\] ~ EV\[i\]//' +m43_any_event_counts s/ && REV\[r\] ~ EV\[i\]// +m44_chain_ignores_the_pick s/(pc = nightc(RC\[r\])) != C)/0)/ +m45_any_night_counts s/if (t >= PS\[k\] && t < PE\[k\]) return k/if (PO[k] == C) return k/ +m46_window_end_inclusive s/t < PE\[k\]/t <= PE[k]/ +m47_window_start_exclusive s/t >= PS\[k\]/t > PS[k]/ +m48_every_lane_checks_the_pick s/EV\[i\] == "^workflow_run\$" && (pc/(pc/ +m49_from_skips_the_picks s/ tree picks.tsv; do/ tree; do/ +m50_picks_any_ref_name s/ | select(.name | test("^\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}\$"))// +m51_query_drops_the_picks s/ nightly: refs(refPrefix/ unread: refs(refPrefix/ +m52_no_straddle_check s/ && (pc = prevc(RC\[r\])) != C)/ \&\& 0)/ +m53_straddle_window_ends_early s/substr(RC\[r\], 12, 2) < "18"/substr(RC\[r\], 12, 2) < "13"/ +m54_prev_is_the_same_night s/if (PE\[j\] == PS\[k\]) return PO\[j\]/if (j == k) return PO[j]/' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 3dec70a60b9431c7c1abaae7d338eb14f86e98a0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 14:19:21 +0200 Subject: [PATCH 16/34] T44: a FAIL line ahead of a big output is no longer dropped (printf | grep -q under pipefail) judge() tested each workflow's output with `printf '%s\n' "$out" | grep -q '^FAIL'`. Under `set -o pipefail`, grep -q exits at the first match, the printf dies of SIGPIPE, and the pipeline reports failure: a FAIL that was found reads as no FAIL, so the guard can pass a broken producer. The self-test's own must-match check had the mirror defect and went RED once in four runs on an unchanged tree. - hasl TEXT ARGS: grep over a here-string, used by judge() and the row check. - The must/mustnot checks in nightly_pick.sh and nightly_c_checkout.sh use a here-string too. - Row sigpipe_fail_before_2mib: a FAIL line followed by 2 MiB must be seen. - Mutant m40_hasl_piped puts the pipe back; the row goes RED on it (3/3 runs). Agent: aprender-a7r Co-Authored-By: Claude Opus 5.5 --- scripts/release/check_producers_chain.sh | 19 ++++++++++++++++--- scripts/release/nightly_c_checkout.sh | 4 ++-- scripts/release/nightly_pick.sh | 4 ++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 93c8d27c81..030c6642df 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -171,6 +171,10 @@ p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by it exit bad }' "$t" } +# hasl TEXT GREP-ARGS...: does TEXT have a matching line. A here-string, never `printf | grep -q`: grep -q +# exits at the first match, the printf dies of SIGPIPE, and pipefail turns a found line into a miss. +hasl() { local text="$1"; shift; grep -q "$@" <<< "$text"; } + judge() { # ROOT local root="$1" wf pick p rc=0 out listed wf="$root/.github/workflows" @@ -188,7 +192,7 @@ judge() { # ROOT case " $(printf '%s' "$PRODUCERS" | tr '\n' ' ') " in *" $p "*) listed=1 ;; esac out="$(judge_file "$f" "$pick" "$listed")" [ -n "$out" ] && printf '%s\n' "$out" - printf '%s\n' "$out" | grep -q -e '^FAIL' && rc=1 + hasl "$out" -e '^FAIL' && rc=1 done p6 "$root" || rc=1 if [ "$rc" -eq 0 ]; then printf 'PASS every nightly producer chains from "%s" and measures its C\n' "$pick" @@ -214,7 +218,7 @@ row() { # NAME WANT-RC MUST -- MUTATION-FN (run inside the fixture's workflows d out="$(judge "$fx" 2>&1)"; rc=$? CASES=$((CASES + 1)) if [ "$rc" != "$want" ]; then printf 'RED %-40s rc=%s want %s\n%s\n' "$name" "$rc" "$want" "$(printf '%s\n' "$out" | grep -e '^FAIL' | head -n 3)"; FAILED=$((FAILED + 1)); return; fi - if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-40s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! hasl "$out" -F -e "$must"; then printf 'RED %-40s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi printf 'ok %s\n' "$name" } changed() { # FILE SED-EXPR: apply, and fail when nothing changed (a no-op mutation is an error, never a pass) @@ -283,6 +287,13 @@ m_sha_bracket() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\ m_sha_case() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\n - run: echo "${{ GitHub.SHA }}"/'; } m_bystander() { printf 'name: Other\non:\n schedule:\n - cron: "0 1 * * *"\njobs:\n a:\n runs-on: x\n steps:\n - uses: actions/checkout@v7\n - run: echo "$GITHUB_SHA"\n' > other.yml; } +sigpipe_row() { # a FAIL line, then 2 MiB: a pipe into grep -q loses it under pipefail (SIGPIPE), a here-string never does + local big=''; printf -v big '%s\n%2097152s' 'FAIL first line' '' + CASES=$((CASES + 1)) + if hasl "$big" -e '^FAIL'; then printf 'ok %s\n' sigpipe_fail_before_2mib + else printf 'RED %-40s a FAIL line ahead of 2 MiB of output was not seen\n' sigpipe_fail_before_2mib; FAILED=$((FAILED + 1)); fi +} + self_test() { TMP_ST="$(mktemp -d "${TMPDIR:-/tmp}/pc-st.XXXXXX")" || caller_error "no temp dir" echo "=== nightly producers chain: each rule must turn RED when its link breaks ===" @@ -338,6 +349,7 @@ self_test() { row p6_no_lanes_table 1 'P6 no LANES table read' -- m_lanes_renamed row p6_listed_without_a_lane 1 'P6 listed producer book.yml has no lane' -- m_lane_dropped row p6_lanes_assigned_twice 1 'P6 LANES is assigned more than once' -- m_lanes_twice + sigpipe_row rm -rf -- "${TMP_ST:?}" if [ "$FAILED" -eq 0 ]; then printf 'SELF-TEST PASSED: %s rows\n' "$CASES"; return 0; fi printf 'SELF-TEST FAILED: %s of %s rows\n' "$FAILED" "$CASES"; return 1 @@ -381,7 +393,8 @@ m35_unlisted_any_event s/ && a\[5\] ~ \/workflow_run\/)/ \&\& 0)/ m36_missing_train_read s/\[ -f "\$t" \] ||/[ 1 ] ||/ m37_no_table_ok s/if (!done || !lanes) {/if (0) {/ m38_a_listed_lane_may_be_missing s/if (!(pp\[k\] in seen)) {/if (0) {/ -m39_a_second_table_ok s/if (assigned > 1) {/if (0) {/' +m39_a_second_table_ok s/if (assigned > 1) {/if (0) {/ +m40_hasl_piped s/\(grep -q "[^"]*"\) <<< \("[^"]*"\)/echo \2 | \1/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh index e5a663dddb..3b338de5b3 100755 --- a/scripts/release/nightly_c_checkout.sh +++ b/scripts/release/nightly_c_checkout.sh @@ -66,8 +66,8 @@ row() { # NAME WANT-RC MUST MUSTNOT -- CMD... out="$("$@" 2>&1)"; rc=$? CASES=$((CASES + 1)) if [ "$rc" != "$want" ]; then printf 'RED %-44s rc=%s want %s\n' "$name" "$rc" "$want"; FAILED=$((FAILED + 1)); return; fi - if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi - if [ -n "$mustnot" ] && printf '%s\n' "$out" | grep -qF -e "$mustnot"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! grep -qF -e "$must" <<< "$out"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$mustnot" ] && grep -qF -e "$mustnot" <<< "$out"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi printf 'ok %s\n' "$name" } same() { # NAME A B : two values must be equal diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index ee9a3c6848..30d54e6ad6 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -32,8 +32,8 @@ row() { # NAME WANT-RC MUST MUSTNOT -- CMD... out="$("$@" 2>&1)"; rc=$? CASES=$((CASES + 1)) if [ "$rc" != "$want" ]; then printf 'RED %-44s rc=%s want %s\n' "$name" "$rc" "$want"; FAILED=$((FAILED + 1)); return; fi - if [ -n "$must" ] && ! printf '%s\n' "$out" | grep -qF -e "$must"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi - if [ -n "$mustnot" ] && printf '%s\n' "$out" | grep -qF -e "$mustnot"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$must" ] && ! grep -qF -e "$must" <<< "$out"; then printf 'RED %-44s missing: %s\n' "$name" "$must"; FAILED=$((FAILED + 1)); return; fi + if [ -n "$mustnot" ] && grep -qF -e "$mustnot" <<< "$out"; then printf 'RED %-44s forbidden: %s\n' "$name" "$mustnot"; FAILED=$((FAILED + 1)); return; fi printf 'ok %s\n' "$name" } same() { # NAME A B : two outputs must be equal From 9b54cd6df269251003cb4c5300f942c6ce13d0ee Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 15:48:02 +0200 Subject: [PATCH 17/34] fix(release): nightly pick checks the whole night and sha, not each line np__args validated NIGHT and SHA with `printf '%s\n' "$1" | grep -qxE`, and grep -x matches line by line: a value whose first line is well formed ("2026-10-09x", a 40-hex sha followed by a second line) passed the check and went on to git. The check now uses [[ =~ ]] with ^...$, which anchors the whole string. Case table: two new rows, multiline_night_is_a_caller_error and multiline_sha_is_a_caller_error, were RED before the fix (rc 2, want 3) and are green after it (36/36). Mutants: m08/m09 now patch the new check; m15/m16 put the per-line check back and are each killed by their new row. --mutants: 16/16 killed, 0 errors. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-pick-v1.yaml | 2 +- scripts/lib/nightly_pick.sh | 5 +++-- scripts/release/nightly_pick.sh | 9 +++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index 6a847538e2..00d6a4f53e 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -87,7 +87,7 @@ falsification_tests: - id: FALSIFY-NP-002 rule: "The case table can still fail on the defects it names" - prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, a read widened so the rolling tag refs/tags/nightly is seen, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" + prediction: "Each planted mutant (a force push that moves the ref, a plain push that fast-forwards a racing ref, an unreadable ancestry refused as RED, a decoy ref matched by its tail, a read widened so the rolling tag refs/tags/nightly is seen, an existing ref ignored, a branch commit accepted, verify accepting any commit, an unpicked night resolving, an unreadable remote read as absent, the night taken as the fire date, a malformed night or commit accepted, either one checked line by line so a well-formed first line carries the rest, a lost race claimed as PICKED) turns --self-test RED; a mutant whose patch does not apply is an error, never a survivor" test: "bash scripts/release/nightly_pick.sh --mutants" if_fails: "The case table is theater: it would pass a picker that lets a merge move the night's C" diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh index 79d048d132..a09dbe39e5 100644 --- a/scripts/lib/nightly_pick.sh +++ b/scripts/lib/nightly_pick.sh @@ -29,8 +29,9 @@ np_night() { # [EPOCH] } np__args() { # NIGHT [SHA] - printf '%s\n' "$1" | grep -qxE '[0-9]{4}-[0-9]{2}-[0-9]{2}' || { printf 'NP caller error: night %s is not YYYY-MM-DD\n' "$1" >&2; return 3; } - [ "$#" -lt 2 ] || printf '%s\n' "$2" | grep -qxE '[0-9a-f]{40}' || { printf 'NP caller error: sha %s is not 40 hex\n' "$2" >&2; return 3; } + # [[ =~ ]] anchors the whole string; `printf | grep -x` matched per line, so "2026-10-09x" passed + [[ $1 =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || { printf 'NP caller error: night %s is not YYYY-MM-DD\n' "$1" >&2; return 3; } + [ "$#" -lt 2 ] || [[ $2 =~ ^[0-9a-f]{40}$ ]] || { printf 'NP caller error: sha %s is not 40 hex\n' "$2" >&2; return 3; } } np__read() { # REPO REMOTE NIGHT -> the ref's sha on stdout; 0 found, 1 absent, 2 unreadable diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index ee9a3c6848..595805a9db 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -113,6 +113,9 @@ self_test() { row unreachable_remote_resolve_is_not_measured 2 "cannot read" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --remote "$FX/absent.git" --night "$n" row malformed_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-1-4 --sha "$c1" row malformed_sha_is_a_caller_error 3 "not 40 hex" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 --sha "${c1:0:10}" + # a multi-line argument: one well-formed line must not carry another past the check (grep -x matches per line) + row multiline_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "2026-10-09"$'\n'"x" --sha "$c1" + row multiline_sha_is_a_caller_error 3 "not 40 hex" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 --sha "$c1"$'\n'"x" row pick_without_sha_is_a_caller_error 3 "caller error" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" repick rm -rf -- "${FX:?}" @@ -129,10 +132,12 @@ m04_verify_accepts_any_sha s/if \[ "\$c" = "\$4" \]; then printf .NP OK/if true; m05_unpicked_night_resolves s/1) printf .NP NOT_MEASURED: night %s has no pick\\n. "\$3" >\&2; return 2 ;;/1) printf "\\n"; return 0 ;;/ m06_unreadable_is_absent s/\[ "\$rc" -eq 0 \] || return 2$/[ "$rc" -eq 0 ] || return 1/ m07_night_is_the_fire_date s/\$((e - 43200))/$e/ -m08_bad_night_accepted s/grep -qxE .\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}. || {/true || {/ -m09_bad_sha_accepted s/grep -qxE .\[0-9a-f\]{40}. || {/true || {/ +m08_bad_night_accepted s/\[\[ \$1 =~ .*\$ \]\] || {/true || {/ +m09_bad_sha_accepted s/\[\[ \$2 =~ .*\$ \]\] || {/true || {/ m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ m11_plain_push_fast_forwards s/ --force-with-lease="refs\/heads\/nightly\/\$3:"// +m15_night_checked_per_line s/\[\[ \$1 =~ \(.*\) \]\] ||/printf "%s\\n" "$1" | grep -qxE "\1" ||/ +m16_sha_checked_per_line s/\[\[ \$2 =~ \(.*\) \]\] ||/printf "%s\\n" "$2" | grep -qxE "\1" ||/ m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/ m14_read_widened_to_the_rolling_tag s/ls-remote --refs "\$2" "refs\/heads\/nightly\/\$3"/ls-remote --refs "\$2" "nightly"/' From d176b37d24c1331fdc681058ee30c7dc527b61ec Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 16:00:56 +0200 Subject: [PATCH 18/34] fix(release): nightly pick review round 1, a trailing newline is a caller error too The per-line check also passed a value with only a trailing newline ("2026-10-09", a sha plus newline). Two rows cover it: trailing_newline_night_is_a_caller_error and trailing_newline_sha_is_a_caller_error. m15/m16, which restore the per-line check, are now each killed by two rows. m15/m16 move after m14. --self-test: 38/38 green. --mutants: 16/16 killed, 0 errors. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/nightly_pick.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index 595805a9db..89267f460a 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -116,6 +116,8 @@ self_test() { # a multi-line argument: one well-formed line must not carry another past the check (grep -x matches per line) row multiline_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "2026-10-09"$'\n'"x" --sha "$c1" row multiline_sha_is_a_caller_error 3 "not 40 hex" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 --sha "$c1"$'\n'"x" + row trailing_newline_night_is_a_caller_error 3 "not YYYY-MM-DD" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "2026-10-09"$'\n' --sha "$c1" + row trailing_newline_sha_is_a_caller_error 3 "not 40 hex" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 --sha "$c1"$'\n' row pick_without_sha_is_a_caller_error 3 "caller error" "PICKED" -- bash "$SCRIPT_PATH" pick --repo "$W" --night 2026-10-09 row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" repick rm -rf -- "${FX:?}" @@ -136,11 +138,11 @@ m08_bad_night_accepted s/\[\[ \$1 =~ .*\$ \]\] || {/true || {/ m09_bad_sha_accepted s/\[\[ \$2 =~ .*\$ \]\] || {/true || {/ m10_race_loser_claims_picked s/printf .C=%s KEPT night %s (another pick won the race)\\n. "\$c" "\$3"/printf '"'"'C=%s PICKED night %s\\n'"'"' "$c" "$3"/ m11_plain_push_fast_forwards s/ --force-with-lease="refs\/heads\/nightly\/\$3:"// -m15_night_checked_per_line s/\[\[ \$1 =~ \(.*\) \]\] ||/printf "%s\\n" "$1" | grep -qxE "\1" ||/ -m16_sha_checked_per_line s/\[\[ \$2 =~ \(.*\) \]\] ||/printf "%s\\n" "$2" | grep -qxE "\1" ||/ m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/ -m14_read_widened_to_the_rolling_tag s/ls-remote --refs "\$2" "refs\/heads\/nightly\/\$3"/ls-remote --refs "\$2" "nightly"/' +m14_read_widened_to_the_rolling_tag s/ls-remote --refs "\$2" "refs\/heads\/nightly\/\$3"/ls-remote --refs "\$2" "nightly"/ +m15_night_checked_per_line s/\[\[ \$1 =~ \(.*\) \]\] ||/printf "%s\\n" "$1" | grep -qxE "\1" ||/ +m16_sha_checked_per_line s/\[\[ \$2 =~ \(.*\) \]\] ||/printf "%s\\n" "$2" | grep -qxE "\1" ||/' mutants() { local tmp name expr killed=0 total=0 errors=0 out tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" From 8e8d78ea9abb066997f496adffa286968474c52f Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 20:05:10 +0200 Subject: [PATCH 19/34] T44: a producer's night is the pick run's created_at, and a failed pick is not measured The night table for the nightly producers: night = UTC date of (created_at - 12h) of the pick run. - Every producer's C step now takes NIGHTLY_PICK_AT from github.event.workflow_run.created_at, not run_started_at. A rerun of either run keeps created_at, but run_started_at moves, so a pick rerun the next day used to name the next night. - Every C step also passes NIGHTLY_PICK_CONCLUSION (github.event.workflow_run.conclusion) to nightly_c_checkout.sh --pick-conclusion. Anything but success is not_measured (exit 2): nothing is exported, the tree is left alone, and there is no fallback to main's head. A missing value is a caller error. - nightly_c_checkout.sh case rows cover the boundaries: a pick created at 01:00Z, 23:30Z, 11:58Z (producer starting after noon) and 11:59:59Z is night N; 12:00:00Z is N+1; failed and cancelled picks are not measured. - New mutant m10: the conclusion is ignored. - check_producers_chain.sh pins the new C-step text. New rows go RED when a producer reads run_started_at, drops or re-sources the conclusion env, or does not pass --pick-conclusion. New mutant m41: any conclusion source passes. Receipts (bash only): - nightly_c_checkout.sh --self-test 32/32, --mutants 10/10 killed - check_producers_chain.sh on the tree PASS, --self-test 57/57, --mutants 41/41 killed Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/beat-speed-nightly.yml | 5 +- .github/workflows/book-contracts.yml | 30 ++++++---- .github/workflows/book.yml | 5 +- .github/workflows/conleche-nightly.yml | 5 +- .github/workflows/coverage-nightly.yml | 5 +- .github/workflows/cuda-nightly.yml | 10 ++-- .github/workflows/examples-nightly.yml | 5 +- .github/workflows/guards-nightly.yml | 5 +- .github/workflows/install-script.yml | 5 +- .github/workflows/mutants-nightly.yml | 5 +- .github/workflows/nightly-bench.yml | 5 +- .github/workflows/nightly.yml | 20 ++++--- .github/workflows/qwen-story-daily.yml | 5 +- .github/workflows/silicon-nightly.yml | 15 +++-- .github/workflows/toolchain-ceiling.yml | 15 +++-- scripts/release/check_producers_chain.sh | 39 ++++++++----- scripts/release/nightly_c_checkout.sh | 72 ++++++++++++++++-------- 17 files changed, 158 insertions(+), 93 deletions(-) diff --git a/.github/workflows/beat-speed-nightly.yml b/.github/workflows/beat-speed-nightly.yml index 859dedfaa2..7da58bbb2d 100644 --- a/.github/workflows/beat-speed-nightly.yml +++ b/.github/workflows/beat-speed-nightly.yml @@ -78,8 +78,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # PERF-031 measurement-environment receipt. # diff --git a/.github/workflows/book-contracts.yml b/.github/workflows/book-contracts.yml index c219687747..f0d6c2eb9b 100644 --- a/.github/workflows/book-contracts.yml +++ b/.github/workflows/book-contracts.yml @@ -70,8 +70,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -119,8 +120,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -175,8 +177,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - uses: dtolnay/rust-toolchain@stable # NO rust-cache: cache-bin false was never the fix. # @@ -212,8 +215,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Every chapter has a contract run: | FAIL=0 @@ -277,8 +281,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: No legacy imports in chapter examples run: | FAIL=0 @@ -323,8 +328,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: No dead links in SUMMARY.md run: | FAIL=0 diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index fdfd85cb07..a27fa9640a 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -83,8 +83,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Self-hosted (#3073): tools go into this run's private $RUNNER_TEMP, never a $HOME path that every # runner on the box execs from (the aprender#2353 race; check_cargo_install_private_root.sh enforces diff --git a/.github/workflows/conleche-nightly.yml b/.github/workflows/conleche-nightly.yml index 369d2a4dc4..3e93b92ab3 100644 --- a/.github/workflows/conleche-nightly.yml +++ b/.github/workflows/conleche-nightly.yml @@ -56,8 +56,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Verdict case table (the classifier this job trusts) run: bash crates/aprender-contracts-staging/lean/conleche.sh --self-test - name: Pinned elan in the job's own directory (sha256-checked, never a host install) diff --git a/.github/workflows/coverage-nightly.yml b/.github/workflows/coverage-nightly.yml index a701b5059a..19bd250ff2 100644 --- a/.github/workflows/coverage-nightly.yml +++ b/.github/workflows/coverage-nightly.yml @@ -108,8 +108,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Provision cargo-llvm-cov WITHOUT touching ~/.cargo/bin (#2353). # diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index 94280ccf1a..31bbeaacdd 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -185,8 +185,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' && (steps.decide.outputs.proceed == 'true') env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # PP-26's release-phase marker gate (scripts/check_perf041_marker.sh, a # dogfood gate) may not be named in a REQUIRED workflow @@ -801,8 +802,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Put the runner's Rust toolchain on PATH run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/examples-nightly.yml b/.github/workflows/examples-nightly.yml index 0db25b56d3..7bc926acaf 100644 --- a/.github/workflows/examples-nightly.yml +++ b/.github/workflows/examples-nightly.yml @@ -40,8 +40,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: dogfood_examples.sh case table (hermetic, scratch workspace) run: bash scripts/dogfood_examples.sh --selftest diff --git a/.github/workflows/guards-nightly.yml b/.github/workflows/guards-nightly.yml index 4fd6895d67..56b90567f3 100644 --- a/.github/workflows/guards-nightly.yml +++ b/.github/workflows/guards-nightly.yml @@ -95,8 +95,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # THE MANIFEST IS THE JOIN KEY, AND IT IS CHECKED BEFORE ANYTHING RUNS. # # A step that leaves ci.yml and never arrives here is a guard that was diff --git a/.github/workflows/install-script.yml b/.github/workflows/install-script.yml index 05490e9559..1aa5d4bfcb 100644 --- a/.github/workflows/install-script.yml +++ b/.github/workflows/install-script.yml @@ -60,8 +60,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # pmat/bashrs are pinned-fleet tools, never installed per-job (BSE-10a, # PMAT-1066) — see ci.yml's identical step for why. diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index 55dfdf6b97..14314921d4 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -47,8 +47,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Everything merged since the last nightly that JUDGED main, as a diff id: day # Not a rolling "24 hours ago" window: that has no memory, so one missed or failed run would drop a day of diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 2db179bc42..abc301a166 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -63,8 +63,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # The receipt validator's own case table must discriminate before any # number it blesses means anything. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 837618d827..9b7bd3fffa 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -77,8 +77,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Case table run: python3 scripts/nightly_manifest.py --self-test @@ -123,8 +124,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # A sibling container through the mounted socket, as in binary-release.yml's # CPU lanes: every bind-mount path must exist on the HOST (the work root is @@ -264,8 +266,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Build apr (host-native, pinned toolchain) run: | @@ -311,8 +314,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # A runner that died uploaded nothing; merge records that arch red, so # an empty download must not stop the other arch from publishing. diff --git a/.github/workflows/qwen-story-daily.yml b/.github/workflows/qwen-story-daily.yml index 8681b427c5..1f71a0754b 100644 --- a/.github/workflows/qwen-story-daily.yml +++ b/.github/workflows/qwen-story-daily.yml @@ -77,8 +77,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Install apr from HEAD shell: bash diff --git a/.github/workflows/silicon-nightly.yml b/.github/workflows/silicon-nightly.yml index 2e955b291f..9d273d7b19 100644 --- a/.github/workflows/silicon-nightly.yml +++ b/.github/workflows/silicon-nightly.yml @@ -55,8 +55,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # Probe the instrument before believing it. The label matcher must classify # six committed cases — including a runner whose labels are a strict SUBSET @@ -114,8 +115,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH @@ -160,8 +162,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: cargo_step.sh falsifier (ENV-vs-CODE instrument) run: bash scripts/cargo_step.sh --self-test - name: Put the runner's Rust toolchain on PATH diff --git a/.github/workflows/toolchain-ceiling.yml b/.github/workflows/toolchain-ceiling.yml index 22c6528c42..6fdde0353d 100644 --- a/.github/workflows/toolchain-ceiling.yml +++ b/.github/workflows/toolchain-ceiling.yml @@ -63,8 +63,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Workspace builds on its DECLARED MSRV run: setsid --wait bash scripts/check_msrv.sh @@ -82,8 +83,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Clippy on current stable (ceiling gate) run: | @@ -111,8 +113,9 @@ jobs: - name: Measure the night's C, not main's head (T44) if: github.event_name == 'workflow_run' env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" - name: Self-test — an unreached feature is refused, and a planted cuda-only finding reddens only the cuda-reaching axes run: | export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 030c6642df..e6767dbd57 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -10,8 +10,10 @@ # P2 lacks the chain: workflow_run with workflows: [], types: [completed], # branches: [main]; # P3 checks this repo out (actions/checkout, unless `repository:` names another repo literally) and the next step, after an optional -# "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"`, -# env NIGHTLY_PICK_AT from github.event.workflow_run.run_started_at, and if exactly +# "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION"`, +# env NIGHTLY_PICK_AT from github.event.workflow_run.created_at (the pick run's own time, which a rerun of either +# run keeps; run_started_at moves on a rerun) and NIGHTLY_PICK_CONCLUSION from github.event.workflow_run.conclusion +# (a failed pick names no night to measure), and if exactly # `github.event_name == 'workflow_run'`, or `github.event_name == 'workflow_run' && ()`; # the C step carries nothing else (no continue-on-error, no other env), a checkout's if must be a bare # expression the C step can copy, a listed producer has at least one checkout of this repo, and no job of it @@ -67,7 +69,7 @@ judge_file() { if (i == 1) { k = s; sub(/^ *- /, "", k) } else if (ind(s) == p) k = substr(s, p + 1) else { gsub(/^[[:space:]]+|[[:space:]]+$/, "", s) - if (s == "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}") continue; return s } + if (s == "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }}" || s == "NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }}") continue; return s } sub(/:.*/, "", k); if (k !~ /^(name|if|env|run)$/) return k } return "" @@ -98,14 +100,16 @@ judge_file() { want = (cif == "\001") ? "github.event_name == \047workflow_run\047" \ : "github.event_name == \047workflow_run\047 && (" cif ")" if (j > n) { fail("P3 checkout in job step " k " is not followed by the C step (end of steps)"); continue } - if (key(S[j], "run") != "bash scripts/release/nightly_c_checkout.sh --at \"$NIGHTLY_PICK_AT\"") { + if (key(S[j], "run") != "bash scripts/release/nightly_c_checkout.sh --at \"$NIGHTLY_PICK_AT\" --pick-conclusion \"$NIGHTLY_PICK_CONCLUSION\"") { fail("P3 checkout in job step " k " is not followed by the C step (step " j " runs something else)"); continue } - if (!hasline(S[j], "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}")) - fail("P3 the C step after job step " k " does not take NIGHTLY_PICK_AT from workflow_run.run_started_at") + if (!hasline(S[j], "NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }}")) + fail("P3 the C step after job step " k " does not take NIGHTLY_PICK_AT from workflow_run.created_at") + if (!hasline(S[j], "NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }}")) + fail("P3 the C step after job step " k " does not take NIGHTLY_PICK_CONCLUSION from workflow_run.conclusion") if (unq(key(S[j], "if")) != want) fail("P3 the C step after job step " k " has if: " key(S[j], "if") " -- want: " want) if ((sx = stray(S[j])) != "") - fail("P3 the C step after job step " k " carries " sx " -- only name, if, env NIGHTLY_PICK_AT, run") + fail("P3 the C step after job step " k " carries " sx " -- only name, if, env NIGHTLY_PICK_AT and NIGHTLY_PICK_CONCLUSION, run") } n = 0; ii = -1 } @@ -224,14 +228,14 @@ row() { # NAME WANT-RC MUST -- MUTATION-FN (run inside the fixture's workflows d changed() { # FILE SED-EXPR: apply, and fail when nothing changed (a no-op mutation is an error, never a pass) local before; before="$(cat "$1")"; sed -i -e "$2" "$1" || return 1; [ "$before" != "$(cat "$1")" ] } -CSTEP='bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT"' +CSTEP='bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION"' synth() { # FILE: a minimal chained producer, built by the caller-supplied steps on stdin { printf 'name: Synth\non:\n workflow_run:\n workflows: ["Nightly pick"]\n types: [completed]\n branches: [main]\n workflow_dispatch:\n' printf 'jobs:\n a:\n runs-on: x\n steps:\n'; cat; } > "$1" } cstep() { # [IF]: the C step text, with the checkout's if when given local cif="github.event_name == 'workflow_run'"; [ -n "${1:-}" ] && cif="$cif && ($1)" - printf ' - name: Measure the night'"'"'s C, not main'"'"'s head (T44)\n if: %s\n env:\n NIGHTLY_PICK_AT: ${{ github.event.workflow_run.run_started_at }}\n run: %s\n' "$cif" "$CSTEP" + printf ' - name: Measure the night'"'"'s C, not main'"'"'s head (T44)\n if: %s\n env:\n NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }}\n NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }}\n run: %s\n' "$cif" "$CSTEP" } m_none() { :; } m_sched_back() { changed nightly.yml 's/^ workflow_run:$/ schedule:\n - cron: "0 3 * * *"\n workflow_run:/'; } @@ -258,7 +262,11 @@ m_no_cstep() { changed toolchain-ceiling.yml '/- name: Measure the night.s m_cstep_or() { changed install-script.yml "0,/if: github.event_name == 'workflow_run'\$/s//if: github.event_name == 'workflow_run' || true/"; } m_cstep_ifdrop() { changed cuda-nightly.yml "s/if: github.event_name == 'workflow_run' && (.*)\$/if: github.event_name == 'workflow_run'/"; } m_cstep_noat() { changed nightly-bench.yml '0,/nightly_c_checkout.sh --at "\$NIGHTLY_PICK_AT"/s//nightly_c_checkout.sh/'; } -m_cstep_wrongat() { changed coverage-nightly.yml '0,/workflow_run.run_started_at/s//workflow_run.head_commit.timestamp/'; } +m_cstep_wrongat() { changed coverage-nightly.yml '0,/workflow_run.created_at/s//workflow_run.head_commit.timestamp/'; } +m_cstep_started() { changed coverage-nightly.yml '0,/workflow_run.created_at/s//workflow_run.run_started_at/'; } +m_cstep_noconcl() { changed nightly-bench.yml '0,/^ *NIGHTLY_PICK_CONCLUSION: .*$/{//d}'; } +m_cstep_badconcl() { changed book.yml '0,/workflow_run.conclusion }}/s//workflow_run.event }}/'; } +m_cstep_noconcarg() { changed install-script.yml '0,/ --pick-conclusion "\$NIGHTLY_PICK_CONCLUSION"/s///'; } m_step_between() { changed examples-nightly.yml '0,/- name: Measure the night.s C/s//- run: make bench\n - name: Measure the night'"'"'s C/'; } m_repo_exempt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/other\n')"; } m_preflight_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - name: Preflight — tools\n run: true\n'; cstep)"; } @@ -269,8 +277,8 @@ m_sha_plain() { changed qwen-story-daily.yml '0,/^ steps:$/s// steps:\ m_sha_env() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: git diff "$GITHUB_SHA"/'; } m_sha_comment() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n # measured at $GITHUB_SHA, before T44/'; } m_sha_c() { changed beat-speed-nightly.yml '0,/^ steps:$/s// steps:\n - run: echo "${NIGHTLY_C:-$GITHUB_SHA}"/'; } -m_cstep_coe() { changed install-script.yml '0,/nightly_c_checkout.sh --at "\$NIGHTLY_PICK_AT"$/s//&\n continue-on-error: true/'; } -m_cstep_envx() { changed nightly-bench.yml '0,/NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}$/s//&\n GIT_DIR: \/elsewhere/'; } +m_cstep_coe() { changed install-script.yml '0,/--pick-conclusion "\$NIGHTLY_PICK_CONCLUSION"$/s//&\n continue-on-error: true/'; } +m_cstep_envx() { changed nightly-bench.yml '0,/NIGHTLY_PICK_AT: \${{ github.event.workflow_run.created_at }}$/s//&\n GIT_DIR: \/elsewhere/'; } m_cstep_qif_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep | sed -e "s/if: \(.*\)\$/if: \"\1\"/")"; } m_quoted_uses() { synth nightly.yml <<< "$(printf ' - uses: "actions/checkout@v7"\n - run: make bench\n')"; } m_no_checkout() { synth nightly.yml <<< ' - run: make bench'; } @@ -314,6 +322,10 @@ self_test() { row p3_if_drops_checkout_if 1 'cuda-nightly.yml: P3' -- m_cstep_ifdrop row p3_no_at 1 'nightly-bench.yml: P3' -- m_cstep_noat row p3_at_from_head_commit 1 'does not take NIGHTLY_PICK_AT' -- m_cstep_wrongat + row p3_at_from_run_started_at 1 'does not take NIGHTLY_PICK_AT' -- m_cstep_started + row p3_no_conclusion_env 1 'does not take NIGHTLY_PICK_CONCLUSION' -- m_cstep_noconcl + row p3_conclusion_from_event 1 'does not take NIGHTLY_PICK_CONCLUSION' -- m_cstep_badconcl + row p3_conclusion_not_passed 1 'install-script.yml: P3' -- m_cstep_noconcarg row p3_step_between 1 'examples-nightly.yml: P3' -- m_step_between row p3_last_step_checkout 1 'end of steps' -- m_last_step row p3_other_repo_exempt 0 'PASS' -- m_repo_exempt @@ -363,6 +375,7 @@ m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ m06_no_repo_exempt s/ || other(S\[k\])) continue/) continue/ m07_any_if s/if (unq(key(S\[j\], "if")) != want)/if (0)/ m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ +m41_any_conclusion_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_CONCLUSION/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_CONCLUSION/ m09_no_sha_rule s|if (tolower(c) ~|if (0 \&\& tolower(c) ~| m10_sha_no_exemption s| \&\& c !~ /NIGHTLY_C\|workflow_run\\.head_sha/)|)| m11_no_missing_rule /P5 listed producer is missing/s/; rc=1; }/; }/ @@ -378,7 +391,7 @@ m20_job_uses_ok s/listed \&\& live(L) \&\& L ~ \/^ uses:\//0 \&\& \/^ uses m21_checkout_if_any s/if (cif ~ \/\\\$\\{\\{|^\[>|\]\/) {/if (0) {/ m22_comment_exempts s/c = L; sub(\/\[\[:space:\]\]#\.\*\/, "", c)/c = L/ m23_no_workflow_sha s/(workflow_)?sha|github_/sha|github_/ -m24_env_any_line s/if (s == "NIGHTLY_PICK_AT: \${{ github.event.workflow_run.run_started_at }}") continue; return s }/continue }/ +m24_env_any_line s/if (s == "NIGHTLY_PICK_AT: \${{ github.event.workflow_run.created_at }}" || s == "NIGHTLY_PICK_CONCLUSION: \${{ github.event.workflow_run.conclusion }}") continue; return s }/continue }/ m25_cstep_if_quoted_no s/if (unq(key(S\[j\], "if")) != want)/if (key(S[j], "if") != want)/ m26_dash_at_si_ends s/(ind(L) < si || (ind(L) == si && L !~ \/^ \*- \/))/ind(L) <= si/ m27_any_repo_exempt s/ || other(S\[k\])) continue/ || S[k] ~ \/repository:\/) continue/ diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh index 3b338de5b3..cfc70a985b 100755 --- a/scripts/release/nightly_c_checkout.sh +++ b/scripts/release/nightly_c_checkout.sh @@ -6,15 +6,18 @@ # This step reads refs/heads/nightly/ (scripts/lib/nightly_pick.sh), switches the tree to that commit when it # is not already there, and then asks np_verify whether HEAD is C. A producer that measures any other commit is RED. # -# The night comes from --at, the pick run's start (github.event.workflow_run.run_started_at), never from the clock -# here: a producer queued for hours still measures the night it was started for. +# The night comes from --at, the pick run's creation time (github.event.workflow_run.created_at), never from the clock +# here and never from run_started_at: a producer queued for hours, or rerun the next day, still measures the night the +# pick was created for, and a rerun of the pick keeps its created_at while its run_started_at moves. +# --pick-conclusion is the pick run's conclusion (github.event.workflow_run.conclusion). Anything but success means the +# pick named no night to measure: the step is not_measured and never falls back to main's head. # # Usage: -# nightly_c_checkout.sh [checkout] [--repo DIR] [--remote NAME] --at ISO8601|EPOCH +# nightly_c_checkout.sh [checkout] [--repo DIR] [--remote NAME] --at ISO8601|EPOCH --pick-conclusion CONCLUSION # nightly_c_checkout.sh --self-test | --mutants # On success it appends NIGHTLY_C and NIGHTLY_NIGHT to $GITHUB_ENV and one line to $GITHUB_STEP_SUMMARY, when set. -# Exit: 0 HEAD is C; 1 RED (HEAD is not C, or the tree could not be switched); 2 not_measured (no pick, or the remote -# could not be read); 3 caller error. +# Exit: 0 HEAD is C; 1 RED (HEAD is not C, or the tree could not be switched); 2 not_measured (the pick did not +# succeed, no pick, or the remote could not be read); 3 caller error. # # Everything runs inside functions, called on the last line: the checkout replaces this file under a running bash. set -uo pipefail @@ -33,8 +36,9 @@ to_epoch() { # ISO8601|EPOCH -> epoch esac } -checkout_c() { # REPO REMOTE AT +checkout_c() { # REPO REMOTE AT CONCLUSION local repo="$1" remote="$2" epoch night c head rc=0 how + [ "$4" = success ] || { printf 'NC NOT_MEASURED: the pick concluded %s, so it named no night to measure\n' "$4"; return 2; } epoch="$(to_epoch "$3")" || { printf 'NC caller error: --at %s is not a time\n' "$3" >&2; return 3; } night="$(np_night "$epoch")" || return 3 c="$(np_resolve "$repo" "$remote" "$night")" || return $? @@ -90,32 +94,32 @@ self_test() { git clone -q "$FX/origin.git" "$W" 2>/dev/null && git clone -q --depth 1 "file://$FX/origin.git" "$S" 2>/dev/null || caller_error "fixture clones" # the defect: a producer that checked out main's head and measured it - row main_head_is_switched_to_c 0 "switched from" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + row main_head_is_switched_to_c 0 "switched from" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion success same the_tree_is_on_c_after_a_merge "$(git -C "$W" rev-parse HEAD)" "$c1" - row a_second_run_finds_c_already 0 "already C" "switched" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" - row a_shallow_checkout_reaches_c 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" + row a_second_run_finds_c_already 0 "already C" "switched" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion success + row a_shallow_checkout_reaches_c 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" --pick-conclusion success same the_shallow_tree_is_on_c "$(git -C "$S" rev-parse HEAD)" "$c1" # a late start still measures the night it was started for: 05:48Z the next day is still night N git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" - row a_late_start_is_still_the_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T05:48:00Z + row a_late_start_is_still_the_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T05:48:00Z --pick-conclusion success # the night comes from --at, not from the clock: an epoch the day after has no pick git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" - row an_unpicked_night_is_not_measured 2 "has no pick" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791230400 + row an_unpicked_night_is_not_measured 2 "has no pick" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791230400 --pick-conclusion success same not_measured_leaves_the_tree "$(git -C "$W" rev-parse HEAD)" "$c2" - row epoch_at_is_accepted 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791147600 + row epoch_at_is_accepted 0 "C=$c1" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 1791147600 --pick-conclusion success # a tree that cannot be switched is RED, never a measurement of the wrong commit git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" printf 'local edit\n' > "$W/f" - row an_unswitchable_tree_is_red 1 "cannot switch" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + row an_unswitchable_tree_is_red 1 "cannot switch" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion success same the_unswitchable_tree_is_left_alone "$(git -C "$W" rev-parse HEAD)" "$c2" git -C "$W" checkout -q -- f # GITHUB_ENV carries C to every later step; it is written only once HEAD is C env="$FX/github_env"; : > "$env" - row env_is_written 0 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" + row env_is_written 0 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at "$at" --pick-conclusion success row env_holds_c 0 "NIGHTLY_C=$c1" "" -- cat "$env" row env_holds_the_night 0 "NIGHTLY_NIGHT=$n" "" -- cat "$env" : > "$env" - row no_env_when_not_measured 2 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at 1791230400 + row no_env_when_not_measured 2 "" "" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$S" --at 1791230400 --pick-conclusion success same not_measured_writes_no_env "$(wc -c < "$env" | tr -d ' ')x" "0x" # a ref that moves while the step runs (a forced re-pick; the ruleset forbids it, the fixture plants it with a # post-checkout hook): the commit just checked out is no longer C, so the step is RED and C is never exported @@ -123,12 +127,29 @@ self_test() { printf '#!/bin/sh\ngit push -q -f "%s" "%s:refs/heads/nightly/%s" 2>/dev/null\n' "$FX/origin.git" "$c2" "$n" > "$FX/hooks/post-checkout" chmod +x "$FX/hooks/post-checkout" git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2"; : > "$env" - row a_ref_moved_mid_step_is_red 1 "is not the C" "NIGHTLY C=" -- env GITHUB_ENV="$env" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0="$FX/hooks" bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" + row a_ref_moved_mid_step_is_red 1 "is not the C" "NIGHTLY C=" -- env GITHUB_ENV="$env" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0="$FX/hooks" bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion success same a_moved_ref_exports_nothing "$(wc -c < "$env" | tr -d ' ')x" "0x" git -C "$A" push -q -f origin "$c1:refs/heads/nightly/$n" || caller_error "fixture restore" - row unreachable_remote_is_not_measured 2 "cannot read" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --remote "$FX/absent.git" --at "$at" + row unreachable_remote_is_not_measured 2 "cannot read" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --remote "$FX/absent.git" --at "$at" --pick-conclusion success + # the night table: night = date(created_at - 12h) of the pick run, and every --at below is that created_at. A + # producer's own start, its queue time and a rerun of either run the next day are not inputs, so the rows that + # differ only in those (a pick created 11:58Z with the producer starting after noon, a rerun the next day) are + # the 11:58Z row; the guard (check_producers_chain.sh) is RED when a producer reads run_started_at instead + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2" + row n1_created_after_midnight_is_that_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T01:00:00Z --pick-conclusion success + row n2_created_before_midnight_is_that_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-04T23:30:00Z --pick-conclusion success + row n3_created_1158z_started_after_noon 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T11:58:00Z --pick-conclusion success + row n5_last_second_is_still_the_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T11:59:59Z --pick-conclusion success + row n6_noon_is_the_next_night 2 "has no pick" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T12:00:00Z --pick-conclusion success + # a pick that did not succeed names no night: not_measured, never main's head, nothing exported, the tree untouched + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2"; : > "$env" + row n9_failed_pick_is_not_measured 2 "the pick concluded failure" "NIGHTLY C=" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion failure + same n9_failed_pick_leaves_the_tree "$(git -C "$W" rev-parse HEAD)" "$c2" + same n9_failed_pick_exports_nothing "$(wc -c < "$env" | tr -d ' ')x" "0x" + row n9_cancelled_pick_is_not_measured 2 "the pick concluded cancelled" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion cancelled + row missing_conclusion_is_a_caller_error 3 "--pick-conclusion" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" row missing_at_is_a_caller_error 3 "--at" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" - row bad_at_is_a_caller_error 3 "not a time" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at yesterday-ish + row bad_at_is_a_caller_error 3 "not a time" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at yesterday-ish --pick-conclusion success row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" measure rm -rf -- "${FX:?}" printf '%s %s/%s case rows green\n' "$([ "$FAILED" -eq 0 ] && echo SELF-TEST-GREEN || echo SELF-TEST-RED)" "$((CASES - FAILED))" "$CASES" @@ -146,7 +167,8 @@ m05_night_from_the_clock s/ night="\$(np_night "\$epoch")" || return 3/ ni m06_fetches_main s/"+refs\/heads\/nightly\/\$night:refs\/remotes\/\$remote\/nightly\/\$night"/main/ m07_switch_failure_ignored s/|| { printf .NC RED: cannot switch/|| true || { printf '"'"'NC RED: cannot switch/ m08_bad_at_accepted s/date -u -d "\$1" +%s 2>\/dev\/null || return 3/date -u +%s/ -m09_env_before_verify /^ np_verify "\$repo"/{h;d};/^ printf .NIGHTLY C=%s night=%s/{x;G}' +m09_env_before_verify /^ np_verify "\$repo"/{h;d};/^ printf .NIGHTLY C=%s night=%s/{x;G} +m10_pick_conclusion_ignored s/^ \[ "\$4" = success \] || {.*$/ true/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" @@ -182,7 +204,7 @@ mutants() { # ------------------------------------------------------------------------------------ main ---- main() { - local cmd repo=. remote=origin at='' + local cmd repo=. remote=origin at='' concl='' [ -f "$LIB" ] || caller_error "no library at $LIB" # shellcheck source=../lib/nightly_pick.sh . "$LIB" || caller_error "cannot source $LIB" @@ -190,9 +212,9 @@ main() { case "$cmd" in --self-test) self_test; return $? ;; --mutants) mutants; return $? ;; - -h | --help) sed -n '2,20p' "$SCRIPT_PATH"; return 0 ;; + -h | --help) sed -n '2,22p' "$SCRIPT_PATH"; return 0 ;; checkout) ;; - --repo | --remote | --at) set -- "$cmd" "$@" ;; + --repo | --remote | --at | --pick-conclusion) set -- "$cmd" "$@" ;; *) caller_error "unknown command '$cmd' (checkout|--self-test|--mutants)" ;; esac while [ "$#" -gt 0 ]; do @@ -200,10 +222,12 @@ main() { --repo) repo="${2:-}"; shift 2 || caller_error "--repo DIR" ;; --remote) remote="${2:-}"; shift 2 || caller_error "--remote NAME" ;; --at) at="${2:-}"; shift 2 || caller_error "--at ISO8601|EPOCH" ;; + --pick-conclusion) concl="${2:-}"; shift 2 || caller_error "--pick-conclusion CONCLUSION" ;; *) caller_error "unknown option '$1'" ;; esac done - [ -n "$at" ] || caller_error "--at ISO8601|EPOCH (the pick run's start) is required" - checkout_c "$repo" "$remote" "$at" + [ -n "$at" ] || caller_error "--at ISO8601|EPOCH (the pick run's created_at) is required" + [ -n "$concl" ] || caller_error "--pick-conclusion CONCLUSION (the pick run's conclusion) is required" + checkout_c "$repo" "$remote" "$at" "$concl" } main "$@"; exit $? From 5a7e4594f54d34866c1d100ffdc48573f9eced90 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 21:53:00 +0200 Subject: [PATCH 20/34] T44: a noon pick names the next night positively, not only by absence The 12:00:00Z row proved only that night N is not chosen. A row now plants night N+1's own pick and asserts 12:00:00Z measures it (night and tree). Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- scripts/release/nightly_c_checkout.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh index cfc70a985b..ac25d73f7b 100755 --- a/scripts/release/nightly_c_checkout.sh +++ b/scripts/release/nightly_c_checkout.sh @@ -141,6 +141,11 @@ self_test() { row n3_created_1158z_started_after_noon 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T11:58:00Z --pick-conclusion success row n5_last_second_is_still_the_night 0 "night=$n" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T11:59:59Z --pick-conclusion success row n6_noon_is_the_next_night 2 "has no pick" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T12:00:00Z --pick-conclusion success + # ...and names it positively: once night N+1 has its own pick, 12:00:00Z measures that pick, not night N's + git -C "$A" push -q origin "$c2:refs/heads/nightly/2026-10-05" || caller_error "fixture next pick" + git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c1" + row n6_noon_measures_the_next_pick 0 "night=2026-10-05" "night=$n" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-05T12:00:00Z --pick-conclusion success + same n6_noon_tree_is_the_next_pick "$(git -C "$W" rev-parse HEAD)" "$c2" # a pick that did not succeed names no night: not_measured, never main's head, nothing exported, the tree untouched git -C "$W" -c advice.detachedHead=false checkout -q --detach "$c2"; : > "$env" row n9_failed_pick_is_not_measured 2 "the pick concluded failure" "NIGHTLY C=" -- env GITHUB_ENV="$env" bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" --pick-conclusion failure From bd003f5592c61c510c633770615ba8e530cc73ee Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 00:06:42 +0200 Subject: [PATCH 21/34] fix(nightly_train): judge the newest commit every verdict lane ran on, over c310's pick proof (#4798) Authored fresh on a7w/4749-c310. J is the newest commit at most 6 behind the head on which every verdict lane has a measured run; a plain run stands for its head_sha, a chained run for the pick of its night only. A night with no pick ref falls to c310's not_measured; a failed pick read is not_measured on every lane, no fallback. Agent: aprender-w4798 Co-Authored-By: Claude Sonnet 5.5 --- Makefile | 2 +- contracts/nightly-train-v1.yaml | 28 ++-- docs/audits/impl-PMAT-4798-receipt.md | 42 +++++ docs/roadmaps/entries/PMAT-4798.yaml | 20 +++ scripts/release/nightly_train.sh | 216 +++++++++++++++++++++----- 5 files changed, 253 insertions(+), 55 deletions(-) create mode 100644 docs/audits/impl-PMAT-4798-receipt.md create mode 100644 docs/roadmaps/entries/PMAT-4798.yaml diff --git a/Makefile b/Makefile index 025a4d28ea..d6dc9e8c38 100644 --- a/Makefile +++ b/Makefile @@ -1557,7 +1557,7 @@ oracle-owl-check: oracle-owl || { echo "FAIL: tests/oracle/tbox-differential.json differs from a fresh run — commit it"; exit 1; } # ── BLD-002 R4: nightly evidence train (report-only) ──────────────────────────────────────────────────────── -# One line a night for main's head: RELEASABLE H= or NOT RELEASABLE: , . The timer runs a bundle copied +# One line a night for main: RELEASABLE H=, J the head unless the line prints a lag note or NOT RELEASABLE: , . The timer runs a bundle copied # out of git at pinned shas, never the working tree; all three default to HEAD, since they ship in one tree. OUT (and optionally INBOX) come from the command line: # make nightly-train-install OUT= [INBOX=] pin, self-test, install + enable the daily 04:45 UTC user timer # make nightly-train-run run the installed unit once, by hand, and print its line diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index e94ce960b1..9631d69d82 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges main's head C from the results its lanes' nightly producers recorded for C (a producer chained from the nightly pick is read by its workflow_run runs only, and only when the pick of the night the run was created in, and of the night before for a run created 12:00Z-17:59Z, is C), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night the run was created in, and of the night before for a run created 12:00Z-17:59Z, is J), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" @@ -21,17 +21,17 @@ five_whys: equations: lane_state: - formula: "state(lane, C) = not_measured if the lane has no producer, the read failed, no run of the producer on main is for C, the newest such run is unfinished, no matched job ran, or its attempt is unread; red if any matched job (every leg of a repeated name votes) ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 or later; else green" - domain: "one lane of the constant lane table, and the producer runs read for main's head C" + formula: "state(lane, J) = not_measured if the lane has no producer, the read failed, no run of the producer on main is for J, the newest such run is unfinished, no matched job ran, or its attempt is unread; red if any matched job (every leg of a repeated name votes) ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 or later; else green" + domain: "one lane of the constant lane table, and the producer runs read for the judged commit J, the one commit every lane is judged on, at most 6 commits behind main's head C" codomain: "{green, red, not_measured} for each lane" invariants: - - "a run on another commit never stands for C" + - "a run on another commit never stands for J; all lanes are judged on the one J, never a per-lane mix" - "an unfinished run, an unread attempt or a failed read is not_measured, never green" - "a retried success is red: a retried green is not a green" - "a run whose matched jobs were skipped, cancelled or neutral, or fewer than the lane requires, measured nothing, and the next older run for C is taken; only the matched jobs decide, the run's own conclusion is not read" verdict_line: - formula: "RELEASABLE H=C iff every verdict lane is green; else NOT RELEASABLE: L, R where L is the first failing verdict lane, red before not_measured, then by hours lost, then table order, and R its run id or not_measured" + formula: "RELEASABLE H=J iff every verdict lane is green; else NOT RELEASABLE: L, R where L is the first failing verdict lane, red before not_measured, then by hours lost, then table order, and R its run id or not_measured" domain: "lane_state over the constant lane table; info lanes are read and reported but never enter the verdict" codomain: "exactly one line on stdout per run" invariants: @@ -43,18 +43,18 @@ equations: proof_obligations: - id: NT-INV-001 type: invariant - property: "RELEASABLE only when every verdict lane is green on C" - formal: "line = RELEASABLE(C) ⇒ ∀ l ∈ verdict: state(l, C) = green" + property: "RELEASABLE only when every verdict lane is green on J" + formal: "line = RELEASABLE(J) ⇒ ∀ l ∈ verdict: state(l, J) = green" applies_to: verdict_line - id: NT-INV-002 type: invariant property: "A lane that cannot be read is never green" - formal: "∀ l: read_failed ∨ no_run(l, C) ∨ unfinished(l, C) ∨ attempt_unread(l, C) ∨ job_list_truncated(l, C) ∨ rollup_truncated(C) ∨ no_producer(l) ⇒ state(l, C) = not_measured" + formal: "∀ l: read_failed ∨ no_run(l, J) ∨ unfinished(l, J) ∨ attempt_unread(l, J) ∨ job_list_truncated(l, J) ∨ rollup_truncated(J) ∨ no_producer(l) ⇒ state(l, J) = not_measured" applies_to: lane_state - id: NT-INV-003 type: invariant property: "A retried success is red" - formal: "∀ l: success(run(l, C)) ∧ attempt(run(l, C)) ≥ 2 ⇒ state(l, C) = red" + formal: "∀ l: success(run(l, J)) ∧ attempt(run(l, J)) ≥ 2 ⇒ state(l, J) = red" applies_to: lane_state - id: NT-INV-004 type: invariant @@ -68,8 +68,8 @@ proof_obligations: applies_to: verdict_line - id: NT-INV-006 type: invariant - property: "A void run is never the evidence; the next older run on C is read instead. Void is decided by the matched jobs only" - formal: "∀ r: (matched_jobs(r) = ∅ ∨ (¬failed(r) ∧ ∃ j ∈ matched_jobs(r): conclusion(j) ∈ {skipped, cancelled, neutral}) ∨ (¬failed(r) ∧ |distinct_jobs(r)| < need(l))) ⇒ pick(l, C) ≠ r" + property: "A void run is never the evidence; the next older run on J is read instead. Void is decided by the matched jobs only" + formal: "∀ r: (matched_jobs(r) = ∅ ∨ (¬failed(r) ∧ ∃ j ∈ matched_jobs(r): conclusion(j) ∈ {skipped, cancelled, neutral}) ∨ (¬failed(r) ∧ |distinct_jobs(r)| < need(l))) ⇒ pick(l, J) ≠ r" applies_to: lane_state - id: NT-INV-007 type: invariant @@ -114,13 +114,13 @@ proof_obligations: - id: NT-INV-015 type: invariant property: "Under --exit-verdict exit 0 means RELEASABLE and every NOT RELEASABLE exits non-zero; a RELEASABLE run still exits non-zero when a later step fails (inbox read-back, NT-INV-013); without the flag the status says only whether the train ran" - formal: "exit_verdict(run) ⇒ (exit(run) = 0 ⇒ line(run) = RELEASABLE(C)) ∧ (line(run) ≠ RELEASABLE(C) ⇒ exit(run) ≠ 0)" + formal: "exit_verdict(run) ⇒ (exit(run) = 0 ⇒ line(run) = RELEASABLE(J)) ∧ (line(run) ≠ RELEASABLE(J) ⇒ exit(run) ≠ 0)" applies_to: verdict_line falsification_tests: - id: FALSIFY-NT-001 rule: "The judge is right on every row of its case table" - prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on C; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run, on a run whose night picked another commit or whose pick was not read, and on a run created 12:00Z-17:59Z whose night before picked another commit; a schedule lane ignores the picks; the query reads refs/heads/nightly/ and only dated night names count" + prediction: "All verdict lanes green is RELEASABLE; a removed lane, a run on another commit, a run in progress, an unread attempt or a failed read is NOT RELEASABLE with not_measured; a green at attempt 2 is red; an info lane red leaves the verdict; reds rank by hours lost; greens in a row follow the scheduled-nights rule; a planted fault prints the trap line; a void newer run falls to the older run on J; a lane missing a required job, a cut job list or no producer is not_measured; stdout is one line; a fault or a closed stdout after the history row adds no second row; an edited bundle refuses and still leaves its line and one history row; under --pinned a missing PIN, an edited PIN or sums that omit PIN refuse while an intact pinned bundle judges; a failed parallel leg of one name is red; a cut rollup is marked; the read refuses under the rate floor, past the call budget and on a GraphQL error; an info lane attempt not read shows 1?; an inbox read-back mismatch exits non-zero; a closed stdout keeps the real verdict; a registry token variable, an empty named one or a credentials file or a legacy config token, under CARGO_HOME or ~/.cargo, refuses to start and the probe is clear only where none is reachable; the read refuses at 999 of 5000 and 199 of 1000, reads at 200 of 1000, and refuses an unread limit; --exit-verdict maps RELEASABLE to 0 and NOT RELEASABLE to 1, and a full RELEASABLE run under it exits 0; a self-test and a planted fault write only a throwaway inbox; a lane chained from the pick is green on its workflow_run run and not_measured on a schedule run, on a run whose night picked another commit or whose pick was not read, and on a run created 12:00Z-17:59Z whose night before picked another commit; a schedule lane ignores the picks; the query reads refs/heads/nightly/ and only dated night names count; the judged commit is the newest of the last 7 on which every verdict producer has a measured run, a newer partial commit never wins, the lag cap counts commits without runs, and a red on J is red; a night with no pick ref is not_measured for a chained lane while a pick read that failed is not_measured for every lane and never the fallback" test: "bash scripts/release/nightly_train.sh --self-test" if_fails: "The train can print RELEASABLE for a commit some release-day check never measured, or stay silent when it fails" @@ -133,7 +133,7 @@ falsification_tests: qa_gate: id: F-NT-001 name: "Main's head is called releasable only from green evidence for every release-day check" - description: "scripts/release/nightly_train.sh reads the night's results for main's head and prints RELEASABLE H= or NOT RELEASABLE: , . Its case table pins every lane state and the verdict set; its mutants prove the table can fail." + description: "scripts/release/nightly_train.sh reads the night's results for one judged commit J of main (the head, or the newest commit up to 6 behind it that every verdict producer ran on) and prints RELEASABLE H= or NOT RELEASABLE: , . Its case table pins every lane state and the verdict set; its mutants prove the table can fail." checks: - "lane_state" - "verdict_line" diff --git a/docs/audits/impl-PMAT-4798-receipt.md b/docs/audits/impl-PMAT-4798-receipt.md new file mode 100644 index 0000000000..8d0b99a585 --- /dev/null +++ b/docs/audits/impl-PMAT-4798-receipt.md @@ -0,0 +1,42 @@ +# Receipt: PMAT-4798 (aprender#4798) nightly judge: newest commit every verdict lane ran on + +Change: `scripts/release/nightly_train.sh`, the matching amendment of `contracts/nightly-train-v1.yaml` (C becomes J), one +`Makefile` comment, and the roadmap entry `docs/roadmaps/entries/PMAT-4798.yaml`. Authored fresh on the c310 pick-window +proof (#4749, `a7w/4749-c310` @ 067e2e0646). Not touched: `.github/workflows/nightly-train.yml` (its step label still +says "main's head"; a workflow edit needs its own gate-keeping proof). + +## Ticket intent +The nightly judged C = main's head at the read and counted a lane only from a run on C, so any merge after the producers +fired voided the night. Now it judges J = the newest commit, at or before the read, at most 6 commits behind the head, on +which EVERY verdict lane has a measured (green or red) run. None such = judged on the head, where a lane without a run +reads not_measured. There is no other fallback. The line prints `(judged , head , lag k commits)` when J is +not the head. + +## How c310's rule carries to J +- A plain (schedule, push) run stands for the commit it ran on: it counts for J when its head_sha is J. +- A chained (`workflow_run`) run's tree is the pick of its night, never its head_sha, so it stands for that pick only: + it counts for J when `nightc(created_at)` == J, and for a run created 12:00Z-17:59Z also `prevc` == J. The J search + applies the same rule when it asks whether a commit has a measured run on every lane. +- Candidates are main's head (rank 0) and its history from the SAME GraphQL query (`history(first: 7)`), written to + `RAW/rank.tsv`; lag is the rank of J (exact). One J for all lanes, never a per-lane mix. A rank list whose rank 0 is + not the head read is ignored. + +## Two unlike failures (a7's condition) +- "No pick for that night": the nightly refs were read and that night has none. The chained lane is not_measured with the + reason "measured the pick no pick for that night" (c310's behaviour); plain lanes are unaffected. Case row + `a_chained_run_without_a_read_pick_is_not_measured`; read-level row `an_absent_pick_is_a_good_read`. +- "Pick read failed": the refs field is not an array (fetch/parse error). `picks_read` writes `pickfail`; `evaluate` turns + the read into `failed: pick read: ...` and EVERY lane is not_measured, with no fallback. Rows + `a_failed_pick_read_is_not_measured_never_a_fallback`, `a_pick_read_with_no_refs_field_failed`. +- Mutants that merge them: `m66_failed_pick_read_falls_back` (evaluate ignores `pickfail`) and + `m67_failed_pick_read_is_an_absent_pick` (the array check is dropped, so a failed read looks like an absent pick). + +## What is measured +- `--self-test`: 87/87 rows. `--mutants`: 67/67 killed (m43-m67; m55-m65 are the J search, m66 and m67 the pick split). +- Merge-queue history on main is not strictly first-parent; those commits carry no main runs and only shorten the + lookback. Conservative. + +## Not claimed +- No replay of 2026-10-05: the saved raw predates the history read, so a replay reads only the head. The first + measurement is the next live read. +- If the verdict is ever wired to the release, require tree(J) == tree(MC). Not done here. diff --git a/docs/roadmaps/entries/PMAT-4798.yaml b/docs/roadmaps/entries/PMAT-4798.yaml new file mode 100644 index 0000000000..1c8fa7f1da --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4798.yaml @@ -0,0 +1,20 @@ +- id: PMAT-4798 + github_issue: 4798 + item_type: task + title: 'nightly_train: judge the newest commit all verdict producers share, not main''s head' + status: inprogress + priority: high + assigned_to: aprender-w4798 + created: 2026-10-05T14:47:26Z + updated: 2026-10-05T14:47:26Z + spec: null + acceptance_criteria: + - 'The nightly judge judges the newest commit, at most 6 behind main''s head, on which every verdict lane has a measured run; none = the head (not_measured); lag printed (judged, head, lag k commits)' + - 'Self-test and mutants pin it; the release path is unchanged' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Landing waits on Q5 shared (operator, via the cop).' + epic: 4678 diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index f665059cb0..81ac5cc5bf 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -8,7 +8,11 @@ # NOT RELEASABLE: , the first failing verdict lane, then (+N more) # NOT RELEASABLE: nightly-train, ... the train itself failed part-way (the EXIT trap prints it) # and every line ends with [C= pin=..]: the commit judged and the shas of the three -# scripts that judged it. H is the commit id of C, main's head at the moment of the read. +# scripts that judged it. H is the commit id of C. C is the head of main at the moment of the read, unless the head has +# no measured run from some verdict producer: then C is the newest commit, at most MAX_LAG commits behind the head, +# on which every verdict producer has a measured run, and the line adds " (judged , head , lag k +# commits)". All lanes are judged on that one commit, never a per-lane mix. None within MAX_LAG = the head, which +# reads not_measured for a lane with no run there; there is no other fallback (#4798). # # READ, DON'T RE-RUN. Night 1 runs nothing. Each lane takes the result its existing nightly producer (a workflow on # main and, optionally, a job-name pattern) recorded for C. A producer that chains from "Nightly pick" is read by @@ -71,6 +75,7 @@ HERE="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_PATH="$HERE/${BASH_SOURCE[0]##*/}" REPO="paiml/aprender" MAX_CALLS=10 +MAX_LAG=6 # J (the commit judged) is at most this many commits behind the head of main RATE_FLOOR=1000 UNIT="aprender-nightly-train" # lane;kind;release-day check;producer workflow;event pattern;job-name pattern[;required job names, default 1] @@ -114,6 +119,8 @@ evaluate() { [ -f "$2/attempts.tsv" ] || : > "$2/attempts.tsv" [ -f "$2/runs.tsv" ] || : > "$2/runs.tsv" [ -f "$2/picks.tsv" ] || : > "$2/picks.tsv" + # a pick read that failed judges every lane not_measured (never the c310 fallback for an absent pick, #4798) + [ -s "$2/pickfail" ] && [ "${rd:-}" = ok ] && rd="failed: pick read: $(head -n 1 "$2/pickfail")" # each picked night as the window of run creation times it owns, [N 12:00Z, N+1 12:00Z), and its C local n o e while IFS=$'\t' read -r n o; do @@ -121,10 +128,12 @@ evaluate() { e="$(date -u -d "$n + 1 day" +%F 2>/dev/null)" || continue printf '%sT12:00:00Z\t%sT12:00:00Z\t%s\n' "$n" "$e" "$o" done < "$2/picks.tsv" > "$2/pickwin.tsv" - awk -F '\t' -v OFS='\t' -v C="$c" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" ' + [ -f "$2/rank.tsv" ] || : > "$2/rank.tsv" + awk -F '\t' -v OFS='\t' -v C="$c" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" -v JOUT="$2/judged" -v MAXLAG="$MAX_LAG" ' FILENAME == ARGV[1] { split($0, a, ";"); n++; L[n] = a[1]; K[n] = a[2]; CK[n] = a[3]; WF[n] = a[4]; EV[n] = a[5]; RE[n] = a[6]; NEED[n] = (a[7] ~ /^[0-9]+$/ ? a[7] + 0 : 1); next } FILENAME == ARGV[2] { AT[$1] = $2; next } FILENAME == ARGV[3] { np++; PS[np] = $1; PE[np] = $2; PO[np] = $3; next } + FILENAME == ARGV[4] { RKO[$1 + 0] = $2; NRK++; next } { key = $3 SUBSEP $10 SUBSEP $13 if (key in seen) next @@ -169,6 +178,34 @@ evaluate() { END { if (MODE == "final") print "check", "run_id", "created_at", "branch", "event", "conclusion", "attempt" > HIST readok = (RD == "ok" && C ~ /^[0-9a-f]{40}$/) + # J, the commit judged (#4798): the newest commit on main, at or before the read, on which EVERY verdict lane with + # a producer has a measured (green or red) run; C itself when it has them. One commit for all lanes, never a + # per-lane mix. The candidates are rank.tsv: the head of main (rank 0) and its history, newest first, read in the + # same query; LAG is the rank of J, the commits between J and the head. Only ranks 0..MAXLAG are candidates. None + # qualifies (or no rank list): J = C and every lane reads as it would on C, which is not_measured for a lane + # with no run there. Nothing falls back past MAXLAG. + J = C; LAG = 0 + if (readok && RKO[0] == C) { + split("", HAS); nprod = 0 + for (i = 1; i <= n; i++) { + if (WF[i] == "-" || K[i] != "verdict") continue + nprod++ + for (q = 1; q <= nr; q++) { + r = RID[q] + if (RW[r] == WF[i] && REV[r] ~ EV[i] && RBR[r] == "main") { + # a chained run stands for the pick of its night, a plain run for its own commit (see the lane loop below) + if (EV[i] == "^workflow_run$") { hm = nightc(RC[r]); if (substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && prevc(RC[r]) != hm) hm = "" } + else hm = RH[r] + s = lst(i, r); if (hm != "" && (s == "green" || s == "red")) HAS[hm, i] = 1 } + } + } + for (k = 0; k <= MAXLAG && k < NRK && nprod > 0; k++) { + h = RKO[k]; all = 1 + for (i = 1; i <= n && all; i++) if (WF[i] != "-" && K[i] == "verdict" && !((h, i) in HAS)) all = 0 + if (all) { J = h; LAG = k; break } + } + } + if (MODE == "final") print J, C, LAG > JOUT for (i = 1; i <= n; i++) { ST = ""; EN = "" if (WF[i] == "-") { if (MODE == "final") out(i, "not_measured", "", "", "no nightly producer on main yet"); continue } @@ -185,17 +222,18 @@ evaluate() { hc = (s == "green" ? "success" : (s == "red" ? "failure" : "cancelled")) print L[i], r, RC[r], "main", REV[r], hc, ha > HIST } - if (RH[r] != C) { if (other == "") other = r " on " substr(RH[r], 1, 10); continue } - # a chained run is filed under the head of main when the pick finished, but measured the pick of its night: count it - # only when that pick is C (a re-dispatched pick keeps an older C and still fires with the newer head) - if (EV[i] == "^workflow_run$" && (pc = nightc(RC[r])) != C) { if (other == "") other = r " measured the pick " (pc == "" ? "unread" : substr(pc, 1, 10)); continue } - # a run created 12:00Z-17:59Z may come from a pick that started before noon: the night before must be C too - if (EV[i] == "^workflow_run$" && substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && (pc = prevc(RC[r])) != C) { if (other == "") other = r " may have measured the pick " (pc == "" ? "unread" : substr(pc, 1, 10)); continue } + # a plain run stands for the commit it ran on. A chained run is filed under the head of main when the pick finished, + # but its tree is the pick of its night (nightly_c_checkout.sh), so it stands for that pick and for no other commit: + # it counts only when that pick is J (#4798; a re-dispatched pick keeps an older C and still fires with the newer head) + if (EV[i] != "^workflow_run$") { if (RH[r] != J) { if (other == "") other = r " on " substr(RH[r], 1, 10); continue } } + else if ((pc = nightc(RC[r])) != J) { if (other == "") other = r " measured the pick " (pc == "" ? "no pick for that night" : substr(pc, 1, 10)); continue } + # a run created 12:00Z-17:59Z may come from a pick that started before noon: the night before must be J too + if (EV[i] == "^workflow_run$" && substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && (pc = prevc(RC[r])) != J) { if (other == "") other = r " may have measured the pick " (pc == "" ? "no pick for that night" : substr(pc, 1, 10)); continue } if (pick == "" && s != "void") { pick = r; pst = s; pwhy = WHY; pS = ST; pE = EN; pD = DUP } else if (pick == "" && pwhy == "") pwhy = WHY } if (pick == "") { - if (MODE == "final") out(i, "not_measured", "", "", (pwhy != "" ? pwhy : "no run on C") (other != "" ? "; newest run " other : (m == 0 ? "; no run on main" : ""))) + if (MODE == "final") out(i, "not_measured", "", "", (pwhy != "" ? pwhy : "no run on the judged commit") (other != "" ? "; newest run " other : (m == 0 ? "; no run on main" : ""))) continue } ST = pS; EN = pE @@ -218,7 +256,7 @@ evaluate() { if (att > 1) out(i, "red", pick, att, "success only at attempt " att ": a retried green is not a green") else out(i, "green", pick, att, "") } - }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/pickwin.tsv" "$2/runs.tsv" + }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/pickwin.tsv" "$2/rank.tsv" "$2/runs.tsv" } # hours LANESOUT REDAGE_OUTPUT_FILE -> lanehours lost (2 decimals) for every lane red_age measured hours() { @@ -231,9 +269,11 @@ hours() { } # decide LANESFILE RAW NOW -> RAW/lanes.tsv, RAW/reds.tsv, RAW/line (the verdict, without the pin suffix) decide() { - local raw="$2" c + local raw="$2" c j lag c="$(cat "$raw/C" 2>/dev/null)" evaluate "$1" "$raw" final > "$raw/lanes.tsv" || return 1 + # the judged commit J and its lag (#4798); the line names main's head and the lag whenever J is not the head + read -r j _ lag < "$raw/judged" 2>/dev/null; [[ "$j" =~ ^[0-9a-f]{40}$ && "$lag" =~ ^[0-9]+$ ]] || { j="$c"; lag=0; } : > "$raw/redage.out" if [ -x "$HERE/red_age.sh" ] || [ -f "$HERE/red_age.sh" ]; then bash "$HERE/red_age.sh" --history "$raw/hist_redage.tsv" --as-of "$3" > "$raw/redage.out" 2>&1 @@ -247,15 +287,16 @@ decide() { "$raw/hours.tsv" "$raw/lanes.tsv" | sort -t "$(printf '\t')" -k1,1gr -k2,2n -k3,3n -k4,4n \ | awk -F '\t' -v OFS='\t' 'BEGIN { print "rank", "lane", "kind", "state", "hours_lost", "run_id", "reason" } { print NR, $5, $6, $7, $8, $9, $10 }' > "$raw/reds.tsv" # the line: the first failing verdict lane (red before not_measured, then hours lost, then table order) - awk -F '\t' -v C="$c" 'FILENAME == ARGV[1] { H[$1] = $2; next } + awk -F '\t' -v C="$j" -v HEAD="$c" -v LAG="${lag:-0}" 'FILENAME == ARGV[1] { H[$1] = $2; next } $2 == "verdict" { nv++; if ($3 == "green") next bad++; h = ($1 in H) ? H[$1] + 0 : -1; s = ($3 == "red" ? 1 : 0) if (best == "" || s > bs || (s == bs && h > bh)) { best = $1; brun = ($3 == "red" ? $5 : "not_measured"); bs = s; bh = h } } END { + note = (C != HEAD && HEAD != "" ? " (judged " substr(C, 1, 10) ", head " substr(HEAD, 1, 10) ", lag " LAG " commits)" : "") if (nv == 0) { print "NOT RELEASABLE: nightly-train, the lane table has no verdict lane"; exit } - if (bad == 0 && C ~ /^[0-9a-f]{40}$/) { print "RELEASABLE H=" C; exit } + if (bad == 0 && C ~ /^[0-9a-f]{40}$/) { print "RELEASABLE H=" C note; exit } if (bad == 0) { print "NOT RELEASABLE: nightly-train, no commit read"; exit } - printf "NOT RELEASABLE: %s, %s%s\n", best, brun, (bad > 1 ? " (+" bad - 1 " more)" : "") + printf "NOT RELEASABLE: %s, %s%s%s\n", best, brun, (bad > 1 ? " (+" bad - 1 " more)" : ""), note }' "$raw/hours.tsv" "$raw/lanes.tsv" > "$raw/line" } # greens HISTORY DATE -> the streak of nightly-train in HISTORY by nightly_greens.sh's rules, or not_measured @@ -273,14 +314,22 @@ call_ok() { [ "$CALLS" -lt "$MAX_CALLS" ] || return 1; CALLS=$((CALLS + 1)); } # gql_query LANES WFJSON -> the one GraphQL query; empty when a producer workflow is missing from the list. # jq 1.6 compatible (the timer PATH may find it first): no reserved words such as $or as variable names. gql_query() { - printf '%s\n' "$1" | awk -F ';' '$4 != "-" { print $4 }' | sort -u | jq -R -s --slurpfile wf "$2" -r --arg repo "$REPO" ' + printf '%s\n' "$1" | awk -F ';' '$4 != "-" { print $4 }' | sort -u | jq -R -s --slurpfile wf "$2" -r --arg repo "$REPO" --argjson hn "$((MAX_LAG + 1))" ' split("\n") | map(select(length > 0)) as $want | ($wf | first | .workflows | map(select(.path as $x | $want | index($x))) ) as $hit | if ($hit | length) != ($want | length) then error("producer workflow missing from the list") else . end | ($repo | split("/")) as $own - | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } } " + | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } history(first: \($hn)) { nodes { oid statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } } " + ([$hit | to_entries[] | "w\(.key): node(id: \"\(.value.node_id)\") { ... on Workflow { id runs(first: 12) { nodes { databaseId createdAt event checkSuite { status conclusion branch { name } commit { oid } checkRuns(first: 100, filterBy: {checkType: ALL}) { pageInfo { hasNextPage } nodes { name status conclusion startedAt completedAt } } } } } } }"] | join(" ")) + " }"' 2>/dev/null } +# picks_read GQLJSON PICKS PICKFAIL -> PICKS (nightC), and PICKFAIL empty when the refs were read. An absent ref is a row +# missing from a good read (PICKFAIL stays empty: the chained lane falls to c310's "no pick for that night"); a response whose +# refs are not an array is a failed read (PICKFAIL holds the reason: every lane is not_measured, no fallback) (#4798) +picks_read() { + : > "$3" + if jq -e '(.data.repository.nightly.nodes | type) == "array"' "$1" > /dev/null 2>&1 && picks_of "$1" > "$2" 2>/dev/null; then : + else : > "$2"; printf 'the nightly pick refs did not parse\n' > "$3"; fi +} # picks_of GQLJSON -> nightC for each refs/heads/nightly/ the query returned; other names are dropped picks_of() { jq -r '(.data.repository.nightly.nodes // [])[] | select(.name | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) | [.name, .target.oid] | @tsv' "$1"; } normalize() { @@ -302,23 +351,29 @@ normalize() { ["wf", ($p | .[$w.id]), $r.databaseId, $r.event, ($r.checkSuite.branch.name // ""), $r.checkSuite.commit.oid, $r.createdAt, $r.checkSuite.status, ($r.checkSuite.conclusion // ""), "#truncated", "COMPLETED", "TRUNCATED", "", "", 1] else empty end) ), - ( ($c.statusCheckRollup.contexts.nodes // [])[] | select(.checkSuite.workflowRun != null) - | .checkSuite.workflowRun.workflow.id as $wid | ["rollup", ($p | .[$wid]), .checkSuite.workflowRun.databaseId, .checkSuite.workflowRun.event, - (.checkSuite.branch.name // ""), $c.oid, .checkSuite.workflowRun.createdAt, .checkSuite.status, (.checkSuite.conclusion // ""), - .name, .status, (.conclusion // ""), (.startedAt // ""), (.completedAt // ""), 1] ), - ( if ($c.statusCheckRollup.contexts.pageInfo.hasNextPage // false) then - [($c.statusCheckRollup.contexts.nodes // [])[] | select(.checkSuite.workflowRun != null)] | unique_by(.checkSuite.workflowRun.databaseId) | .[] - | .checkSuite.workflowRun.workflow.id as $wid | ["rollup", ($p | .[$wid]), .checkSuite.workflowRun.databaseId, .checkSuite.workflowRun.event, - (.checkSuite.branch.name // ""), $c.oid, .checkSuite.workflowRun.createdAt, .checkSuite.status, (.checkSuite.conclusion // ""), - "#truncated", "COMPLETED", "TRUNCATED", "", "", 1] - else empty end ) + # the head and the newer history of main (#4798): a push run on an older main commit can fall out of the 12 newest runs of its workflow + ( [$c] + [($c.history.nodes // [])[] | select(.oid != $c.oid)] | .[] | . as $x + | ( ($x.statusCheckRollup.contexts.nodes // [])[] | select(.checkSuite.workflowRun != null) + | .checkSuite.workflowRun.workflow.id as $wid | ["rollup", ($p | .[$wid]), .checkSuite.workflowRun.databaseId, .checkSuite.workflowRun.event, + (.checkSuite.branch.name // ""), $x.oid, .checkSuite.workflowRun.createdAt, .checkSuite.status, (.checkSuite.conclusion // ""), + .name, .status, (.conclusion // ""), (.startedAt // ""), (.completedAt // ""), 1] ), + ( if ($x.statusCheckRollup.contexts.pageInfo.hasNextPage // false) then + [($x.statusCheckRollup.contexts.nodes // [])[] | select(.checkSuite.workflowRun != null)] | unique_by(.checkSuite.workflowRun.databaseId) | .[] + | .checkSuite.workflowRun.workflow.id as $wid | ["rollup", ($p | .[$wid]), .checkSuite.workflowRun.databaseId, .checkSuite.workflowRun.event, + (.checkSuite.branch.name // ""), $x.oid, .checkSuite.workflowRun.createdAt, .checkSuite.status, (.checkSuite.conclusion // ""), + "#truncated", "COMPLETED", "TRUNCATED", "", "", 1] + else empty end ) ) | map(tostring) | join("\t")' "$1" } +# rank_rows GQL -> "rankoid", the head of main (rank 0) and then its history, newest first, the head once (#4798) +rank_rows() { + jq -r '.data.repository.defaultBranchRef.target as $c | ([$c.oid] + [($c.history.nodes // [])[] | .oid | select(. != $c.oid)]) | to_entries[] | "\(.key)\t\(.value)"' "$1" +} # fetch RAW CACHE -> RAW/{C,tree,read,runs.tsv,attempts.tsv,graphql.json,workflows.json}; never fails: a failed read # is recorded in RAW/read and makes every producer lane not_measured fetch() { local raw="$1" cache="$2" lim rem fl hdr st q a ids r - printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv"; : > "$raw/picks.tsv" + printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv"; : > "$raw/picks.tsv"; : > "$raw/rank.tsv"; : > "$raw/pickfail" read -r lim rem <<< "$(gh api rate_limit --jq '"\(.resources.core.limit) \(.resources.core.remaining)"' 2>/dev/null)" case "$lim:$rem" in *[!0-9:]*|:*|*:) printf 'failed: rate_limit unreadable\n' > "$raw/read"; return 0 ;; esac # the floor is a fifth of the token's own hourly limit, capped at RATE_FLOOR: the fleet PAT (5000) keeps its 1000 @@ -346,9 +401,12 @@ fetch() { jq -r '.data.repository.defaultBranchRef | select(.name == "main") | .target.oid // empty' "$raw/graphql.json" > "$raw/C" 2>/dev/null jq -r '.data.repository.defaultBranchRef.target.tree.oid // empty' "$raw/graphql.json" > "$raw/tree" 2>/dev/null # the night's picks (refs/heads/nightly/): a chained lane counts a run only when its night picked C - picks_of "$raw/graphql.json" > "$raw/picks.tsv" 2>/dev/null + # "no pick for that night" (the ref is absent: a row missing from a good read) is not "pick read failed" (the refs came + # back unusable): the second is written to pickfail and judges every lane not_measured, it never falls back to the first + picks_read "$raw/graphql.json" "$raw/picks.tsv" "$raw/pickfail" grep -q -E '^[0-9a-f]{40}$' "$raw/C" || { printf 'failed: no head of main in the response\n' > "$raw/read"; return 0; } normalize "$raw/graphql.json" "$raw/workflows.json" > "$raw/runs.tsv" 2>/dev/null || { printf 'failed: response did not normalize\n' > "$raw/read"; return 0; } + rank_rows "$raw/graphql.json" > "$raw/rank.tsv" 2>/dev/null || : > "$raw/rank.tsv" printf 'ok\n' > "$raw/read" # run_attempt for each green verdict candidate (GraphQL has none); an unread attempt stays unread -> not_measured ids="$(evaluate "$LANES" "$raw" cand | sort -u)" @@ -382,7 +440,7 @@ no_token() { fi done } -# verdict_rc LINE -> 0 for RELEASABLE, 1 for anything else (--exit-verdict: a green job means a releasable head) +# verdict_rc LINE -> 0 for RELEASABLE, 1 for anything else (--exit-verdict: a green job means a releasable judged commit J, named with the head and the lag when J is not the head) verdict_rc() { case "$1" in "RELEASABLE "*) return 0 ;; *) return 1 ;; esac; } inbox_line() { # inbox_line LINE GREENS -> append <= 300 bytes and read it back local l who="${NIGHTLY_TRAIN_REPORTER:-nightly-r4}" # the reporter is a role, named by the caller, never written here @@ -411,7 +469,7 @@ on_exit() { exit 2 } run_train() { - local from="$1" raw line g concl budget pin_t pin_g pin_r hrow + local from="$1" raw line g concl budget pin_t pin_g pin_r hrow j HSHA trap on_exit EXIT DAY="${NOW%%T*}" # before any step that can fail, so every failure leaves its line and its history row step no-token @@ -436,11 +494,13 @@ run_train() { mkdir -p "$raw" || exit 1 step read if [ -n "$from" ]; then - for f in C read runs.tsv attempts.tsv tree picks.tsv; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done + for f in C read runs.tsv attempts.tsv tree picks.tsv rank.tsv pickfail; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done else fetch "$raw" "$OUTDIR/cache"; fi CSHA="$(cat "$raw/C" 2>/dev/null)" step judge decide "$LANES" "$raw" "$NOW" || exit 1 + # from here C is the commit judged (#4798); HSHA keeps main's head for the bundle + HSHA="$CSHA"; read -r j _ _ < "$raw/judged" 2>/dev/null; [[ "${j:-}" =~ ^[0-9a-f]{40}$ ]] && CSHA="$j" step rank [ -s "$raw/line" ] || exit 1 cp "$raw/reds.tsv" "$OUTDIR/$DAY/reds.tsv" || exit 1 @@ -458,7 +518,8 @@ run_train() { { printf 'lane\tkind\tstate\tproducer\trun_id\trun_head\tconclusion\tattempt\tstarted\tended\treason\thours_lost\n' awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { H[$1] = $2; next } { print $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, (($1 in H) ? H[$1] : "-") }' "$raw/hours.tsv" "$raw/lanes.tsv" - printf '# C\t%s\n# tree\t%s\n# read\t%s\n# command\t%s\n# pin\ttrain=%s greens=%s redage=%s\n' "$CSHA" "$(cat "$raw/tree")" "$(cat "$raw/read")" "$PROG ${ARGS_SEEN:-}" "${pin_t:-unpinned}" "${pin_g:-}" "${pin_r:-}" + printf '# head\t%s\n' "$HSHA" + printf '# C\t%s\n# tree\t%s\n# read\t%s\n# command\t%s\n# pin\ttrain=%s greens=%s redage=%s\n' "$CSHA" "$(if [ "$CSHA" = "$HSHA" ]; then cat "$raw/tree"; else printf 'not read (the read returns the tree of the head only)'; fi)" "$(cat "$raw/read")" "$PROG ${ARGS_SEEN:-}" "${pin_t:-unpinned}" "${pin_g:-}" "${pin_r:-}" printf '# budget\tfirst start %s, last end %s, %s (6 h budget; info only, not in the verdict)\n' "${budget%% *}" "${budget##* }" "$(span "${budget%% *}" "${budget##* }")" printf '# github calls\t%s\n# greens in a row\t%s\n# as of\t%s\n' "$CALLS" "$g" "$NOW" } > "$OUTDIR/$DAY/bundle.tsv" || exit 1 @@ -551,6 +612,7 @@ v-c;verdict;check C;.github/workflows/c.yml;^push$;^(gate|test)$;2 i-d;info;check D;.github/workflows/d.yml;^schedule$;' ST_C="$(printf 'a%.0s' $(seq 40))" ST_X="$(printf 'b%.0s' $(seq 40))" +ST_Y="$(printf 'd%.0s' $(seq 40))" # fixture DIR -> every lane green on C, attempt 1 fixture() { mkdir -p "$1" @@ -565,7 +627,10 @@ fixture() { } > "$1/runs.tsv" printf '101\t1\n201\t1\n301\t1\n' > "$1/attempts.tsv" printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_C" "$ST_X" > "$1/picks.tsv" # run 201 (02:00Z on the 4th) is night 2026-10-03 + printf '0\t%s\n' "$ST_C" > "$1/rank.tsv" } +# ranks DIR SHA... -> DIR/rank.tsv: main's head and history, newest first (rank 0 = the head, aaaa...) +ranks() { local d="$1" k=0 s; shift; : > "$d/rank.tsv"; for s in "$@"; do printf '%s\t%s\n' "$k" "$s" >> "$d/rank.tsv"; k=$((k + 1)); done; } # st_decide DIR -> the line, the lanes and the reds, as one text st_decide() { decide "$ST_LANES" "$1" 2026-10-04T06:00:00Z; cat "$1/line" "$1/lanes.tsv" "$1/reds.tsv"; } self_test() { @@ -597,7 +662,9 @@ self_test() { d="$tmp/repick"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; sed -i "1s/$ST_C/$ST_X/" "$d/picks.tsv" row a_repicked_night_never_credits_main_head 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" d="$tmp/nopick"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; : > "$d/picks.tsv" - row a_chained_run_without_a_read_pick_is_not_measured 0 "201 measured the pick unread" "RELEASABLE H=" -- st_chainl "$d" + row a_chained_run_without_a_read_pick_is_not_measured 0 "201 measured the pick no pick for that night" "RELEASABLE H=" -- st_chainl "$d" + d="$tmp/pickfail"; fixture "$d"; printf 'the nightly pick refs did not parse\n' > "$d/pickfail" + row a_failed_pick_read_is_not_measured_never_a_fallback 0 "failed: pick read: the nightly pick refs did not parse" "RELEASABLE H=" -- st_decide "$d" d="$tmp/ownnight"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T13:00:00Z\t/' "$d/runs.tsv" row a_chained_run_is_judged_by_its_own_night 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" d="$tmp/atnoon"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T12:00:00Z\t/' "$d/runs.tsv" @@ -612,6 +679,10 @@ self_test() { d="$tmp/schedpick"; fixture "$d"; sed -i "s/$ST_C/$ST_X/" "$d/picks.tsv" row a_schedule_lane_ignores_the_picks 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" st_picks() { printf '%s' '{"data":{"repository":{"nightly":{"nodes":[{"name":"2026-10-03","target":{"oid":"c3"}},{"name":"probe","target":{"oid":"p"}}]}}}}' > "$tmp/gqlp.json"; picks_of "$tmp/gqlp.json"; } + st_pr() { printf '%s' "$2" > "$tmp/gqlr.json"; picks_read "$tmp/gqlr.json" "$tmp/pr.tsv" "$tmp/pr.fail"; printf 'rows=%s fail=%s\n' "$(wc -l < "$tmp/pr.tsv")" "$(cat "$tmp/pr.fail")"; } + row an_absent_pick_is_a_good_read 0 "rows=0 fail=" "did not parse" -- st_pr x '{"data":{"repository":{"nightly":{"nodes":[]}}}}' + row a_pick_read_with_no_refs_field_failed 0 "fail=the nightly pick refs did not parse" "rows=1" -- st_pr x '{"data":{"repository":{"nightly":null}}}' + row picks_read_keeps_the_dated_rows 0 "rows=1 fail=" "did not parse" -- st_pr x '{"data":{"repository":{"nightly":{"nodes":[{"name":"2026-10-03","target":{"oid":"c3"}}]}}}}' row picks_of_reads_dated_nights_only 0 "$(printf '2026-10-03\tc3')" "probe" -- st_picks d="$tmp/fromchain"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv" st_a_from_run_reads_the_picks() { (LANES="$(printf '%s\n' "$ST_LANES" | sed -e '/^v-b;/s/\^schedule\$/^workflow_run$/')"; NOW=2026-10-04T06:00:00Z; OUTDIR="$tmp/ofc"; INBOXF=""; run_train "$d"); } @@ -673,6 +744,17 @@ self_test() { normalize "$tmp/cut.json" "$tmp/wf.json" } row normalize_marks_a_cut_job_list 0 "$(printf '#truncated\tCOMPLETED\tTRUNCATED')" "" -- st_cut + st_hist() { + local ctx='{"pageInfo":{"hasNextPage":false},"nodes":[{"name":"gate","status":"COMPLETED","conclusion":"SUCCESS","startedAt":"s","completedAt":"e","checkSuite":{"status":"COMPLETED","conclusion":"SUCCESS","branch":{"name":"main"},"workflowRun":{"databaseId":RID,"event":"push","createdAt":"c","workflow":{"id":"WC"}}}}]}' + printf '%s' '{"data":{"repository":{"defaultBranchRef":{"name":"main","target":{"oid":"'"$ST_C"'","tree":{"oid":"t"},"statusCheckRollup":{"contexts":'"${ctx/RID/301}"'},"history":{"nodes":[{"oid":"'"$ST_C"'","statusCheckRollup":{"contexts":'"${ctx/RID/301}"'}},{"oid":"'"$ST_X"'","statusCheckRollup":{"contexts":'"${ctx/RID/302}"'}}]}}}}}}' > "$tmp/hist.json" + normalize "$tmp/hist.json" "$tmp/wf.json" | cut -f1,3,6 | sort | uniq -c + } + # #4798: a push run on an older main commit is read from the history rollup, and the head is not read twice + row normalize_reads_main_history_rollups 0 "$(printf '1 rollup\t302\t%s' "$ST_X")" "$(printf '2 rollup\t301')" -- st_hist + st_rank() { rank_rows "$tmp/hist.json"; } + row rank_rows_are_the_head_then_the_history_once 0 "$(printf '0\t%s\n1\t%s' "$ST_C" "$ST_X")" "$(printf '2\t')" -- st_rank + d="$tmp/rankhead"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_Y" "$ST_X" + row a_rank_list_for_another_head_is_ignored 0 "NOT RELEASABLE: v-a, not_measured (+2 more)" "RELEASABLE H=" -- st_decide "$d" d="$tmp/noprod"; fixture "$d" st_a_lane_without_a_producer_is_not_measured() { decide "$ST_LANES v-e;verdict;check E;-;-;-" "$d" 2026-10-04T06:00:00Z; cat "$d/line" "$d/lanes.tsv"; } @@ -700,6 +782,47 @@ v-e;verdict;check E;-;-;-" "$d" 2026-10-04T06:00:00Z; cat "$d/line" "$d/lanes.ts d="$tmp/legs"; fixture "$d" printf 'wf\t.github/workflows/a.yml\t101\tschedule\tmain\t%s\t2026-10-04T01:00:00Z\tCOMPLETED\tSUCCESS\tjob-a\tCOMPLETED\tFAILURE\t2026-10-04T01:00:30Z\t2026-10-04T01:20:00Z\t2\n' "$ST_C" >> "$d/runs.tsv" row a_failed_parallel_leg_of_the_same_name_is_red 0 "NOT RELEASABLE: v-a, 101" "RELEASABLE H=" -- st_decide "$d" + # J (#4798): main moved after the producers ran, so the head has no run; every verdict producer ran on one commit. + d="$tmp/moved"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_X" + row a_moved_head_judges_the_commit_every_producer_ran_on 0 "RELEASABLE H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits)" "NOT RELEASABLE" -- st_decide "$d" + # one commit for every lane, never a per-lane mix: v-a ran on X only, the rest on Y only + d="$tmp/split"; fixture "$d"; sed -i "s/$ST_C/$ST_Y/g; /\t101\t/s/$ST_Y/$ST_X/" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_X" "$ST_Y" + row producers_split_across_commits_are_not_measured 0 "NOT RELEASABLE: v-a, not_measured (+2 more)" "RELEASABLE H=" -- st_decide "$d" + # a newer commit that only some producers ran on does not win; the lag counts it and the head + d="$tmp/partial"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_Y" "$ST_X" + printf 'wf\t.github/workflows/a.yml\t102\tschedule\tmain\t%s\t2026-10-04T05:00:00Z\tCOMPLETED\tSUCCESS\tjob-a\tCOMPLETED\tSUCCESS\t2026-10-04T05:01:00Z\t2026-10-04T05:30:00Z\t1\n' "$ST_Y" >> "$d/runs.tsv" + row a_newer_partial_commit_does_not_win 0 "RELEASABLE H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 2 commits)" "NOT RELEASABLE" -- st_decide "$d" + # a commit with no run at all still counts in the lag: the lag is the history position, not the commits seen in runs + d="$tmp/gap"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_Y" "$ST_X" + row a_commit_without_runs_counts_in_the_lag 0 "(judged bbbbbbbbbb, head aaaaaaaaaa, lag 2 commits)" "NOT RELEASABLE" -- st_decide "$d" + # newest = nearest the head in history, not the latest run time: the older commit's runs started later + d="$tmp/order"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_X" "$ST_Y" + awk -F '\t' -v OFS='\t' -v y="$ST_Y" '{ $3 = $3 + 1000; $6 = y; $7 = "2026-10-05T01:00:00Z"; print }' "$d/runs.tsv" >> "$d/runs.tsv.y" && cat "$d/runs.tsv.y" >> "$d/runs.tsv" + row the_nearest_commit_wins_not_the_latest_run 0 "H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits)" "NOT RELEASABLE" -- st_decide "$d" + # a red on J is a red, named with the head and the lag + d="$tmp/movedred"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g; /\t201\t/s/\tx\tCOMPLETED\tSUCCESS\t/\tx\tCOMPLETED\tFAILURE\t/" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_X" + row a_red_on_the_judged_commit_is_red 0 "NOT RELEASABLE: v-b, 201 (judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits)" "RELEASABLE H=" -- st_decide "$d" + # the lookback is capped at MAX_LAG commits behind the head: rank 6 is judged, rank 7 is not_measured, not a stale verdict + d="$tmp/edge"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_X" + row a_judged_commit_at_the_lag_cap_is_judged 0 "RELEASABLE H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 6 commits)" "NOT RELEASABLE" -- st_decide "$d" + d="$tmp/stale"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; ranks "$d" "$ST_C" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_Y" "$ST_X" + row a_judged_commit_past_the_lag_cap_is_not_measured 0 "NOT RELEASABLE: v-a, not_measured (+2 more)" "RELEASABLE H=" -- st_decide "$d" + # no history read (an old raw, a failed history): only the head is a candidate, as before + d="$tmp/norank"; fixture "$d"; sed -i "s/$ST_C/$ST_X/g" "$d/runs.tsv"; : > "$d/rank.tsv" + row no_history_judges_only_the_head 0 "NOT RELEASABLE: v-a, not_measured (+2 more)" "RELEASABLE H=" -- st_decide "$d" + st_moved_bundle() { + local m="$tmp/mb/raw" t='\t' + mkdir -p "$m"; printf '0\t%s\n1\t%s\n' "$ST_C" "$ST_X" > "$m/rank.tsv"; printf '%s\n' "$ST_C" > "$m/C"; printf 'ok\n' > "$m/read"; printf 'cccc\n' > "$m/tree"; printf '2026-10-03\t%s\n' "$ST_X" > "$m/picks.tsv"; printf '501\t1\n502\t1\n503\t1\n' > "$m/attempts.tsv" + { + printf "rollup${t}.github/workflows/ci.yml${t}501${t}push${t}main${t}%s${t}2026-10-03T23:00:00Z${t}COMPLETED${t}SUCCESS${t}ci / gate${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" + printf "rollup${t}.github/workflows/ci.yml${t}501${t}push${t}main${t}%s${t}2026-10-03T23:00:00Z${t}COMPLETED${t}SUCCESS${t}workspace-test${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" + printf "wf${t}.github/workflows/examples-nightly.yml${t}502${t}workflow_run${t}main${t}%s${t}2026-10-04T01:00:00Z${t}COMPLETED${t}SUCCESS${t}examples${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" + printf "wf${t}.github/workflows/nightly.yml${t}503${t}workflow_run${t}main${t}%s${t}2026-10-04T01:10:00Z${t}COMPLETED${t}SUCCESS${t}x86_64-unknown-linux-gnu on box${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" + } > "$m/runs.tsv" + bash "$SCRIPT_PATH" --out "$tmp/mbout" --from "$m" --now 2026-10-04T06:00:00Z > /dev/null 2>&1 + cat "$tmp/mbout/2026-10-04/line"; grep -E '^(ci-main|deep-examples|deep-bins-build) |^# (head|C|tree) ' "$tmp/mbout/2026-10-04/bundle.tsv" + } + row a_moved_head_bundle_names_judged_and_head 0 "(judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits) [C=bbbbbbbbbb pin=" "$(printf '# tree\tcccc')" -- st_moved_bundle d="$tmp/infoatt"; fixture "$d" row an_info_lane_attempt_is_marked_inferred 0 "$(printf 'i-d\tinfo\tgreen\t.github/workflows/d.yml \t401\t%s\tsuccess\t1?' "$ST_C")" "" -- st_decide "$d" st_rcut() { @@ -788,7 +911,7 @@ v-e;verdict;check E;-;-;-" "$d" 2026-10-04T06:00:00Z; cat "$d/line" "$d/lanes.ts } # the planted mutants: name, TAB, a sed expression applied to this file NT_MUTANTS='m01_not_measured_counts_as_green s/nv++; if (\$3 == "green") next/nv++; if ($3 != "red") next/ -m02_run_on_any_commit s/if (RH\[r\] != C) {/if (0) {/ +m02_run_on_any_commit s/if (RH\[r\] != J) {/if (0) {/ m03_in_progress_run_counts s/if (RST\[r\] != "COMPLETED") {/if (0) {/ m04_retried_green_is_green s/if (att > 1) out/if (att > 9) out/ m05_failed_read_is_read s/readok = (RD == "ok" \&\& /readok = (/ @@ -811,7 +934,7 @@ m21_failure_row_after_history s/\[ -z "\$PRINTED\$HISTDONE" \]/[ -z "$PRINTED" ] m22_self_test_inbox_inherited s/^ export INBOX="\$tmp\/inbox.md" # whatever/ : # whatever/ m23_drill_writes_named_inbox s/then INBOXF="\$(mktemp)" || exit 3; DRILL=1;/then DRILL="";/ m24_one_leg_hides_another /fb\[JN\[r, k\]\] = 1; bad = bad/d -m25_cut_rollup_read s/if (\$c.statusCheckRollup.contexts.pageInfo.hasNextPage \/\/ false) then/if false then/ +m25_cut_rollup_read s/if (\$x.statusCheckRollup.contexts.pageInfo.hasNextPage \/\/ false) then/if false then/ m26_day_set_late /^ DAY=.*# before any step/d;s/^ step outdir$/ DAY="${NOW%%T*}"; step outdir/ m27_pinned_not_required s/if \[ -n "\$PINNED" \] || \[ -f "\$HERE\/PIN" \]; then/if [ -f "$HERE\/PIN" ]; then/ m28_unhashed_pin_accepted s/|| { PIN="unhashed"; exit 1; }/|| :/ @@ -830,17 +953,30 @@ m40_config_token_ignored s/^ if grep -qsE / if false \&\& grep -qs m41_home_cargo_unchecked s/ "\$HOME\/.cargo"; do$/; do/ m42_floor_ignores_the_limit s/fl=\$((lim \/ 5))/fl=$RATE_FLOOR/ m43_any_event_counts s/ && REV\[r\] ~ EV\[i\]// -m44_chain_ignores_the_pick s/(pc = nightc(RC\[r\])) != C)/0)/ +m44_chain_ignores_the_pick s/(pc = nightc(RC\[r\])) != J)/0)/ m45_any_night_counts s/if (t >= PS\[k\] && t < PE\[k\]) return k/if (PO[k] == C) return k/ m46_window_end_inclusive s/t < PE\[k\]/t <= PE[k]/ m47_window_start_exclusive s/t >= PS\[k\]/t > PS[k]/ -m48_every_lane_checks_the_pick s/EV\[i\] == "^workflow_run\$" && (pc/(pc/ -m49_from_skips_the_picks s/ tree picks.tsv; do/ tree; do/ +m48_every_lane_checks_the_pick s/if (EV\[i\] != "\^workflow_run\$") {/if (0) {/ +m49_from_skips_the_picks s/ tree picks.tsv rank.tsv pickfail; do/ tree rank.tsv pickfail; do/ m50_picks_any_ref_name s/ | select(.name | test("^\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}\$"))// m51_query_drops_the_picks s/ nightly: refs(refPrefix/ unread: refs(refPrefix/ -m52_no_straddle_check s/ && (pc = prevc(RC\[r\])) != C)/ \&\& 0)/ +m52_no_straddle_check s/ && (pc = prevc(RC\[r\])) != J)/ \&\& 0)/ m53_straddle_window_ends_early s/substr(RC\[r\], 12, 2) < "18"/substr(RC\[r\], 12, 2) < "13"/ -m54_prev_is_the_same_night s/if (PE\[j\] == PS\[k\]) return PO\[j\]/if (j == k) return PO[j]/' +m54_prev_is_the_same_night s/if (PE\[j\] == PS\[k\]) return PO\[j\]/if (j == k) return PO[j]/ +m55_judge_only_the_head s/if (all) { J = h; LAG = k; break }/if (all \&\& k == 0) { J = h; LAG = k; break }/ +m56_lag_cap_ignored s/k <= MAXLAG \&\& k < NRK/k < NRK/ +m57_lag_not_printed s/note = (C != HEAD \&\& /note = (0 \&\& / +m58_a_partial_commit_counts s/!((h, i) in HAS)) all = 0/0) all = 0/ +m59_bundle_keeps_the_head s/\[\[ "\${j:-}" =~ \^\[0-9a-f\]{40}\$ \]\] \&\& CSHA="\$j"/:/ +m60_history_not_read s/\[\$c\] + \[(\$c.history.nodes \/\/ \[\])\[\] | select(.oid != \$c.oid)\]/[$c]/ +m61_head_read_twice s/ | select(.oid != \$c.oid)\]/]/ +m62_cap_off_by_one s/k <= MAXLAG/k < MAXLAG/ +m63_rank_head_unchecked s/readok \&\& RKO\[0\] == C/readok/ +m64_oldest_commit_wins s/if (all) { J = h; LAG = k; break }/if (all) { J = h; LAG = k }/ +m65_rank_history_dropped s/\[(\$c.history.nodes \/\/ \[\])\[\] | .oid | select(. != \$c.oid)\]/[]/ +m66_failed_pick_read_falls_back s/\[ -s "\$2\/pickfail" \] \&\& \[ "\${rd:-}" = ok \] \&\& rd=/false \&\& rd=/ +m67_failed_pick_read_is_an_absent_pick s/nodes | type) == "array"/nodes | type) != "zzz"/' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 91e43bb7c1a641adcfdf0166a88167f7b7b33b38 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 00:27:55 +0200 Subject: [PATCH 22/34] chore(quorum): PMAT-4798 receipt, 3 PASS non-author lanes (gemini, haiku, gpt-oss), paths scrubbed Agent: aprender-w4798 Co-Authored-By: Claude Sonnet 5.5 --- docs/audits/quorum-PMAT-4798.json | 283 ++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 docs/audits/quorum-PMAT-4798.json diff --git a/docs/audits/quorum-PMAT-4798.json b/docs/audits/quorum-PMAT-4798.json new file mode 100644 index 0000000000..26b22e19b7 --- /dev/null +++ b/docs/audits/quorum-PMAT-4798.json @@ -0,0 +1,283 @@ +{ + "ticket": "PMAT-4798", + "base": "067e2e0646", + "base_resolved": "067e2e0646", + "base_note": "no origin/067e2e0646 exists; judged against the local ref", + "head": "bd003f5592c61c510c633770615ba8e530cc73ee", + "diff_sha256": "47d2e9bf5150d31a95ca25468005a3645e7eafe0aafc0ce5bd4e6a1ca218b9e0", + "width": 3, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 62508, + "prompt_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "author": { + "model": "claude-sonnet-5-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff completely and correctly implements the requirements of PMAT-4798. It properly queries the most recent 7 commits from main, establishes `J` as the newest commit (up to `MAX_LAG=6` commits behind the head) on which every verdict lane has a measured run, and defaults back to the head if no such commit exists. The output format `(judged , head , lag k commits)` is accurately appended to the verdict when `J` differs from the head. The tests, mutants, and invariants rigorously verify the edge cases such as partial commits, failed GraphQL reads, chained run pick resolutions, and lag limits. The release path behavior correctly matches the contract, and no constraints are weakened.", + "findings": [], + "raw_bytes": 2698, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "role": "independent", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "executor": "agy", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "1d52976fdfc391af107971499f018f7eda64b3691f937db9c6bbbffa0cb24d8b", + "output_sha256": "5c4e9e3cc7572c4202ede4e934e0332f7f6dec1fc419716715c45f6091690ae5", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 2, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements PMAT-4798: J search selects the newest commit (at most 6 behind head) on which every verdict lane has a measured run. Chained runs counted by pick, plain runs by head_sha. Two-unlike-failures split correctly handles absent pick (good read, no pickfail) vs. failed pick read (pickfail written, all lanes not_measured). Lag printed when J ≠ head. GraphQL queries 7 commits via history. Self-test comprehensive (87 rows planned per receipt). Mutants m55–m67 target J logic; prior mutants updated to reference J. No refutations found.", + "findings": [], + "raw_bytes": 2068, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "executor": "claude-code", + "grounding_check": "parity", + "role": "counted", + "model": "claude-haiku-4-5", + "model_measured": "claude-haiku-4-5", + "model_source": "flag", + "family": "claude", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "2df2e42cc426663958e4f76331cef19495720fd8c5e2b8c560ea2c3c31014cc2", + "output_sha256": "2c569d4903394f5f534b78eba9e083c4d1f6d493aa21cd756bf856315c389422", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "claude-haiku-4-5", + "exhausted": false, + "attempts": [ + { + "model": "claude-haiku-4-5", + "family": "claude", + "outcome": "answered" + } + ], + "skipped": [] + } + }, + { + "lane": 3, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "All changes align with ticket PMAT-4798. No refutations found.", + "findings": [], + "raw_bytes": 1389, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "role": "independent", + "model": "gpt-oss-120b-medium", + "model_measured": "gpt-oss-120b-medium", + "model_source": "measured", + "family": "openai", + "executor": "agy", + "brief_sha256": "3ce8b690042d2d7e5c1349013cff2065062a3c2cca0e199f29247597282535e5", + "trace": { + "input_sha256": "b3c73fc67fa2e6ca379f4f80fa48707cabe10c3cbb49bc9731edd33680d1ab4a", + "output_sha256": "dff13e691b5a04980af31525d3b7ef4c5e2ebb17b1705b624a0206c002166d2e", + "store": null, + "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" + }, + "fallback": { + "judged_by": "gpt-oss-120b-medium", + "exhausted": false, + "attempts": [ + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "capacity", + "reason": "failed to send message: send failed; already reported to the user: Eligibility check failed: failed to get load code assist response: UNAVAILABLE (code 503): The service is currently unavailable." + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [ + "lane 3: judged by gpt-oss-120b-medium after falling through gpt-oss-120b-medium (capacity: failed to send message: send failed; already reported to the user: Eligibility check failed: failed to get load code assist response: UNAVAILABLE (code 503): The service is currently unavailable.) (PMAT-321)" + ], + "fallback": { + "same_family_width": 2, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "claude-haiku-4-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "configured" + }, + { + "model": "claude-sonnet-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "claude-opus-5-5", + "family": "claude", + "disposition": "claude-code", + "why": "run by Claude Code on its own budget: a configured seat runs every round, a fallback step only when every non-Claude family is measured out (PMAT-360, operator standing rule)" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + }, + { + "family": "openai", + "model": "gpt-oss-120b-medium", + "probe": 1, + "outcome": "live" + } + ], + "degraded": null, + "prah": { + "source": "none", + "path": null, + "why": "no --prah, none on PATH, and no install receipt at /file" + }, + "bucket": { + "ledger": "/state", + "window_s": 18000, + "pace": "off", + "buckets": { + "claude-gpt": { + "bucket": "claude-gpt", + "state": "unmeasured", + "reason": "agy-bucket.sh could not read the ledger; the bucket is not consulted this round" + }, + "gemini": { + "bucket": "gemini", + "state": "unmeasured", + "reason": "agy-bucket.sh could not read the ledger; the bucket is not consulted this round" + } + }, + "closed": [] + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "cheap_seat": "claude-haiku-4-5", + "advisory_lane": { + "state": "unavailable", + "counts": false, + "row": { + "NotRun": "Busy" + }, + "verdict": "unavailable", + "why": "no host answered inside the budget", + "served_by": null, + "gpu_proof": null, + "apr": null, + "apr_binary_sha256": null, + "apr_tag": null, + "apr_commit": null, + "apr_build_why": null, + "model": null, + "model_sha256": null, + "rc": 2, + "wall_s": 1, + "collected_s": 1242, + "budget_s": 196, + "budget_basis": "ledger: 2 x p95 98 s over 126 -cuda Verdict rows", + "brief": { + "bytes": 62508, + "sent_bytes": 62677, + "max_bytes": 196608 + }, + "trace": { + "input_sha256": "ef0432863a345be5255db200fd42f06dfd530c6e4f8ad5e8da44155f86645043", + "raw": "advisory.json" + }, + "attempts": [ + { + "host": "", + "ok": false, + "wall_s": 1, + "error": "busy: executor yielded (unit inactive; last yield: 2026-10-03T00:47:22Z host=-a5b5 event=stopped gpu-priority lock=/tmp/apr-gpu.priority)", + "ts": "2026-10-05T22:07:00.887Z", + "wall_ms": 363, + "request_id": "01a10e1b-1f15-75fe-866f-2a0d4a995c20", + "load1": 0.15, + "cold": null + } + ], + "ledger": "/state", + "raw": "advisory.json", + "agrees_with_counted": null, + "counted": "PASS" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=3 author=claude-sonnet-5-5/claude same_family=1/2" + } +} From c1215d2cb1ba25f1a7788272d042fcefd7cbd3f3 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 00:42:19 +0200 Subject: [PATCH 23/34] test(nightly_train): two rows and two mutants for the J search's chained-run rules (#4798) Agent: aprender-w4798 Co-Authored-By: Claude Sonnet 5.5 --- docs/audits/impl-PMAT-4798-receipt.md | 2 +- scripts/release/nightly_train.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/audits/impl-PMAT-4798-receipt.md b/docs/audits/impl-PMAT-4798-receipt.md index 8d0b99a585..77bd1dd5c3 100644 --- a/docs/audits/impl-PMAT-4798-receipt.md +++ b/docs/audits/impl-PMAT-4798-receipt.md @@ -32,7 +32,7 @@ not the head. `m67_failed_pick_read_is_an_absent_pick` (the array check is dropped, so a failed read looks like an absent pick). ## What is measured -- `--self-test`: 87/87 rows. `--mutants`: 67/67 killed (m43-m67; m55-m65 are the J search, m66 and m67 the pick split). +- `--self-test`: 89/89 rows. `--mutants`: 69/69 killed (m43-m69; m55-m65 and m68-m69 the J search, m66 and m67 the pick split). - Merge-queue history on main is not strictly first-parent; those commits carry no main runs and only shorten the lookback. Conservative. diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index 81ac5cc5bf..1704acdee5 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -676,6 +676,12 @@ self_test() { row a_run_after_noon_needs_the_night_before_too 0 "201 may have measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_chainl "$d" sed -i 's/\t2026-10-04T17:59:00Z\t/\t2026-10-04T18:00:00Z\t/' "$d/runs.tsv" row a_run_from_six_pm_needs_its_own_night_only 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_chain "$d" + # dupx DIR: every plain run also exists on X (ids +1000), so X has a measured run on each plain lane (#4798 J search) + dupx() { awk -F '\t' -v OFS='\t' -v X="$ST_X" '{ print } $3 != 201 { $3 = $3 + 1000; $6 = X; print }' "$1/runs.tsv" > "$1/runs.new" && mv "$1/runs.new" "$1/runs.tsv"; printf '1101\t1\n1201\t1\n1301\t1\n1401\t1\n' >> "$1/attempts.tsv"; ranks "$1" "$ST_C" "$ST_X"; } + d="$tmp/jchain"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; dupx "$d"; printf '2026-10-03\t%s\n' "$ST_X" > "$d/picks.tsv" + row the_search_credits_a_chained_run_to_its_pick_not_the_head 0 "RELEASABLE H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits)" "NOT RELEASABLE" -- st_chain "$d" + d="$tmp/jstraddle"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T13:00:00Z\t/' "$d/runs.tsv"; dupx "$d"; printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_C" "$ST_X" > "$d/picks.tsv" + row the_search_skips_a_commit_whose_chained_run_straddles_two_picks 0 "NOT RELEASABLE: v-b" "(judged" -- st_chain "$d" d="$tmp/schedpick"; fixture "$d"; sed -i "s/$ST_C/$ST_X/" "$d/picks.tsv" row a_schedule_lane_ignores_the_picks 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" st_picks() { printf '%s' '{"data":{"repository":{"nightly":{"nodes":[{"name":"2026-10-03","target":{"oid":"c3"}},{"name":"probe","target":{"oid":"p"}}]}}}}' > "$tmp/gqlp.json"; picks_of "$tmp/gqlp.json"; } @@ -976,7 +982,9 @@ m63_rank_head_unchecked s/readok \&\& RKO\[0\] == C/readok/ m64_oldest_commit_wins s/if (all) { J = h; LAG = k; break }/if (all) { J = h; LAG = k }/ m65_rank_history_dropped s/\[(\$c.history.nodes \/\/ \[\])\[\] | .oid | select(. != \$c.oid)\]/[]/ m66_failed_pick_read_falls_back s/\[ -s "\$2\/pickfail" \] \&\& \[ "\${rd:-}" = ok \] \&\& rd=/false \&\& rd=/ -m67_failed_pick_read_is_an_absent_pick s/nodes | type) == "array"/nodes | type) != "zzz"/' +m67_failed_pick_read_is_an_absent_pick s/nodes | type) == "array"/nodes | type) != "zzz"/ +m68_search_files_a_chained_run_under_the_head s/hm = nightc(RC\[r\]);/hm = RH[r];/ +m69_search_ignores_the_straddle s/ \&\& prevc(RC\[r\]) != hm) hm = ""/ \&\& 0) hm = ""/' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 17a578377a37ad2399a585b78459a29d4c8500e9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 01:02:37 +0200 Subject: [PATCH 24/34] T44: the night table reads time in integer arithmetic, so the darwin runner can run it build-darwin runs nightly_c_checkout.sh on the macOS runner, whose date is BSD. `date -u -d` is GNU-only: there --at was "not a time" (rc 3) and the self-test went 12/34, so build-darwin failed every chained night and publish-darwin was skipped. The lib now does the conversion itself (np_date, np_days, np_epoch: days_from_civil and civil_from_days, round-tripped so a day the calendar lacks is refused). nightly_c_checkout.sh and `nightly_pick.sh --at` both use it. build-darwin runs both self-tests on the runner, on every event. Measured on the macOS runner, with the runner's own PATH: before: nightly_pick 35/38, nightly_c_checkout 12/34 (rc 1) after: nightly_pick 51/51, nightly_c_checkout 37/37 (rc 0) Linux: mutants 22/22 and 10/10 killed. check_producers_chain PASS. bashrs finding set unchanged or smaller. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 7 +++++ scripts/lib/nightly_pick.sh | 41 +++++++++++++++++++++++++-- scripts/release/nightly_c_checkout.sh | 11 +++++-- scripts/release/nightly_pick.sh | 31 +++++++++++++++++--- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9b7bd3fffa..852c5635de 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -269,6 +269,13 @@ jobs: NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + # the night table on this runner's own BSD userland: a GNU-only call there (date -d) fails the C step above + # every night and skips publish-darwin; the case tables name it on any event, not only on a chained night + - name: Night table self-test on macOS (BSD userland) + run: | + set -euo pipefail + bash scripts/release/nightly_pick.sh --self-test + bash scripts/release/nightly_c_checkout.sh --self-test - name: Build apr (host-native, pinned toolchain) run: | diff --git a/scripts/lib/nightly_pick.sh b/scripts/lib/nightly_pick.sh index a09dbe39e5..aa7f477b23 100644 --- a/scripts/lib/nightly_pick.sh +++ b/scripts/lib/nightly_pick.sh @@ -16,16 +16,53 @@ # Return codes: 0 ok; 1 refused (RED); 2 not_measured (could not read or write the remote); 3 caller error. # # np_night [EPOCH] print the night of EPOCH (default now) +# np_date DAYS print the UTC date DAYS after 1970-01-01 (no date -d: portable to BSD date) +# np_days YYYY-MM-DD print its days after 1970-01-01; 3 for a day the calendar lacks +# np_epoch YYYY-MM-DDTHH:MM:SSZ print its epoch; 3 for any other form. No date -d anywhere: the darwin runner has BSD date # np_pick REPO REMOTE NIGHT SHA create refs/heads/nightly/NIGHT -> SHA unless it exists; prints "C=" + PICKED|KEPT # np_resolve REPO REMOTE NIGHT print the night's C; 2 when the night has no pick # np_verify REPO REMOTE NIGHT SHA # 0 when SHA is the night's C, 1 when it is not, 2 when the night has no pick np_night() { # [EPOCH] - local e="${1:-}" + local e="${1:-}" s [ -n "$e" ] || e="$(date -u +%s)" || return 3 case "$e" in '' | *[!0-9]*) printf 'NP caller error: epoch %s is not a number\n' "$e" >&2; return 3 ;; esac - date -u -d "@$((e - 43200))" +%F || return 3 + s=$((e - 43200)) + np_date $((s >= 0 ? s / 86400 : -((86399 - s) / 86400))) +} + +np_date() { # DAYS since 1970-01-01 -> YYYY-MM-DD, in integer arithmetic: date -d @N is GNU-only and the darwin + # runner's BSD date refuses it (H. Hinnant's civil_from_days, proleptic Gregorian) + local z era doe yoe doy mp m + z=$(($1 + 719468)) + era=$(((z >= 0 ? z : z - 146096) / 146097)); doe=$((z - era * 146097)) + yoe=$(((doe - doe / 1460 + doe / 36524 - doe / 146096) / 365)) + doy=$((doe - (365 * yoe + yoe / 4 - yoe / 100))); mp=$(((5 * doy + 2) / 153)) + m=$((mp < 10 ? mp + 3 : mp - 9)) + printf '%04d-%02d-%02d\n' $((yoe + era * 400 + (m <= 2))) "$m" $((doy - (153 * mp + 2) / 5 + 1)) +} + +np_days() { # YYYY-MM-DD -> days since 1970-01-01 (H. Hinnant's days_from_civil); 3 for a date the calendar lacks + local y m d days back + [[ $1 =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})$ ]] || return 3 + # a leading 1 keeps "08" decimal: 108 - 100 + y=$((1${BASH_REMATCH[1]} - 10000)); m=$((1${BASH_REMATCH[2]} - 100)); d=$((1${BASH_REMATCH[3]} - 100)) + [ "$m" -gt 2 ] || y=$((y - 1)) + days=$(((y / 400) * 146097 + (y % 400) * 365 + (y % 400) / 4 - (y % 400) / 100 + (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + d - 1 - 719468)) + # back through np_date: 2026-02-30 comes back 2026-03-02, so it is refused + back="$(np_date "$days")" + [ "$back" = "$1" ] || return 3 + printf '%s\n' "$days" +} + +np_epoch() { # YYYY-MM-DDTHH:MM:SSZ (the form workflow_run.created_at takes) -> epoch; 3 for any other form + local ymd hh mm ss days + [[ $1 =~ ^([0-9]{4}-[0-9]{2}-[0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z$ ]] || return 3 + ymd="${BASH_REMATCH[1]}"; hh=$((1${BASH_REMATCH[2]} - 100)); mm=$((1${BASH_REMATCH[3]} - 100)); ss=$((1${BASH_REMATCH[4]} - 100)) + (( hh < 24 && mm < 60 && ss < 60 )) || return 3 + days="$(np_days "$ymd")" || return 3 + printf '%s\n' $((days * 86400 + hh * 3600 + mm * 60 + ss)) } np__args() { # NIGHT [SHA] diff --git a/scripts/release/nightly_c_checkout.sh b/scripts/release/nightly_c_checkout.sh index ac25d73f7b..5a500279c9 100755 --- a/scripts/release/nightly_c_checkout.sh +++ b/scripts/release/nightly_c_checkout.sh @@ -28,10 +28,11 @@ LIB="$HERE/../lib/nightly_pick.sh" caller_error() { printf 'nightly_c_checkout: caller error: %s\n' "$1" >&2; exit 3; } -to_epoch() { # ISO8601|EPOCH -> epoch +to_epoch() { # YYYY-MM-DDTHH:MM:SSZ|EPOCH -> epoch. The ISO form is the one workflow_run.created_at takes, read by the + # lib in integer arithmetic (date -d is GNU-only and the darwin runner's BSD date refuses it) case "$1" in '') return 3 ;; - *[!0-9]*) date -u -d "$1" +%s 2>/dev/null || return 3 ;; + *[!0-9]*) np_epoch "$1" || return 3 ;; *) printf '%s\n' "$1" ;; esac } @@ -155,6 +156,10 @@ self_test() { row missing_conclusion_is_a_caller_error 3 "--pick-conclusion" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "$at" row missing_at_is_a_caller_error 3 "--at" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" row bad_at_is_a_caller_error 3 "not a time" "" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at yesterday-ish --pick-conclusion success + # --at is read in integer arithmetic (BSD date has no -d): only created_at's form, and only a day the calendar has + row at_on_a_day_the_calendar_lacks_is_a_caller_error 3 "not a time" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-02-30T01:00:00Z --pick-conclusion success + row at_hour_24_is_a_caller_error 3 "not a time" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at 2026-10-04T24:00:00Z --pick-conclusion success + row at_without_its_zone_is_a_caller_error 3 "not a time" "NIGHTLY C=" -- bash "$SCRIPT_PATH" checkout --repo "$W" --at "2026-10-04 20:31:07" --pick-conclusion success row unknown_command_is_a_caller_error 3 "caller error" "" -- bash "$SCRIPT_PATH" measure rm -rf -- "${FX:?}" printf '%s %s/%s case rows green\n' "$([ "$FAILED" -eq 0 ] && echo SELF-TEST-GREEN || echo SELF-TEST-RED)" "$((CASES - FAILED))" "$CASES" @@ -171,7 +176,7 @@ m04_no_pick_is_ok s/ c="\$(np_resolve "\$repo" "\$remote" "\$night")" || retu m05_night_from_the_clock s/ night="\$(np_night "\$epoch")" || return 3/ night="$(np_night)" || return 3/ m06_fetches_main s/"+refs\/heads\/nightly\/\$night:refs\/remotes\/\$remote\/nightly\/\$night"/main/ m07_switch_failure_ignored s/|| { printf .NC RED: cannot switch/|| true || { printf '"'"'NC RED: cannot switch/ -m08_bad_at_accepted s/date -u -d "\$1" +%s 2>\/dev\/null || return 3/date -u +%s/ +m08_bad_at_accepted s/np_epoch "\$1" || return 3/date -u +%s/ m09_env_before_verify /^ np_verify "\$repo"/{h;d};/^ printf .NIGHTLY C=%s night=%s/{x;G} m10_pick_conclusion_ignored s/^ \[ "\$4" = success \] || {.*$/ true/' mutants() { diff --git a/scripts/release/nightly_pick.sh b/scripts/release/nightly_pick.sh index 118f9f6a8b..70a0abd08e 100755 --- a/scripts/release/nightly_pick.sh +++ b/scripts/release/nightly_pick.sh @@ -10,8 +10,8 @@ # one pick instead: producers start on the pick (workflow_run or dispatch) and verify the commit they were handed. # # Usage: -# nightly_pick.sh night [--at EPOCH] -# nightly_pick.sh pick [--repo DIR] [--remote NAME] [--night N | --at EPOCH] --sha SHA +# nightly_pick.sh night [--at EPOCH|YYYY-MM-DDTHH:MM:SSZ] +# nightly_pick.sh pick [--repo DIR] [--remote NAME] [--night N | --at EPOCH|YYYY-MM-DDTHH:MM:SSZ] --sha SHA # nightly_pick.sh resolve [--repo DIR] [--remote NAME] --night N # nightly_pick.sh verify [--repo DIR] [--remote NAME] --night N --sha SHA # nightly_pick.sh --self-test | --mutants @@ -59,6 +59,21 @@ self_test() { row night_mid_evening_is_today 0 "2026-10-04" "" -- bash "$SCRIPT_PATH" night --at 1791154800 # 2026-10-04T23:00Z row night_late_fire_is_still_the_night 0 "2026-10-04" "" -- bash "$SCRIPT_PATH" night --at 1791179280 # 2026-10-05T05:48Z row night_turns_at_noon 0 "2026-10-05" "" -- bash "$SCRIPT_PATH" night --at 1791201600 # 2026-10-05T12:00Z + # the night is integer arithmetic, not `date -d` (GNU-only): it must turn months, years and both leap rules + row night_turns_the_year 0 "2026-12-31" "" -- bash "$SCRIPT_PATH" night --at 1798801200 # 2027-01-01T11:00Z + row night_ends_a_short_february 0 "2026-02-28" "" -- bash "$SCRIPT_PATH" night --at 1772366399 # 2026-03-01T11:59:59Z + row night_of_a_leap_day 0 "2028-02-29" "" -- bash "$SCRIPT_PATH" night --at 1835503200 # 2028-03-01T06:00Z + row night_of_a_400_year_leap_day 0 "2000-02-29" "" -- bash "$SCRIPT_PATH" night --at 951890400 # 2000-03-01T06:00Z + row night_skips_a_century_leap_day 0 "2100-02-28" "" -- bash "$SCRIPT_PATH" night --at 4107564000 # 2100-03-01T06:00Z + row night_opens_a_century_year 0 "2100-03-01" "" -- bash "$SCRIPT_PATH" night --at 4107650400 # 2100-03-02T06:00Z + row night_before_the_epoch_day 0 "1969-12-31" "" -- bash "$SCRIPT_PATH" night --at 0 + # --at as workflow_run.created_at gives it, read by np_epoch: the same night as its epoch, and only real UTC times + row night_from_created_at 0 "2026-10-04" "" -- bash "$SCRIPT_PATH" night --at 2026-10-05T11:59:59Z + row night_from_created_at_turns_at_noon 0 "2026-10-05" "" -- bash "$SCRIPT_PATH" night --at 2026-10-05T12:00:00Z + row night_from_created_at_in_a_leap_year 0 "2028-02-29" "" -- bash "$SCRIPT_PATH" night --at 2028-03-01T06:00:00Z + row night_at_a_day_the_calendar_lacks_is_a_caller_error 3 "is not an epoch" "" -- bash "$SCRIPT_PATH" night --at 2026-02-30T01:00:00Z + row night_at_hour_24_is_a_caller_error 3 "is not an epoch" "" -- bash "$SCRIPT_PATH" night --at 2026-10-04T24:00:00Z + row night_at_without_its_zone_is_a_caller_error 3 "is not an epoch" "" -- bash "$SCRIPT_PATH" night --at "2026-10-04 23:00:00" row unpicked_night_is_not_measured 2 "has no pick" "" -- bash "$SCRIPT_PATH" resolve --repo "$W" --night "$n" row first_pick_publishes_the_ref 0 "C=$c1 PICKED night $n" "" -- bash "$SCRIPT_PATH" pick --repo "$W" --night "$n" --sha "$c1" row the_ref_is_on_the_remote 0 "$c1" "" -- git -C "$W" ls-remote "$FX/origin.git" "refs/heads/nightly/$n" @@ -142,7 +157,13 @@ m12_unknown_commit_is_red /cannot tell whether/s/return 2 ;;/return 1 ;;/ m13_ref_name_matched_by_tail s/NF && \$2 != r { found = 1 }/NF \&\& 0 { found = 1 }/ m14_read_widened_to_the_rolling_tag s/ls-remote --refs "\$2" "refs\/heads\/nightly\/\$3"/ls-remote --refs "\$2" "nightly"/ m15_night_checked_per_line s/\[\[ \$1 =~ \(.*\) \]\] ||/printf "%s\\n" "$1" | grep -qxE "\1" ||/ -m16_sha_checked_per_line s/\[\[ \$2 =~ \(.*\) \]\] ||/printf "%s\\n" "$2" | grep -qxE "\1" ||/' +m16_sha_checked_per_line s/\[\[ \$2 =~ \(.*\) \]\] ||/printf "%s\\n" "$2" | grep -qxE "\1" ||/ +m17_no_century_rule s/ + doe \/ 36524 - doe \/ 146096) \/ 365/) \/ 365/ +m18_jan_feb_keep_the_march_year s/ + (m <= 2))) "$m"/)) "$m"/ +m19_negative_rounds_toward_zero s/s >= 0 ? s \/ 86400 : -((86399 - s) \/ 86400)/s \/ 86400/ +m20_calendar_unchecked s/^ \[ "\$back" = "\$1" \] || return 3$/ true/ +m21_clock_unchecked s/^ (( hh < 24 .*$/ true/ +m22_no_leap_days s/ + (y % 400) \/ 4 - (y % 400) \/ 100//' mutants() { local tmp name expr killed=0 total=0 errors=0 out tmp="$(mktemp -d "${TMPDIR:-/tmp}/np-mu.XXXXXX")" || caller_error "no temp dir" @@ -186,11 +207,13 @@ while [ "$#" -gt 0 ]; do --repo) repo="${2:-}"; shift 2 || caller_error "--repo DIR" ;; --remote) remote="${2:-}"; shift 2 || caller_error "--remote NAME" ;; --night) night="${2:-}"; shift 2 || caller_error "--night YYYY-MM-DD" ;; - --at) at="${2:-}"; shift 2 || caller_error "--at EPOCH" ;; + --at) at="${2:-}"; shift 2 || caller_error "--at EPOCH|YYYY-MM-DDTHH:MM:SSZ" ;; --sha) sha="${2:-}"; shift 2 || caller_error "--sha SHA" ;; *) caller_error "unknown option '$1'" ;; esac done +# an ISO --at is read by the lib in integer arithmetic, so the night is the same on GNU and BSD date +case "$at" in *[!0-9]*) e="$(np_epoch "$at")" || caller_error "--at $at is not an epoch or YYYY-MM-DDTHH:MM:SSZ"; at="$e" ;; esac if [ "$cmd" = night ]; then np_night "$at"; exit $?; fi if [ -z "$night" ]; then [ "$cmd" = pick ] || caller_error "--night YYYY-MM-DD" From 5d27d8e546971fb527fd3ff3afe0ab9ec780000b Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 01:10:50 +0200 Subject: [PATCH 25/34] T44: cuda-nightly's host preflight runs C's own copy, after the C step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ARM64 CUDA job ran `ci_self_hosted_preflight.sh --cuda` between its checkout and the C step. So it ran main's head's copy of the preflight, not C's, on the night it was meant to judge C. check_producers_chain P3 allowed one "Preflight…" step in that gap, and cuda-nightly was the only producer that used it. The preflight now comes right after the C step, and P3 has no gap: the C step must be the very next step after the checkout. Row p3_preflight_between now expects P3 to fail. Mutant m05 puts the old skip back, and that row kills it. check_producers_chain: PASS on the tree, self-test 57 rows, mutants 41/41. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cuda-nightly.yml | 19 +++++++++++-------- scripts/release/check_producers_chain.sh | 15 +++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index 31bbeaacdd..73fb33d3d3 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -170,11 +170,20 @@ jobs: - uses: actions/checkout@v7 if: steps.decide.outputs.proceed == 'true' - # The first thing gx10 does once it has a tree. NOT before `decide` and - # NOT before the checkout, for the reason the comment below spells out: + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' && (steps.decide.outputs.proceed == 'true') + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + + # The first thing this job does once its tree is C (the step above). NOT before + # `decide`, and NOT before the checkout, for the reason the comment below spells out: # on a yield night there is no working tree, so a step that `bash`es a # repo path exits 127 and turns a deliberately-yielded night RED. It # carries the same `if:` as the checkout it follows. + # It comes AFTER the C step, not between the checkout and it: there it + # ran main's head's copy of the preflight, so it did not judge C's own. # # gx10 is the box where run 34448908554 died on `gh: command not found` # after a 21 MB CUDA build. Everything below this line assumes cargo, @@ -182,12 +191,6 @@ jobs: - name: Preflight — this box has the tools this job assumes if: steps.decide.outputs.proceed == 'true' run: bash scripts/ci_self_hosted_preflight.sh --cuda - - name: Measure the night's C, not main's head (T44) - if: github.event_name == 'workflow_run' && (steps.decide.outputs.proceed == 'true') - env: - NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} - NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} - run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" # PP-26's release-phase marker gate (scripts/check_perf041_marker.sh, a # dogfood gate) may not be named in a REQUIRED workflow diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index e6767dbd57..4def915cef 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -9,8 +9,8 @@ # P1 still has a `schedule:` trigger (a live line in its on: block); # P2 lacks the chain: workflow_run with workflows: [], types: [completed], # branches: [main]; -# P3 checks this repo out (actions/checkout, unless `repository:` names another repo literally) and the next step, after an optional -# "Preflight…" step, is not the C step: run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION"`, +# P3 checks this repo out (actions/checkout, unless `repository:` names another repo literally) and the very next step +# is not the C step (a host preflight goes after it, so it runs C's own copy): run `bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION"`, # env NIGHTLY_PICK_AT from github.event.workflow_run.created_at (the pick run's own time, which a rerun of either # run keeps; run_started_at moves on a rerun) and NIGHTLY_PICK_CONCLUSION from github.event.workflow_run.conclusion # (a failed pick names no night to measure), and if exactly @@ -85,15 +85,14 @@ judge_file() { for (i = 1; i <= m; i++) { s = a[i]; gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); if (s == want) return 1 } return 0 } - function flush( k, j, cif, want, nm) { + function flush( k, j, cif, want) { if (!listed) { n = 0; ii = -1; return } # only a listed producer owes the C step for (k = 1; k <= n; k++) { if (unq(key(S[k], "uses")) ~ /^\.\//) { fail("P3 job step " k " uses a local action -- a checkout inside it cannot be judged here"); continue } if (unq(key(S[k], "uses")) !~ /^actions\/checkout@/ || other(S[k])) continue checkouts++ - j = k + 1; nm = key(S[j], "name") - if (j <= n && nm ~ /^Preflight/) j++ + j = k + 1 cif = unq(key(S[k], "if")) if (cif ~ /\$\{\{|^[>|]/) { fail("P3 checkout in job step " k " has an if: the C step cannot copy (${{ }} or a block): " cif); continue } @@ -269,7 +268,7 @@ m_cstep_badconcl() { changed book.yml '0,/workflow_run.conclusion }}/s//workflow m_cstep_noconcarg() { changed install-script.yml '0,/ --pick-conclusion "\$NIGHTLY_PICK_CONCLUSION"/s///'; } m_step_between() { changed examples-nightly.yml '0,/- name: Measure the night.s C/s//- run: make bench\n - name: Measure the night'"'"'s C/'; } m_repo_exempt() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n'; cstep; printf ' - uses: actions/checkout@v7\n with:\n repository: paiml/other\n')"; } -m_preflight_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - name: Preflight — tools\n run: true\n'; cstep)"; } +m_preflight_between() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n - name: Preflight — tools\n run: true\n'; cstep)"; } m_flow_if_ok() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep 'a || b')"; } m_flow_if_bare() { synth nightly.yml <<< "$(printf ' - uses: actions/checkout@v7\n if: a || b\n'; cstep | sed -e "s/'workflow_run'\$/'workflow_run' \&\& a || b/")"; } m_last_step() { synth nightly.yml <<< ' - uses: actions/checkout@v7'; } @@ -329,7 +328,7 @@ self_test() { row p3_step_between 1 'examples-nightly.yml: P3' -- m_step_between row p3_last_step_checkout 1 'end of steps' -- m_last_step row p3_other_repo_exempt 0 'PASS' -- m_repo_exempt - row p3_preflight_between 0 'nightly.yml: chained' -- m_preflight_ok + row p3_preflight_between 1 'nightly.yml: P3' -- m_preflight_between row p3_checkout_if_with_or 0 'nightly.yml: chained' -- m_flow_if_ok row p3_unparenthesised_or 1 'nightly.yml: P3' -- m_flow_if_bare row p4_github_sha 1 'qwen-story-daily.yml: P4' -- m_sha_plain @@ -371,7 +370,7 @@ MUTANTS='m01_no_schedule_rule s/if (sched) fail/if (0) fail/ m02_any_branch s/list1(L) == "main") cb = 1/1) cb = 1/ m03_any_type s/list1(L) == "completed") ct = 1/1) ct = 1/ m04_any_workflow s/list1(L) == pick) cw = 1/1) cw = 1/ -m05_no_preflight_skip s/nm ~ \/^Preflight\/) j++/0) j++/ +m05_preflight_skipped s/^ j = k + 1$/&; if (j <= n && key(S[j], "name") ~ \/^Preflight\/) j++/ m06_no_repo_exempt s/ || other(S\[k\])) continue/) continue/ m07_any_if s/if (unq(key(S\[j\], "if")) != want)/if (0)/ m08_any_at_source s/if (!hasline(S\[j\], "NIGHTLY_PICK_AT/if (0 \&\& !hasline(S[j], "NIGHTLY_PICK_AT/ From 270e91e4722acd87eaa1fc458cbac2cb4ef7c1e0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 01:41:35 +0200 Subject: [PATCH 26/34] T44: the pick's night is its run's created_at; the chain guard fails closed Three review findings on check_producers_chain and the pick: C1 (false green). The pick took its night from the clock (`night`, no --at). Producers take it from the pick run's created_at, which a rerun keeps. If night N's pick was rerun after 12:00Z on N+1, it picked nightly/N+1 at main's head, while every chained producer built N's pick. From 18:00Z on the train credited that build to N+1's pick. The pick now reads its own run's created_at (`gh api .../runs/$GITHUB_RUN_ID`, actions: read) and passes it as --at, which takes the ISO form through the lib. New rule P7 checks that both lines are live and exact and that no other night call exists. Five rows. Mutants m44-m47. The old nightly-pick.yml goes RED on P7. C2. An unreadable producer gave no FAIL line and PASS, because awk's exit status was dropped. A non-zero judge_file exit is now a FAIL for that file. Row p5_unreadable_producer, mutant m43. C3. P1 missed a quoted or spaced `"schedule":` key, which GitHub runs. Two rows, mutants m41 and m42. check_producers_chain: PASS on the tree, self-test 65 rows, mutants 48/48. bashrs finding set unchanged. pv validate nightly-pick-v1: 0 errors. The aprender-contracts cargo tests were not run here (contract text only); CI shows them. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly-pick.yml | 13 ++++-- contracts/nightly-pick-v1.yaml | 2 +- scripts/release/check_producers_chain.sh | 53 +++++++++++++++++++++--- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-pick.yml b/.github/workflows/nightly-pick.yml index a75f0c94e0..311213c8fa 100644 --- a/.github/workflows/nightly-pick.yml +++ b/.github/workflows/nightly-pick.yml @@ -10,8 +10,9 @@ # night) and runs `scripts/release/nightly_pick.sh verify --night N --sha `. # # The schedule below is a fallback trigger only. A dispatch is the reliable one, and a re-pick is -# idempotent: it prints KEPT and leaves the ref as it is. The night is the UTC date of (now - 12h), -# so a pick that fires late still lands on its own night. +# idempotent: it prints KEPT and leaves the ref as it is. The night is the UTC date of (created_at - 12h) +# of this run, so a pick that fires late still lands on its own night, and a rerun the next evening +# lands on the night its producers build (they read the same created_at), never on the next one. # # The ref is a BRANCH because repository rulesets cover only branches and tags; a ruleset on refs/heads/nightly/* # (update, deletion and non-fast-forward refused; creation allowed; no bypass) keeps it immutable on the server. It is @@ -27,6 +28,7 @@ on: permissions: contents: write + actions: read # this run's own created_at, which is the night (a rerun keeps it) concurrency: group: nightly-pick @@ -52,9 +54,14 @@ jobs: - name: Pick C for the night id: pick + env: + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - night="$(bash scripts/release/nightly_pick.sh night)" + # the clock the producers read (workflow_run.created_at), not now: a rerun after noon UTC the next + # day would otherwise pick the next night while every chained producer builds this one + at="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at)" + night="$(bash scripts/release/nightly_pick.sh night --at "$at")" rc=0 bash scripts/release/nightly_pick.sh pick --night "$night" --sha "$GITHUB_SHA" || rc=$? [ "$rc" -eq 0 ] || { echo "pick exited $rc"; exit "$rc"; } diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index 00d6a4f53e..f57d34baa2 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -22,7 +22,7 @@ five_whys: equations: night: formula: "night(t) = UTC date of (t − 12 h)" - domain: "t = the pick's wall-clock time in UTC" + domain: "t = the pick run's created_at in UTC, which a rerun keeps and every chained producer reads (check_producers_chain P7)" codomain: "YYYY-MM-DD" invariants: - "a night runs from 12:00Z to 12:00Z, so a pick that fires hours late still lands on the night it was meant for" diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index 4def915cef..d15f82f615 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -6,7 +6,8 @@ # them instead: each one starts on the pick's workflow_run and, after every checkout, switches the tree to the commit # the pick pinned at nightly/ (scripts/release/nightly_c_checkout.sh). This guard reads the workflow text and # is RED when a producer: -# P1 still has a `schedule:` trigger (a live line in its on: block); +# P1 still has a `schedule:` trigger (a live line in its on: block; the key quoted or with a space before its colon +# is the same key to GitHub); # P2 lacks the chain: workflow_run with workflows: [], types: [completed], # branches: [main]; # P3 checks this repo out (actions/checkout, unless `repository:` names another repo literally) and the very next step @@ -24,6 +25,9 @@ # P6 is read by a lane of scripts/release/nightly_train.sh's LANES table whose event pattern is not exactly # ^workflow_run$ (its scheduled runs are gone), or a lane reads workflow_run runs of a workflow not listed, # or a listed producer has no lane, or LANES is assigned more than once. +# P7 (nightly-pick.yml itself) does not take its night from its own run's created_at, the clock every producer +# reads: a rerun after noon UTC the next day would pick the next night while the producers build this one. +# A listed or unlisted workflow this file cannot read is RED, never a silent pass. # Limits: it reads the workflow text, not the scripts a step calls, so a later run step that fetches or checks out # main again is not seen; an origin/main a step fetches as a declared comparand (guards-nightly's SATD ceiling) is # not a measurement and is not judged here. It reads `on:` as a block mapping with keys at indent 2; any other @@ -117,7 +121,7 @@ judge_file() { /^on:[[:space:]]*$/ { inon = 1; next } inon && live(L) && ind(L) == 0 { inon = 0; wr = 0 } inon && live(L) { - if (L ~ /^ schedule:/) sched = 1 + if (L ~ /^ ["\047]?schedule["\047]?[[:space:]]*:/) sched = 1 # "schedule": is the same key to GitHub if (L ~ /^ workflow_run:/) { wr = 1; next } if (ind(L) <= 2) wr = 0 if (wr && L ~ /^ workflows:/ && list1(L) == pick) cw = 1 @@ -150,6 +154,19 @@ judge_file() { }' "$1" } +P7_AT='at="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at)"' +P7_NIGHT='night="$(bash scripts/release/nightly_pick.sh night --at "$at")"' +p7() { # PICKFILE -> FAIL lines; the pick takes its night from its own run's created_at, the clock every producer reads + local t + t="$(sed -e 's/^[[:space:]]*//' "$1")" || { printf 'FAIL nightly-pick.yml: P7 unreadable\n'; return 1; } + # both lines live and exact, and no other live night call (one taking the clock, not created_at) + if hasl "$t" -x -F -e "$P7_AT" && hasl "$t" -x -F -e "$P7_NIGHT" && ! printf '%s\n' "$t" | grep -e 'nightly_pick\.sh night' | grep -v -e '^#' | grep -q -v -x -F -e "$P7_NIGHT"; then + return 0 + fi + printf 'FAIL nightly-pick.yml: P7 the night is not taken from this run'"'"'s created_at (want %s, then %s)\n' "$P7_AT" "$P7_NIGHT" + return 1 +} + p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by its workflow_run runs, and only those local t="$1/scripts/release/nightly_train.sh" [ -f "$t" ] || { printf 'FAIL nightly_train.sh: P6 missing, so its LANES table cannot be read\n'; return 1; } @@ -179,7 +196,7 @@ p6() { # ROOT -> FAIL lines; the train's LANES must read a listed producer by it hasl() { local text="$1"; shift; grep -q "$@" <<< "$text"; } judge() { # ROOT - local root="$1" wf pick p rc=0 out listed + local root="$1" wf pick p rc=0 out listed jr wf="$root/.github/workflows" [ -f "$wf/nightly-pick.yml" ] || { printf 'FAIL nightly-pick.yml is missing: there is no pick to chain from\n'; return 1; } pick="$(sed -n -e 's/^name:[[:space:]]*//p' "$wf/nightly-pick.yml" | head -n 1 | sed -e 's/[[:space:]]*$//' -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/")" @@ -193,11 +210,14 @@ judge() { # ROOT [ "$p" = nightly-pick ] && continue listed=0 case " $(printf '%s' "$PRODUCERS" | tr '\n' ' ') " in *" $p "*) listed=1 ;; esac - out="$(judge_file "$f" "$pick" "$listed")" + out="$(judge_file "$f" "$pick" "$listed" 2>&1)"; jr=$? + # a file awk cannot open was judged by nothing: it is RED, never a silent pass + [ "$jr" -eq 0 ] || printf -v out '%s\nFAIL %s.yml: unreadable (exit %s), so it was not judged' "$out" "$p" "$jr" [ -n "$out" ] && printf '%s\n' "$out" hasl "$out" -e '^FAIL' && rc=1 done p6 "$root" || rc=1 + p7 "$wf/nightly-pick.yml" || rc=1 if [ "$rc" -eq 0 ]; then printf 'PASS every nightly producer chains from "%s" and measures its C\n' "$pick" else printf 'RED a nightly producer is not chained to the pick, or does not measure its C\n'; fi return "$rc" @@ -239,6 +259,14 @@ cstep() { # [IF]: the C step text, with the checkout's if when given m_none() { :; } m_sched_back() { changed nightly.yml 's/^ workflow_run:$/ schedule:\n - cron: "0 3 * * *"\n workflow_run:/'; } m_sched_comment() { changed nightly.yml 's/^ workflow_run:$/ # schedule:\n # - cron: "0 3 * * *"\n workflow_run:/'; } +m_sched_quoted() { changed book.yml 's/^ workflow_run:$/ "schedule":\n - cron: "0 3 * * *"\n workflow_run:/'; } +m_sched_spaced() { changed nightly.yml 's/^ workflow_run:$/ schedule :\n - cron: "0 3 * * *"\n workflow_run:/'; } +m_unreadable() { chmod 000 book.yml && ! [ -r book.yml ]; } # as root the file stays readable: the row fails closed +m_pick_clock() { changed nightly-pick.yml 's/ night --at "\$at")"$/ night)"/'; } +m_pick_no_at() { changed nightly-pick.yml '/^ *at="\$(gh api /d'; } +m_pick_at_cmt() { changed nightly-pick.yml 's/^\( *\)at="\$(gh api /\1# at="$(gh api /'; } +m_pick_2nd_call() { changed nightly-pick.yml 's/^\( *\)rc=0$/\1n2="$(bash scripts\/release\/nightly_pick.sh night)"\n\1rc=0/'; } +m_pick_cmt_ok() { changed nightly-pick.yml 's/^\( *\)rc=0$/\1# was: night="$(bash scripts\/release\/nightly_pick.sh night)"\n\1rc=0/'; } m_chain_other() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly"]/'; } m_chain_two() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly pick", "CI"]/'; } m_chain_type() { changed book.yml '0,/types: \[completed\]/s//types: [requested]/'; } @@ -307,6 +335,14 @@ self_test() { row base_real_producers 0 'PASS' -- m_none row p1_schedule_back 1 'P1 still has a schedule' -- m_sched_back row p1_schedule_commented 0 'PASS' -- m_sched_comment + row p1_schedule_quoted 1 'book.yml: P1 still has a schedule' -- m_sched_quoted + row p1_schedule_spaced_colon 1 'nightly.yml: P1 still has a schedule' -- m_sched_spaced + row p5_unreadable_producer 1 'book.yml: unreadable' -- m_unreadable + row p7_pick_night_from_clock 1 'nightly-pick.yml: P7' -- m_pick_clock + row p7_pick_without_created_at 1 'nightly-pick.yml: P7' -- m_pick_no_at + row p7_pick_created_at_comment 1 'nightly-pick.yml: P7' -- m_pick_at_cmt + row p7_pick_second_night_call 1 'nightly-pick.yml: P7' -- m_pick_2nd_call + row p7_pick_old_call_in_comment 0 'PASS' -- m_pick_cmt_ok row p2_other_workflow 1 'book.yml: P2' -- m_chain_other row p2_two_workflows 1 'book.yml: P2' -- m_chain_two row p2_requested_type 1 'book.yml: P2' -- m_chain_type @@ -406,7 +442,14 @@ m36_missing_train_read s/\[ -f "\$t" \] ||/[ 1 ] ||/ m37_no_table_ok s/if (!done || !lanes) {/if (0) {/ m38_a_listed_lane_may_be_missing s/if (!(pp\[k\] in seen)) {/if (0) {/ m39_a_second_table_ok s/if (assigned > 1) {/if (0) {/ -m40_hasl_piped s/\(grep -q "[^"]*"\) <<< \("[^"]*"\)/echo \2 | \1/' +m40_hasl_piped s/\(grep -q "[^"]*"\) <<< \("[^"]*"\)/echo \2 | \1/ +m41_quoted_schedule_unseen s|\["\\047\]?schedule\["\\047\]?|schedule| +m42_spaced_colon_unseen s|schedule\["\\047\]?\[\[:space:\]\]\*:|schedule["\\047]?:| +m43_unreadable_passes s/^ \[ "\$jr" -eq 0 \] || printf -v out / true || printf -v out / +m44_no_p7 s/^ p7 "\$wf\/nightly-pick.yml" || rc=1$/ true/ +m45_p7_other_night_call_ok s/grep -q -v -x -F -e "\$P7_NIGHT"; then/false; then/ +m46_p7_at_not_exact s/hasl "\$t" -x -F -e "\$P7_AT"/hasl "$t" -F -e "$P7_AT"/ +m47_p7_comment_counts s/ | grep -v -e .\^#. | / | /' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" From 8726ad4be5fb063d8810d9cde587231d69c39bbd Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 02:45:18 +0200 Subject: [PATCH 27/34] T44: a chained run is judged by the pick run it followed, and P7 reads its calls whole nightly_train.sh: a producer chained from "Nightly pick" builds the pick of the pick run's created_at, which a rerun keeps. The train filed the run under the night of its own createdAt, so a pick rerun the next evening built one pick and was judged green on another. Each chained run now belongs to the night of the newest Nightly pick run created at or before it. When an older pick run was updated after that one was created, the run is not_measured, since that rerun may be what fired it. The query reads the pick runs (createdAt, updatedAt). A missing pick workflow fails the query, and a list that does not parse fails the pick read for every lane. check_producers_chain.sh P7: the live night calls went through a pipe into grep -q. grep -q exits on the first match, the writer takes SIGPIPE, and pipefail turned that into a pass, so a second live call ahead of 1.4 MB of exact ones passed. The calls are now read whole, then tested with hasl. contracts/nightly-train-v1.yaml: the next older run is taken for J, not C. The description states the attribution by the pick run. New rows: a rerun the next evening is credited to the old night (must be RED); a rerun after the next pick is not_measured; a pick run not rerun judges its own night; no pick run seen; a pick-run list kept or unread; the query reads the pick runs or refuses without the workflow; a second live call ahead of 1.4 MB. Mutant ids m41 are no longer duplicated. nightly_train --self-test 97/97, --mutants 75/75 killed check_producers_chain --self-test 66/66, --mutants 49/49 killed check_producers_chain on the tree: PASS pv validate contracts/nightly-train-v1.yaml: valid bashrs finding set unchanged or smaller on both scripts Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-train-v1.yaml | 4 +- scripts/release/check_producers_chain.sh | 18 +++-- scripts/release/nightly_train.sh | 92 +++++++++++++++++++----- 3 files changed, 88 insertions(+), 26 deletions(-) diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index 9631d69d82..aa9682d756 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night the run was created in, and of the night before for a run created 12:00Z-17:59Z, is J), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night of the newest Nightly pick run created at or before it, and of the night before for a run created 12:00Z-17:59Z, is J; a rerun keeps its created_at, so an older pick run updated after that one, or a pick-run list that was not read, is not_measured), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" @@ -28,7 +28,7 @@ equations: - "a run on another commit never stands for J; all lanes are judged on the one J, never a per-lane mix" - "an unfinished run, an unread attempt or a failed read is not_measured, never green" - "a retried success is red: a retried green is not a green" - - "a run whose matched jobs were skipped, cancelled or neutral, or fewer than the lane requires, measured nothing, and the next older run for C is taken; only the matched jobs decide, the run's own conclusion is not read" + - "a run whose matched jobs were skipped, cancelled or neutral, or fewer than the lane requires, measured nothing, and the next older run for J is taken; only the matched jobs decide, the run's own conclusion is not read" verdict_line: formula: "RELEASABLE H=J iff every verdict lane is green; else NOT RELEASABLE: L, R where L is the first failing verdict lane, red before not_measured, then by hours lost, then table order, and R its run id or not_measured" diff --git a/scripts/release/check_producers_chain.sh b/scripts/release/check_producers_chain.sh index d15f82f615..9b05198190 100644 --- a/scripts/release/check_producers_chain.sh +++ b/scripts/release/check_producers_chain.sh @@ -157,10 +157,12 @@ judge_file() { P7_AT='at="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --jq .created_at)"' P7_NIGHT='night="$(bash scripts/release/nightly_pick.sh night --at "$at")"' p7() { # PICKFILE -> FAIL lines; the pick takes its night from its own run's created_at, the clock every producer reads - local t + local t calls t="$(sed -e 's/^[[:space:]]*//' "$1")" || { printf 'FAIL nightly-pick.yml: P7 unreadable\n'; return 1; } # both lines live and exact, and no other live night call (one taking the clock, not created_at) - if hasl "$t" -x -F -e "$P7_AT" && hasl "$t" -x -F -e "$P7_NIGHT" && ! printf '%s\n' "$t" | grep -e 'nightly_pick\.sh night' | grep -v -e '^#' | grep -q -v -x -F -e "$P7_NIGHT"; then + # the live calls are read whole first: grep -q at the end of a pipe exits early, and pipefail turns the SIGPIPE into a pass + calls="$(printf '%s\n' "$t" | sed -n -e '/^#/d' -e '/nightly_pick\.sh night/p')" + if hasl "$t" -x -F -e "$P7_AT" && hasl "$t" -x -F -e "$P7_NIGHT" && ! hasl "$calls" -v -x -F -e "$P7_NIGHT"; then return 0 fi printf 'FAIL nightly-pick.yml: P7 the night is not taken from this run'"'"'s created_at (want %s, then %s)\n' "$P7_AT" "$P7_NIGHT" @@ -267,6 +269,10 @@ m_pick_no_at() { changed nightly-pick.yml '/^ *at="\$(gh api /d'; } m_pick_at_cmt() { changed nightly-pick.yml 's/^\( *\)at="\$(gh api /\1# at="$(gh api /'; } m_pick_2nd_call() { changed nightly-pick.yml 's/^\( *\)rc=0$/\1n2="$(bash scripts\/release\/nightly_pick.sh night)"\n\1rc=0/'; } m_pick_cmt_ok() { changed nightly-pick.yml 's/^\( *\)rc=0$/\1# was: night="$(bash scripts\/release\/nightly_pick.sh night)"\n\1rc=0/'; } +m_pick_2nd_big() { # a second live night call ahead of 1.4 MB of exact ones: a pipe into grep -q loses it under pipefail + awk -v l=" $P7_NIGHT" 'BEGIN { for (i = 0; i < 20000; i++) print l }' > "$TMP_ST/p7fill" + m_pick_2nd_call && changed nightly-pick.yml '/^ *rc=0$/r '"$TMP_ST/p7fill" +} m_chain_other() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly"]/'; } m_chain_two() { changed book.yml 's/workflows: \["Nightly pick"\]/workflows: ["Nightly pick", "CI"]/'; } m_chain_type() { changed book.yml '0,/types: \[completed\]/s//types: [requested]/'; } @@ -342,6 +348,7 @@ self_test() { row p7_pick_without_created_at 1 'nightly-pick.yml: P7' -- m_pick_no_at row p7_pick_created_at_comment 1 'nightly-pick.yml: P7' -- m_pick_at_cmt row p7_pick_second_night_call 1 'nightly-pick.yml: P7' -- m_pick_2nd_call + row p7_pick_second_call_big 1 'nightly-pick.yml: P7' -- m_pick_2nd_big row p7_pick_old_call_in_comment 0 'PASS' -- m_pick_cmt_ok row p2_other_workflow 1 'book.yml: P2' -- m_chain_other row p2_two_workflows 1 'book.yml: P2' -- m_chain_two @@ -443,13 +450,14 @@ m37_no_table_ok s/if (!done || !lanes) {/if (0) {/ m38_a_listed_lane_may_be_missing s/if (!(pp\[k\] in seen)) {/if (0) {/ m39_a_second_table_ok s/if (assigned > 1) {/if (0) {/ m40_hasl_piped s/\(grep -q "[^"]*"\) <<< \("[^"]*"\)/echo \2 | \1/ -m41_quoted_schedule_unseen s|\["\\047\]?schedule\["\\047\]?|schedule| +m49_quoted_schedule_unseen s|\["\\047\]?schedule\["\\047\]?|schedule| m42_spaced_colon_unseen s|schedule\["\\047\]?\[\[:space:\]\]\*:|schedule["\\047]?:| m43_unreadable_passes s/^ \[ "\$jr" -eq 0 \] || printf -v out / true || printf -v out / m44_no_p7 s/^ p7 "\$wf\/nightly-pick.yml" || rc=1$/ true/ -m45_p7_other_night_call_ok s/grep -q -v -x -F -e "\$P7_NIGHT"; then/false; then/ +m45_p7_other_night_call_ok s/ \&\& ! hasl "\$calls" -v -x -F -e "\$P7_NIGHT"; then/; then/ m46_p7_at_not_exact s/hasl "\$t" -x -F -e "\$P7_AT"/hasl "$t" -F -e "$P7_AT"/ -m47_p7_comment_counts s/ | grep -v -e .\^#. | / | /' +m47_p7_comment_counts s| -e ./\^#/d. | | +m48_p7_piped_grep_q s/! hasl "\$calls" \(-v -x -F -e "\$P7_NIGHT"\)/! printf "%s\\n" "$calls" \| grep -q \1/' mutants() { local tmp name expr killed=0 total=0 errors=0 out cut reds cut="$(grep -n -m1 -e "^# -* the case table" "$SCRIPT_PATH" | cut -d: -f1)" diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index 1704acdee5..e375e02f4c 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -19,11 +19,12 @@ # its workflow_run runs only (scripts/release/check_producers_chain.sh P6 keeps this table in step with the # triggers). Such a run is filed under main's head when the pick finished, while its tree is the night's pick # (nightly_c_checkout.sh), and the two differ when a merge lands while the pick runs or a pick is dispatched again -# later (it keeps the older C). So the run counts only when the pick of its night, refs/heads/nightly/ for the -# window [N 12:00Z, N+1 12:00Z) that holds the run's creation time, is C; an unread pick is not_measured. The -# producer takes its night from the pick's start, this check from the run's creation, so a run created 12:00Z-17:59Z -# also needs the night before to be C (the scheduled pick fires at 20:30Z). Limit: a pick that starts before noon -# and finishes after 18:00Z is judged by the later night. A lane is +# later (it keeps the older C). So the run counts only when the pick of its night is C. Its night is the night of +# the newest Nightly pick run created at or before it, refs/heads/nightly/ for the window [N 12:00Z, N+1 12:00Z) +# that holds that pick run's created_at: a pick rerun the next evening keeps its created_at and its producers build +# the old night's pick, so the run's own creation time names the wrong night. An older pick run updated after the +# newest one (a rerun that may be what chained the run), an unread pick or an unread pick-run list is not_measured. +# A run created 12:00Z-17:59Z also needs the night before to be C (the scheduled pick fires at 20:30Z). A lane is # green the newest completed run of its producer on C ran every matched job to success, at attempt 1; # red a matched job ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 # or later (a retried green is not a green: FLOW-003 G4, CI retries 2 -> 0); @@ -129,11 +130,12 @@ evaluate() { printf '%sT12:00:00Z\t%sT12:00:00Z\t%s\n' "$n" "$e" "$o" done < "$2/picks.tsv" > "$2/pickwin.tsv" [ -f "$2/rank.tsv" ] || : > "$2/rank.tsv" - awk -F '\t' -v OFS='\t' -v C="$c" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" -v JOUT="$2/judged" -v MAXLAG="$MAX_LAG" ' + awk -F '\t' -v OFS='\t' -v C="$c" -v PRF="$2/pickruns.tsv" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" -v JOUT="$2/judged" -v MAXLAG="$MAX_LAG" ' FILENAME == ARGV[1] { split($0, a, ";"); n++; L[n] = a[1]; K[n] = a[2]; CK[n] = a[3]; WF[n] = a[4]; EV[n] = a[5]; RE[n] = a[6]; NEED[n] = (a[7] ~ /^[0-9]+$/ ? a[7] + 0 : 1); next } FILENAME == ARGV[2] { AT[$1] = $2; next } FILENAME == ARGV[3] { np++; PS[np] = $1; PE[np] = $2; PO[np] = $3; next } FILENAME == ARGV[4] { RKO[$1 + 0] = $2; NRK++; next } + FILENAME == PRF { npr++; PRC[npr] = "" $1; PRU[npr] = "" $2; next } { key = $3 SUBSEP $10 SUBSEP $13 if (key in seen) next @@ -146,6 +148,17 @@ evaluate() { function nightk(t, k) { for (k = 1; k <= np; k++) if (t >= PS[k] && t < PE[k]) return k; return 0 } function nightc(t, k) { k = nightk(t); return (k ? PO[k] : "") } function prevc(t, k, j) { k = nightk(t); if (!k) return ""; for (j = 1; j <= np; j++) if (PE[j] == PS[k]) return PO[j]; return "" } + # chainc(t) -> the C a chained run created at t measured: the night of the newest Nightly pick run created at or before t + # (a producer reads the created_at of its pick run, which a rerun keeps, so a rerun the next evening is the old night); + # "" with CWHY when no pick run is read at or before t, or an older pick run was updated after that one was created + # (a rerun of it may have fired these producers: which pick they measured is unknown, so it is not_measured) + function chainc(t, k, b) { + b = ""; CWHY = "no pick for that night" + for (k = 1; k <= npr; k++) if (PRC[k] <= t && b < PRC[k]) b = "" PRC[k] + if (b == "") { CWHY = "no pick run seen at or before it"; return "" } + for (k = 1; k <= npr; k++) if (b > PRC[k] && b < PRU[k]) { CWHY = "the pick run of " PRC[k] " was rerun after " b; return "" } + return nightc(b) + } # lane state of run r for lane i: green | red | pending | void; sets WHY, ST, EN, DUP function lst(i, r, k, b, fb, nm, pend, bad, ok, tot) { split("", b); split("", fb); ST = ""; EN = ""; DUP = 0; WHY = "" @@ -194,7 +207,7 @@ evaluate() { r = RID[q] if (RW[r] == WF[i] && REV[r] ~ EV[i] && RBR[r] == "main") { # a chained run stands for the pick of its night, a plain run for its own commit (see the lane loop below) - if (EV[i] == "^workflow_run$") { hm = nightc(RC[r]); if (substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && prevc(RC[r]) != hm) hm = "" } + if (EV[i] == "^workflow_run$") { hm = chainc(RC[r]); if (substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && prevc(RC[r]) != hm) hm = "" } else hm = RH[r] s = lst(i, r); if (hm != "" && (s == "green" || s == "red")) HAS[hm, i] = 1 } } @@ -226,7 +239,7 @@ evaluate() { # but its tree is the pick of its night (nightly_c_checkout.sh), so it stands for that pick and for no other commit: # it counts only when that pick is J (#4798; a re-dispatched pick keeps an older C and still fires with the newer head) if (EV[i] != "^workflow_run$") { if (RH[r] != J) { if (other == "") other = r " on " substr(RH[r], 1, 10); continue } } - else if ((pc = nightc(RC[r])) != J) { if (other == "") other = r " measured the pick " (pc == "" ? "no pick for that night" : substr(pc, 1, 10)); continue } + else if ((pc = chainc(RC[r])) != J) { if (other == "") other = r " measured the pick " (pc == "" ? CWHY : substr(pc, 1, 10)); continue } # a run created 12:00Z-17:59Z may come from a pick that started before noon: the night before must be J too if (EV[i] == "^workflow_run$" && substr(RC[r], 12, 2) >= "12" && substr(RC[r], 12, 2) < "18" && (pc = prevc(RC[r])) != J) { if (other == "") other = r " may have measured the pick " (pc == "" ? "no pick for that night" : substr(pc, 1, 10)); continue } if (pick == "" && s != "void") { pick = r; pst = s; pwhy = WHY; pS = ST; pE = EN; pD = DUP } @@ -256,7 +269,7 @@ evaluate() { if (att > 1) out(i, "red", pick, att, "success only at attempt " att ": a retried green is not a green") else out(i, "green", pick, att, "") } - }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/pickwin.tsv" "$2/rank.tsv" "$2/runs.tsv" + }' <(printf '%s\n' "$1") "$2/attempts.tsv" "$2/pickwin.tsv" "$2/rank.tsv" "$2/pickruns.tsv" "$2/runs.tsv" } # hours LANESOUT REDAGE_OUTPUT_FILE -> lanehours lost (2 decimals) for every lane red_age measured hours() { @@ -318,9 +331,12 @@ gql_query() { split("\n") | map(select(length > 0)) as $want | ($wf | first | .workflows | map(select(.path as $x | $want | index($x))) ) as $hit | if ($hit | length) != ($want | length) then error("producer workflow missing from the list") else . end + | ($wf | first | .workflows | map(select(.path == ".github/workflows/nightly-pick.yml")) | first) as $pk + | if $pk == null then error("the Nightly pick workflow is missing from the list") else . end | ($repo | split("/")) as $own | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } history(first: \($hn)) { nodes { oid statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } } " - + ([$hit | to_entries[] | "w\(.key): node(id: \"\(.value.node_id)\") { ... on Workflow { id runs(first: 12) { nodes { databaseId createdAt event checkSuite { status conclusion branch { name } commit { oid } checkRuns(first: 100, filterBy: {checkType: ALL}) { pageInfo { hasNextPage } nodes { name status conclusion startedAt completedAt } } } } } } }"] | join(" ")) + " }"' 2>/dev/null + + ([$hit | to_entries[] | "w\(.key): node(id: \"\(.value.node_id)\") { ... on Workflow { id runs(first: 12) { nodes { databaseId createdAt event checkSuite { status conclusion branch { name } commit { oid } checkRuns(first: 100, filterBy: {checkType: ALL}) { pageInfo { hasNextPage } nodes { name status conclusion startedAt completedAt } } } } } } }"] | join(" ")) + + " pk: node(id: \"\($pk.node_id)\") { ... on Workflow { runs(first: 20) { nodes { createdAt updatedAt } } } } }"' 2>/dev/null } # picks_read GQLJSON PICKS PICKFAIL -> PICKS (nightC), and PICKFAIL empty when the refs were read. An absent ref is a row # missing from a good read (PICKFAIL stays empty: the chained lane falls to c310's "no pick for that night"); a response whose @@ -330,6 +346,14 @@ picks_read() { if jq -e '(.data.repository.nightly.nodes | type) == "array"' "$1" > /dev/null 2>&1 && picks_of "$1" > "$2" 2>/dev/null; then : else : > "$2"; printf 'the nightly pick refs did not parse\n' > "$3"; fi } +# pickruns_read GQLJSON PICKRUNS PICKFAIL -> PICKRUNS (created_atupdated_at of each Nightly pick run read). A chained run +# is the night of the newest pick run created at or before it; a run list that is not an array is a failed read, written +# to PICKFAIL: every lane is not_measured, as for the refs +pickruns_read() { + jq -e '(.data.pk.runs.nodes | type) == "array"' "$1" > /dev/null 2>&1 && jq -r '.data.pk.runs.nodes[] | select((.createdAt | type) == "string" and (.updatedAt | type) == "string") | [.createdAt, .updatedAt] | @tsv' "$1" > "$2" 2>/dev/null && return 0 + printf '' > "$2" + printf 'the Nightly pick runs did not parse\n' > "$3" +} # picks_of GQLJSON -> nightC for each refs/heads/nightly/ the query returned; other names are dropped picks_of() { jq -r '(.data.repository.nightly.nodes // [])[] | select(.name | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) | [.name, .target.oid] | @tsv' "$1"; } normalize() { @@ -373,7 +397,7 @@ rank_rows() { # is recorded in RAW/read and makes every producer lane not_measured fetch() { local raw="$1" cache="$2" lim rem fl hdr st q a ids r - printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv"; : > "$raw/picks.tsv"; : > "$raw/rank.tsv"; : > "$raw/pickfail" + printf 'failed: not read\n' > "$raw/read"; : > "$raw/C"; : > "$raw/tree"; : > "$raw/runs.tsv"; : > "$raw/attempts.tsv"; : > "$raw/picks.tsv"; : > "$raw/rank.tsv"; : > "$raw/pickfail"; : > "$raw/pickruns.tsv" read -r lim rem <<< "$(gh api rate_limit --jq '"\(.resources.core.limit) \(.resources.core.remaining)"' 2>/dev/null)" case "$lim:$rem" in *[!0-9:]*|:*|*:) printf 'failed: rate_limit unreadable\n' > "$raw/read"; return 0 ;; esac # the floor is a fifth of the token's own hourly limit, capped at RATE_FLOOR: the fleet PAT (5000) keeps its 1000 @@ -404,6 +428,7 @@ fetch() { # "no pick for that night" (the ref is absent: a row missing from a good read) is not "pick read failed" (the refs came # back unusable): the second is written to pickfail and judges every lane not_measured, it never falls back to the first picks_read "$raw/graphql.json" "$raw/picks.tsv" "$raw/pickfail" + pickruns_read "$raw/graphql.json" "$raw/pickruns.tsv" "$raw/pickfail" grep -q -E '^[0-9a-f]{40}$' "$raw/C" || { printf 'failed: no head of main in the response\n' > "$raw/read"; return 0; } normalize "$raw/graphql.json" "$raw/workflows.json" > "$raw/runs.tsv" 2>/dev/null || { printf 'failed: response did not normalize\n' > "$raw/read"; return 0; } rank_rows "$raw/graphql.json" > "$raw/rank.tsv" 2>/dev/null || : > "$raw/rank.tsv" @@ -494,7 +519,7 @@ run_train() { mkdir -p "$raw" || exit 1 step read if [ -n "$from" ]; then - for f in C read runs.tsv attempts.tsv tree picks.tsv rank.tsv pickfail; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done + for f in C read runs.tsv attempts.tsv tree picks.tsv rank.tsv pickfail pickruns.tsv; do [ "$from/$f" -ef "$raw/$f" ] || cp "$from/$f" "$raw/$f" 2>/dev/null || : > "$raw/$f"; done else fetch "$raw" "$OUTDIR/cache"; fi CSHA="$(cat "$raw/C" 2>/dev/null)" step judge @@ -627,6 +652,8 @@ fixture() { } > "$1/runs.tsv" printf '101\t1\n201\t1\n301\t1\n' > "$1/attempts.tsv" printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_C" "$ST_X" > "$1/picks.tsv" # run 201 (02:00Z on the 4th) is night 2026-10-03 + # the Nightly pick runs, created_at and updated_at: one at noon each day, so a run belongs to the night whose window holds it + printf '2026-10-03T12:00:00Z\t2026-10-03T12:05:00Z\n2026-10-04T12:00:00Z\t2026-10-04T12:05:00Z\n' > "$1/pickruns.tsv" printf '0\t%s\n' "$ST_C" > "$1/rank.tsv" } # ranks DIR SHA... -> DIR/rank.tsv: main's head and history, newest first (rank 0 = the head, aaaa...) @@ -682,6 +709,24 @@ self_test() { row the_search_credits_a_chained_run_to_its_pick_not_the_head 0 "RELEASABLE H=$ST_X (judged bbbbbbbbbb, head aaaaaaaaaa, lag 1 commits)" "NOT RELEASABLE" -- st_chain "$d" d="$tmp/jstraddle"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-04T13:00:00Z\t/' "$d/runs.tsv"; dupx "$d"; printf '2026-10-03\t%s\n2026-10-04\t%s\n' "$ST_C" "$ST_X" > "$d/picks.tsv" row the_search_skips_a_commit_whose_chained_run_straddles_two_picks 0 "NOT RELEASABLE: v-b" "(judged" -- st_chain "$d" + # a pick rerun the next evening keeps its created_at, so its producers built the old night's pick: they are created on + # the new night, but the newest pick run created before them is the old one (measured: one pick built, another judged green) + st_rr() { printf '2026-10-05\t%s\n2026-10-06\t%s\n' "$ST_X" "$ST_C" > "$1/picks.tsv"; printf '2026-10-05T20:30:00Z\t%s\n2026-10-06T20:30:00Z\t2026-10-06T20:35:00Z\n' "$2" > "$1/pickruns.tsv"; st_chainl "$1"; } + d="$tmp/rerun"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-06T19:05:00Z\t/' "$d/runs.tsv" + row a_rerun_pick_next_evening_credits_the_old_night 0 "201 measured the pick bbbbbbbbbb" "RELEASABLE H=" -- st_rr "$d" 2026-10-06T19:00:00Z + d="$tmp/rerun2"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-06T21:05:00Z\t/' "$d/runs.tsv" + row a_rerun_after_the_next_pick_is_not_measured 0 "201 measured the pick the pick run of 2026-10-05T20:30:00Z was rerun after 2026-10-06T20:30:00Z" "RELEASABLE H=" -- st_rr "$d" 2026-10-06T21:00:00Z + d="$tmp/rerun3"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-06T21:05:00Z\t/' "$d/runs.tsv" + row a_pick_run_not_rerun_judges_its_own_night 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_rr "$d" 2026-10-05T20:35:00Z + d="$tmp/noprun"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; : > "$d/pickruns.tsv" + row a_chained_run_without_a_read_pick_run_is_not_measured 0 "201 measured the pick no pick run seen at or before it" "RELEASABLE H=" -- st_chainl "$d" + st_prr() { + printf '%s' "$1" > "$tmp/gqlrr.json"; : > "$tmp/prr.fail" + pickruns_read "$tmp/gqlrr.json" "$tmp/prr.tsv" "$tmp/prr.fail" + cat "$tmp/prr.tsv" "$tmp/prr.fail" + } + row pickruns_read_keeps_each_run 0 "2026-10-05T20:35:00Z" "did not parse" -- st_prr '{"data":{"pk":{"runs":{"nodes":[{"createdAt":"2026-10-05T20:30:00Z","updatedAt":"2026-10-05T20:35:00Z"}]}}}}' + row a_pick_run_list_that_is_not_read_failed 0 "the Nightly pick runs did not parse" "2026" -- st_prr '{"data":{"pk":null}}' d="$tmp/schedpick"; fixture "$d"; sed -i "s/$ST_C/$ST_X/" "$d/picks.tsv" row a_schedule_lane_ignores_the_picks 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" st_picks() { printf '%s' '{"data":{"repository":{"nightly":{"nodes":[{"name":"2026-10-03","target":{"oid":"c3"}},{"name":"probe","target":{"oid":"p"}}]}}}}' > "$tmp/gqlp.json"; picks_of "$tmp/gqlp.json"; } @@ -728,7 +773,7 @@ self_test() { row verdict_lanes_are_release_day_checks 0 "verdict=ci-main deep-doctests deep-nodefault deep-examples deep-bins-build deep-bins-smoke dogfood models readiness milestone cleanroom-cpu cleanroom-gpu assets preflight publish-dryrun;coverage=info" "" -- \ st_verdict_lanes_are_release_day_checks st_norm() { - printf '{"workflows":[{"node_id":"WA","path":".github/workflows/a.yml"},{"node_id":"WC","path":".github/workflows/c.yml"}]}\n' > "$tmp/wf.json" + printf '{"workflows":[{"node_id":"WA","path":".github/workflows/a.yml"},{"node_id":"WC","path":".github/workflows/c.yml"},{"node_id":"WP","path":".github/workflows/nightly-pick.yml"}]}\n' > "$tmp/wf.json" printf '%s' '{"data":{"repository":{"defaultBranchRef":{"name":"main","target":{"oid":"'"$ST_C"'","tree":{"oid":"t"},"statusCheckRollup":{"contexts":{"nodes":[{},{"name":"gate","status":"COMPLETED","conclusion":"SUCCESS","startedAt":"s","completedAt":"e","checkSuite":{"status":"COMPLETED","conclusion":"SUCCESS","branch":{"name":"main"},"workflowRun":{"databaseId":301,"event":"push","createdAt":"c","workflow":{"id":"WC"}}}}]}}}}},"w0":{"id":"WA","runs":{"nodes":[{"databaseId":101,"createdAt":"c1","event":"schedule","checkSuite":{"status":"COMPLETED","conclusion":"SUCCESS","branch":{"name":"main"},"commit":{"oid":"'"$ST_C"'"},"checkRuns":{"nodes":[{"name":"job-a","status":"COMPLETED","conclusion":"SUCCESS","startedAt":"s1","completedAt":"e1"},{"name":"job-a","status":"COMPLETED","conclusion":"FAILURE","startedAt":"s0","completedAt":"e0"}]}}}]}}}}' > "$tmp/gql.json" normalize "$tmp/gql.json" "$tmp/wf.json" } @@ -737,6 +782,9 @@ self_test() { row gql_query_names_every_producer 0 'w1: node(id: "WC")' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" row gql_query_refuses_a_missing_producer 5 "" "query" -- gql_query "$ST_LANES" "$tmp/wf.json" row gql_query_reads_the_nightly_picks 0 'nightly: refs(refPrefix: "refs/heads/nightly/"' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" + row gql_query_reads_the_pick_runs 0 'pk: node(id: "WP") { ... on Workflow { runs(first: 20) { nodes { createdAt updatedAt' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" + st_gql_nopick() { sed -e 's/,{"node_id":"WP","path":".github\/workflows\/nightly-pick.yml"}//' "$tmp/wf.json" > "$tmp/wfnp.json"; gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wfnp.json"; } + row gql_query_refuses_without_the_pick_workflow 5 "" "query" -- st_gql_nopick d="$tmp/voidnew"; fixture "$d" printf 'wf\t.github/workflows/a.yml\t102\tschedule\tmain\t%s\t2026-10-04T05:00:00Z\tCOMPLETED\tCANCELLED\tjob-a\tCOMPLETED\tCANCELLED\t\t\t1\n' "$ST_C" >> "$d/runs.tsv" row a_void_newer_run_falls_to_the_older_green 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_decide "$d" @@ -818,7 +866,7 @@ v-e;verdict;check E;-;-;-" "$d" 2026-10-04T06:00:00Z; cat "$d/line" "$d/lanes.ts row no_history_judges_only_the_head 0 "NOT RELEASABLE: v-a, not_measured (+2 more)" "RELEASABLE H=" -- st_decide "$d" st_moved_bundle() { local m="$tmp/mb/raw" t='\t' - mkdir -p "$m"; printf '0\t%s\n1\t%s\n' "$ST_C" "$ST_X" > "$m/rank.tsv"; printf '%s\n' "$ST_C" > "$m/C"; printf 'ok\n' > "$m/read"; printf 'cccc\n' > "$m/tree"; printf '2026-10-03\t%s\n' "$ST_X" > "$m/picks.tsv"; printf '501\t1\n502\t1\n503\t1\n' > "$m/attempts.tsv" + mkdir -p "$m"; printf '0\t%s\n1\t%s\n' "$ST_C" "$ST_X" > "$m/rank.tsv"; printf '%s\n' "$ST_C" > "$m/C"; printf 'ok\n' > "$m/read"; printf 'cccc\n' > "$m/tree"; printf '2026-10-03\t%s\n' "$ST_X" > "$m/picks.tsv"; printf '2026-10-03T20:30:00Z\t2026-10-03T20:35:00Z\n' > "$m/pickruns.tsv"; printf '501\t1\n502\t1\n503\t1\n' > "$m/attempts.tsv" { printf "rollup${t}.github/workflows/ci.yml${t}501${t}push${t}main${t}%s${t}2026-10-03T23:00:00Z${t}COMPLETED${t}SUCCESS${t}ci / gate${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" printf "rollup${t}.github/workflows/ci.yml${t}501${t}push${t}main${t}%s${t}2026-10-03T23:00:00Z${t}COMPLETED${t}SUCCESS${t}workspace-test${t}COMPLETED${t}SUCCESS${t}${t}${t}1\n" "$ST_X" @@ -843,7 +891,7 @@ v-e;verdict;check E;-;-;-" "$d" 2026-10-04T06:00:00Z; cat "$d/line" "$d/lanes.ts ' *actions/workflows*) printf "HTTP/2.0 200 OK\r\netag: \"e1\"\r\n\r\n"; cat "$STUB_WF" ;;' \ ' *graphql*) echo "{\"errors\":[{\"message\":\"stub\"}]}" ;;' \ ' *) echo 1 ;;' 'esac' > "$tmp/bin/gh"; chmod +x "$tmp/bin/gh" - printf '{"workflows":[%s]}\n' '{"node_id":"WA","path":".github/workflows/a.yml"},{"node_id":"WB","path":".github/workflows/b.yml"},{"node_id":"WC","path":".github/workflows/c.yml"},{"node_id":"WD","path":".github/workflows/d.yml"}' > "$tmp/wfall.json" + printf '{"workflows":[%s]}\n' '{"node_id":"WA","path":".github/workflows/a.yml"},{"node_id":"WB","path":".github/workflows/b.yml"},{"node_id":"WC","path":".github/workflows/c.yml"},{"node_id":"WD","path":".github/workflows/d.yml"},{"node_id":"WP","path":".github/workflows/nightly-pick.yml"}' > "$tmp/wfall.json" st_fetch() { local f; f="$(realpath -m -s -- "$tmp/f$1")" mkdir -p "$f" @@ -959,12 +1007,12 @@ m40_config_token_ignored s/^ if grep -qsE / if false \&\& grep -qs m41_home_cargo_unchecked s/ "\$HOME\/.cargo"; do$/; do/ m42_floor_ignores_the_limit s/fl=\$((lim \/ 5))/fl=$RATE_FLOOR/ m43_any_event_counts s/ && REV\[r\] ~ EV\[i\]// -m44_chain_ignores_the_pick s/(pc = nightc(RC\[r\])) != J)/0)/ +m44_chain_ignores_the_pick s/(pc = chainc(RC\[r\])) != J)/0)/ m45_any_night_counts s/if (t >= PS\[k\] && t < PE\[k\]) return k/if (PO[k] == C) return k/ m46_window_end_inclusive s/t < PE\[k\]/t <= PE[k]/ m47_window_start_exclusive s/t >= PS\[k\]/t > PS[k]/ m48_every_lane_checks_the_pick s/if (EV\[i\] != "\^workflow_run\$") {/if (0) {/ -m49_from_skips_the_picks s/ tree picks.tsv rank.tsv pickfail; do/ tree rank.tsv pickfail; do/ +m49_from_skips_the_picks s/ tree picks.tsv rank.tsv pickfail pickruns.tsv; do/ tree rank.tsv pickfail pickruns.tsv; do/ m50_picks_any_ref_name s/ | select(.name | test("^\[0-9\]{4}-\[0-9\]{2}-\[0-9\]{2}\$"))// m51_query_drops_the_picks s/ nightly: refs(refPrefix/ unread: refs(refPrefix/ m52_no_straddle_check s/ && (pc = prevc(RC\[r\])) != J)/ \&\& 0)/ @@ -983,8 +1031,14 @@ m64_oldest_commit_wins s/if (all) { J = h; LAG = k; break }/if (all) { J = h; LA m65_rank_history_dropped s/\[(\$c.history.nodes \/\/ \[\])\[\] | .oid | select(. != \$c.oid)\]/[]/ m66_failed_pick_read_falls_back s/\[ -s "\$2\/pickfail" \] \&\& \[ "\${rd:-}" = ok \] \&\& rd=/false \&\& rd=/ m67_failed_pick_read_is_an_absent_pick s/nodes | type) == "array"/nodes | type) != "zzz"/ -m68_search_files_a_chained_run_under_the_head s/hm = nightc(RC\[r\]);/hm = RH[r];/ -m69_search_ignores_the_straddle s/ \&\& prevc(RC\[r\]) != hm) hm = ""/ \&\& 0) hm = ""/' +m68_search_files_a_chained_run_under_the_head s/hm = chainc(RC\[r\]);/hm = RH[r];/ +m69_search_ignores_the_straddle s/ \&\& prevc(RC\[r\]) != hm) hm = ""/ \&\& 0) hm = ""/ +m70_a_chained_run_by_its_own_creation s/chainc(RC/nightc(RC/g +m71_an_older_pick_rerun_is_ignored s/if (b > PRC\[k\] \&\& b < PRU\[k\])/if (0)/ +m72_an_unread_pick_run_list_passes s/^ printf .the Nightly pick runs did not parse.n. > "\$3"$/ true/ +m73_the_pick_runs_are_not_queried s/ + " pk: node(id: / + " pq: node(id: / +m74_the_pick_workflow_may_be_missing s/if \$pk == null then error/if false then error/ +m75_from_skips_the_pick_runs s/ pickfail pickruns.tsv; do/ pickfail; do/' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 186ed5841a53f641205921c5fd3d903d5dd7f474 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 03:30:11 +0200 Subject: [PATCH 28/34] T44: the train's query nests each field where GitHub has it, and a full pick-run list must reach back 30 days nightly_train.sh asked for `nightly: refs(...)` inside defaultBranchRef. GitHub refuses that ("Field 'refs' doesn't exist on type 'Ref'"), so every read failed and every lane printed not_measured. Fixture answers never saw the nesting. The field now sits under repository. Run once against GitHub with the real workflow list: no errors key, and nightly refs, history and pick runs all came back. The old query, sent the same way, returned that error. A new row measures the brace depth of each top-level field in the query. Pick-run window: the query read the 20 newest Nightly pick runs, and a rerun of an older one was unseen, so its producers were credited to the newest night (measured: green on A from B's builds). It now reads 100. A chained run is not_measured when the list is full and its oldest run is less than 30 days, GitHub's rerun limit, before the run. The receipt states the chained rule as chainc, not nightc. The contract description and the script header state the window rule. New rows: a full list under 30 days is not_measured (must be RED); a full list over 30 days judges; the query's field depths. nightly_train --self-test 100/100, --mutants 81/81 killed check_producers_chain on the tree: PASS pv validate contracts/nightly-train-v1.yaml: valid bashrs finding set unchanged or smaller Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- contracts/nightly-train-v1.yaml | 2 +- docs/audits/impl-PMAT-4798-receipt.md | 5 ++- scripts/release/nightly_train.sh | 55 ++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/contracts/nightly-train-v1.yaml b/contracts/nightly-train-v1.yaml index aa9682d756..335fbd3d87 100644 --- a/contracts/nightly-train-v1.yaml +++ b/contracts/nightly-train-v1.yaml @@ -2,7 +2,7 @@ metadata: version: 1.0.0 created: '2026-10-04' author: PAIML Engineering - description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night of the newest Nightly pick run created at or before it, and of the night before for a run created 12:00Z-17:59Z, is J; a rerun keeps its created_at, so an older pick run updated after that one, or a pick-run list that was not read, is not_measured), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." + description: "BLD-002 row R4, the nightly evidence train: one entry point judges one commit J of main from the results its lanes' nightly producers recorded for J (a plain producer by the run on J; a producer chained from the nightly pick by its workflow_run runs only, and only when the pick of the night of the newest Nightly pick run created at or before it, and of the night before for a run created 12:00Z-17:59Z, is J; a rerun keeps its created_at, so an older pick run updated after that one, a pick-run list that was not read, or a full list of 100 pick runs whose oldest is less than 30 days (the rerun limit) before the run, is not_measured), and prints one line, RELEASABLE H= or NOT RELEASABLE: , . scripts/release/nightly_train.sh is the judge. A lane that cannot be read prints not_measured, never pass. J is the head C unless the head lacks a measured run from some verdict producer; then J is the newest commit at most 6 behind C on which every verdict producer has a green or red run, and the line adds (judged J, head C, lag k commits); none such means J = C. A night with no pick ref is not_measured for a chained lane; a pick read that failed is not_measured for every lane and never falls back to that. Report-only: it blocks nothing and no release script reads it." kind: pattern references: - "scripts/release/nightly_train.sh — the judge, its case table (--self-test) and its planted mutants (--mutants)" diff --git a/docs/audits/impl-PMAT-4798-receipt.md b/docs/audits/impl-PMAT-4798-receipt.md index 77bd1dd5c3..e939378a3c 100644 --- a/docs/audits/impl-PMAT-4798-receipt.md +++ b/docs/audits/impl-PMAT-4798-receipt.md @@ -15,7 +15,10 @@ not the head. ## How c310's rule carries to J - A plain (schedule, push) run stands for the commit it ran on: it counts for J when its head_sha is J. - A chained (`workflow_run`) run's tree is the pick of its night, never its head_sha, so it stands for that pick only: - it counts for J when `nightc(created_at)` == J, and for a run created 12:00Z-17:59Z also `prevc` == J. The J search + it counts for J when `chainc(created_at)` == J: the night of the newest Nightly pick run created at or before it (a + rerun keeps its created_at, so a rerun the next evening is the old night), not_measured when an older pick run was + updated after that one, no pick run was seen, or the list is full (100 runs) and reaches back less than 30 days, the + rerun limit (a rerun of an older run would be unseen); for a run created 12:00Z-17:59Z also `prevc` == J. The J search applies the same rule when it asks whether a commit has a measured run on every lane. - Candidates are main's head (rank 0) and its history from the SAME GraphQL query (`history(first: 7)`), written to `RAW/rank.tsv`; lag is the rank of J (exact). One J for all lanes, never a per-lane mix. A rank list whose rank 0 is diff --git a/scripts/release/nightly_train.sh b/scripts/release/nightly_train.sh index e375e02f4c..9a9ad1ccaa 100755 --- a/scripts/release/nightly_train.sh +++ b/scripts/release/nightly_train.sh @@ -23,7 +23,8 @@ # the newest Nightly pick run created at or before it, refs/heads/nightly/ for the window [N 12:00Z, N+1 12:00Z) # that holds that pick run's created_at: a pick rerun the next evening keeps its created_at and its producers build # the old night's pick, so the run's own creation time names the wrong night. An older pick run updated after the -# newest one (a rerun that may be what chained the run), an unread pick or an unread pick-run list is not_measured. +# newest one (a rerun that may be what chained the run), an unread pick or an unread pick-run list is not_measured, and +# so is a full pick-run list (PICK_RUNS) whose oldest run is less than 30 days, the rerun limit, before the run. # A run created 12:00Z-17:59Z also needs the night before to be C (the scheduled pick fires at 20:30Z). A lane is # green the newest completed run of its producer on C ran every matched job to success, at attempt 1; # red a matched job ended failure, timed_out or startup_failure, or the run succeeded only at attempt 2 @@ -77,6 +78,7 @@ SCRIPT_PATH="$HERE/${BASH_SOURCE[0]##*/}" REPO="paiml/aprender" MAX_CALLS=10 MAX_LAG=6 # J (the commit judged) is at most this many commits behind the head of main +PICK_RUNS=100 # Nightly pick runs listed; a full list reaching back less than 30 days (the rerun limit) is not_measured RATE_FLOOR=1000 UNIT="aprender-nightly-train" # lane;kind;release-day check;producer workflow;event pattern;job-name pattern[;required job names, default 1] @@ -130,7 +132,7 @@ evaluate() { printf '%sT12:00:00Z\t%sT12:00:00Z\t%s\n' "$n" "$e" "$o" done < "$2/picks.tsv" > "$2/pickwin.tsv" [ -f "$2/rank.tsv" ] || : > "$2/rank.tsv" - awk -F '\t' -v OFS='\t' -v C="$c" -v PRF="$2/pickruns.tsv" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" -v JOUT="$2/judged" -v MAXLAG="$MAX_LAG" ' + awk -F '\t' -v OFS='\t' -v C="$c" -v PRF="$2/pickruns.tsv" -v PRN="$PICK_RUNS" -v RD="${rd:-failed: no read status}" -v MODE="$3" -v HIST="$2/hist_redage.tsv" -v JOUT="$2/judged" -v MAXLAG="$MAX_LAG" ' FILENAME == ARGV[1] { split($0, a, ";"); n++; L[n] = a[1]; K[n] = a[2]; CK[n] = a[3]; WF[n] = a[4]; EV[n] = a[5]; RE[n] = a[6]; NEED[n] = (a[7] ~ /^[0-9]+$/ ? a[7] + 0 : 1); next } FILENAME == ARGV[2] { AT[$1] = $2; next } FILENAME == ARGV[3] { np++; PS[np] = $1; PE[np] = $2; PO[np] = $3; next } @@ -151,11 +153,21 @@ evaluate() { # chainc(t) -> the C a chained run created at t measured: the night of the newest Nightly pick run created at or before t # (a producer reads the created_at of its pick run, which a rerun keeps, so a rerun the next evening is the old night); # "" with CWHY when no pick run is read at or before t, or an older pick run was updated after that one was created - # (a rerun of it may have fired these producers: which pick they measured is unknown, so it is not_measured) - function chainc(t, k, b) { + # (a rerun of it may have fired these producers: which pick they measured is unknown, so it is not_measured), or the + # list is full (PRN runs) and its oldest run is less than 30 days before t: a run older than the list may be rerun + # for 30 days, and that rerun would be unseen + function dz(y, m, d, era, yoe, doy, doe, mp) { + y -= (m <= 2); era = int(y / 400); yoe = y - era * 400; mp = (m + 9) % 12 + doy = int((153 * mp + 2) / 5) + d - 1; doe = yoe * 365 + int(yoe / 4) - int(yoe / 100) + doy + return era * 146097 + doe - 719468 + } + function sz(s) { return dz(substr(s, 1, 4) + 0, substr(s, 6, 2) + 0, substr(s, 9, 2) + 0) * 86400 + substr(s, 12, 2) * 3600 + substr(s, 15, 2) * 60 + substr(s, 18, 2) } + function chainc(t, k, b, o) { b = ""; CWHY = "no pick for that night" for (k = 1; k <= npr; k++) if (PRC[k] <= t && b < PRC[k]) b = "" PRC[k] if (b == "") { CWHY = "no pick run seen at or before it"; return "" } + o = b; for (k = 1; k <= npr; k++) if (PRC[k] < o) o = "" PRC[k] + if (npr >= PRN && sz(o) > sz(t) - 30 * 86400) { CWHY = "the " npr " pick runs listed reach back only to " o; return "" } for (k = 1; k <= npr; k++) if (b > PRC[k] && b < PRU[k]) { CWHY = "the pick run of " PRC[k] " was rerun after " b; return "" } return nightc(b) } @@ -327,16 +339,16 @@ call_ok() { [ "$CALLS" -lt "$MAX_CALLS" ] || return 1; CALLS=$((CALLS + 1)); } # gql_query LANES WFJSON -> the one GraphQL query; empty when a producer workflow is missing from the list. # jq 1.6 compatible (the timer PATH may find it first): no reserved words such as $or as variable names. gql_query() { - printf '%s\n' "$1" | awk -F ';' '$4 != "-" { print $4 }' | sort -u | jq -R -s --slurpfile wf "$2" -r --arg repo "$REPO" --argjson hn "$((MAX_LAG + 1))" ' + printf '%s\n' "$1" | awk -F ';' '$4 != "-" { print $4 }' | sort -u | jq -R -s --slurpfile wf "$2" -r --arg repo "$REPO" --argjson hn "$((MAX_LAG + 1))" --argjson pn "$PICK_RUNS" ' split("\n") | map(select(length > 0)) as $want | ($wf | first | .workflows | map(select(.path as $x | $want | index($x))) ) as $hit | if ($hit | length) != ($want | length) then error("producer workflow missing from the list") else . end | ($wf | first | .workflows | map(select(.path == ".github/workflows/nightly-pick.yml")) | first) as $pk | if $pk == null then error("the Nightly pick workflow is missing from the list") else . end | ($repo | split("/")) as $own - | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } history(first: \($hn)) { nodes { oid statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } } " + | "query { repository(owner: \"\($own | first)\", name: \"\($own | last)\") { defaultBranchRef { name target { ... on Commit { oid tree { oid } statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } history(first: \($hn)) { nodes { oid statusCheckRollup { contexts(first: 100) { pageInfo { hasNextPage } nodes { ... on CheckRun { name status conclusion startedAt completedAt checkSuite { status conclusion branch { name } workflowRun { databaseId event createdAt workflow { id } } } } } } } } } } } } nightly: refs(refPrefix: \"refs/heads/nightly/\", first: 40, orderBy: {field: ALPHABETICAL, direction: DESC}) { nodes { name target { oid } } } } " + ([$hit | to_entries[] | "w\(.key): node(id: \"\(.value.node_id)\") { ... on Workflow { id runs(first: 12) { nodes { databaseId createdAt event checkSuite { status conclusion branch { name } commit { oid } checkRuns(first: 100, filterBy: {checkType: ALL}) { pageInfo { hasNextPage } nodes { name status conclusion startedAt completedAt } } } } } } }"] | join(" ")) - + " pk: node(id: \"\($pk.node_id)\") { ... on Workflow { runs(first: 20) { nodes { createdAt updatedAt } } } } }"' 2>/dev/null + + " pk: node(id: \"\($pk.node_id)\") { ... on Workflow { runs(first: \($pn)) { nodes { createdAt updatedAt } } } } }"' 2>/dev/null } # picks_read GQLJSON PICKS PICKFAIL -> PICKS (nightC), and PICKFAIL empty when the refs were read. An absent ref is a row # missing from a good read (PICKFAIL stays empty: the chained lane falls to c310's "no pick for that night"); a response whose @@ -718,6 +730,16 @@ self_test() { row a_rerun_after_the_next_pick_is_not_measured 0 "201 measured the pick the pick run of 2026-10-05T20:30:00Z was rerun after 2026-10-06T20:30:00Z" "RELEASABLE H=" -- st_rr "$d" 2026-10-06T21:00:00Z d="$tmp/rerun3"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/; s/\t2026-10-04T02:00:00Z\t/\t2026-10-06T21:05:00Z\t/' "$d/runs.tsv" row a_pick_run_not_rerun_judges_its_own_night 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_rr "$d" 2026-10-05T20:35:00Z + # a full list (100 runs, the two picks and 98 from DAY): a rerun of a run older than the list is unseen (measured: green + # on A from B's builds), so the list must reach back 30 days, the rerun limit + st_rrf() { + printf '2026-10-05\t%s\n2026-10-06\t%s\n' "$ST_X" "$ST_C" > "$1/picks.tsv" + awk -v D="$2" 'BEGIN { print "2026-10-05T20:30:00Z\t2026-10-05T20:35:00Z"; print "2026-10-06T20:30:00Z\t2026-10-06T20:35:00Z" + for (i = 0; i < 98; i++) { s = sprintf("%sT%02d:%02d:00Z", D, int(i / 60), i % 60); print s "\t" s } }' > "$1/pickruns.tsv" + st_chainl "$1" + } + row a_full_pick_run_list_under_30_days_is_not_measured 0 "201 measured the pick the 100 pick runs listed reach back only to 2026-10-01T00:00:00Z" "RELEASABLE H=" -- st_rrf "$d" 2026-10-01 + row a_full_pick_run_list_over_30_days_judges 0 "RELEASABLE H=$ST_C" "NOT RELEASABLE" -- st_rrf "$d" 2026-09-06 d="$tmp/noprun"; fixture "$d"; sed -i '/\t201\t/s/\tschedule\t/\tworkflow_run\t/' "$d/runs.tsv"; : > "$d/pickruns.tsv" row a_chained_run_without_a_read_pick_run_is_not_measured 0 "201 measured the pick no pick run seen at or before it" "RELEASABLE H=" -- st_chainl "$d" st_prr() { @@ -782,7 +804,16 @@ self_test() { row gql_query_names_every_producer 0 'w1: node(id: "WC")' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" row gql_query_refuses_a_missing_producer 5 "" "query" -- gql_query "$ST_LANES" "$tmp/wf.json" row gql_query_reads_the_nightly_picks 0 'nightly: refs(refPrefix: "refs/heads/nightly/"' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" - row gql_query_reads_the_pick_runs 0 'pk: node(id: "WP") { ... on Workflow { runs(first: 20) { nodes { createdAt updatedAt' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" + row gql_query_reads_the_pick_runs 0 'pk: node(id: "WP") { ... on Workflow { runs(first: 100) { nodes { createdAt updatedAt' "" -- gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" + # the depth of each top-level field in the query: GitHub refuses refs nested in defaultBranchRef (measured: "Field + # 'refs' doesn't exist on type 'Ref'", every lane not_measured), and a fixture answer never sees the nesting + st_gql_depth() { + gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wf.json" | awk -v RS='\001' '{ + for (i = 1; i <= length($0); i++) { c = substr($0, i, 1); d += (c == "{") - (c == "}") + if (substr($0, i, 9) == " nightly:" || substr($0, i, 5) == " pk: " || substr($0, i, 5) == " w0: ") printf "%s=%d ", substr($0, i + 1, 2), d } + printf "end=%d\n", d }' + } + row gql_query_nests_each_field_where_github_has_it 0 "ni=2 w0=1 pk=1 end=0" "" -- st_gql_depth st_gql_nopick() { sed -e 's/,{"node_id":"WP","path":".github\/workflows\/nightly-pick.yml"}//' "$tmp/wf.json" > "$tmp/wfnp.json"; gql_query "$(printf '%s\n' "$ST_LANES" | awk -F ';' '$4 ~ /[ac][.]yml$/')" "$tmp/wfnp.json"; } row gql_query_refuses_without_the_pick_workflow 5 "" "query" -- st_gql_nopick d="$tmp/voidnew"; fixture "$d" @@ -1038,7 +1069,13 @@ m71_an_older_pick_rerun_is_ignored s/if (b > PRC\[k\] \&\& b < PRU\[k\])/if (0)/ m72_an_unread_pick_run_list_passes s/^ printf .the Nightly pick runs did not parse.n. > "\$3"$/ true/ m73_the_pick_runs_are_not_queried s/ + " pk: node(id: / + " pq: node(id: / m74_the_pick_workflow_may_be_missing s/if \$pk == null then error/if false then error/ -m75_from_skips_the_pick_runs s/ pickfail pickruns.tsv; do/ pickfail; do/' +m75_from_skips_the_pick_runs s/ pickfail pickruns.tsv; do/ pickfail; do/ +m76_a_full_pick_run_list_is_trusted s/if (npr >= PRN \&\& sz(o)/if (0 \&\& sz(o)/ +m77_the_page_size_is_not_passed s/ -v PRN="[^"]*"// +m78_the_query_reads_twenty s/runs(first: \\(\$pn))/runs(first: 20)/ +m79_the_rerun_limit_is_31_days s/sz(t) - 30 \* 86400/sz(t) - 31 * 86400/ +m80_the_oldest_run_is_the_newest s/if (PRC\[k\] < o) o = "" PRC\[k\]/if (0) o = ""/ +m81_refs_nested_in_the_branch s/} } nightly: refs/} nightly: refs/; s/{ name target { oid } } } } "/{ name target { oid } } } } } "/' # each planted mutant must change the file, still parse, and turn at least one row RED mutants() { local tmp pass=0 fail=0 name expr o rc From 31873feb1f71060de6ca90aad50049a3db0def02 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 6 Oct 2026 13:20:13 +0200 Subject: [PATCH 29/34] T44 #4875: the pmat hunt runs one hop from the pick, after Coverage Nightly Under the pick chain the story and coverage start together; coverage takes 60-70 min and the story job 30, so the story's hunts read not_measured on most nights. The hunt now runs in qwen-hunt-nightly, chained from Coverage Nightly, on that run's own coverage-json artifact (matched on coverage.sha, never head_sha). The story keeps its hunts behind PMAT_HUNT, default 0. - lib_story_pmat.sh: STORY_HUNTS, one table of the eight beats' hunts; qwen-story.sh runs line N via story_hunt N, story_pmat_hunt.sh runs all. - check_producers_chain.sh: HOPS. A hop chains from its listed upstream's name:, its C step reads the upstream run; an unlisted file chained from a listed producer is P5 RED (the old guard passed it vacuously); a hop counts as listed for P6. 8 rows, 9 mutants (74 rows, 58/58 killed). - nightly_train.sh: the qwen-hunt lane. - mutants-nightly.yml: the timeout comment states the chained window. - contracts: FALSIFY-QWEN-STORY-004 reads story_hunt; the pick's clock domain names the one-hop case and its rerun limit. Agent: aprender-a7w Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mutants-nightly.yml | 2 +- .github/workflows/qwen-hunt-nightly.yml | 189 +++++++++++++++++++++++ .github/workflows/qwen-story-daily.yml | 16 +- contracts/nightly-pick-v1.yaml | 2 +- contracts/qwen-story-v1.yaml | 6 +- scripts/check_no_pipe_into_grep_q.sh | 2 +- scripts/check_story_pmat_hunt.sh | 58 +++++-- scripts/lib_story_pmat.sh | 27 ++++ scripts/qwen-story.sh | 36 +---- scripts/release/check_producers_chain.sh | 97 ++++++++++-- scripts/release/nightly_train.sh | 1 + scripts/story_pmat_hunt.sh | 41 +++++ 12 files changed, 406 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/qwen-hunt-nightly.yml create mode 100755 scripts/story_pmat_hunt.sh diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index 40336e6884..caef0af505 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -33,7 +33,7 @@ jobs: concurrency: # PP-19: the gpu runner is a perf host; serialise against every other consumer of it (check_perf_concurrency_groups.sh) group: perf-gx10 cancel-in-progress: false - timeout-minutes: 180 # ends by 06:00Z, before PR traffic rises; a run cut off is red and the next night re-judges from the last success + timeout-minutes: 180 # a 00:30-03:30Z start ends by about 03:30-06:30Z, later when queued behind the perf group; a run cut off is red and the next night re-judges from the last success env: IMAGE: localhost:5000/sovereign-ci:stable MUTANTS_JOBS: ${{ vars.MUTANTS_NIGHTLY_JOBS || '8' }} diff --git a/.github/workflows/qwen-hunt-nightly.yml b/.github/workflows/qwen-hunt-nightly.yml new file mode 100644 index 0000000000..fc1a52d745 --- /dev/null +++ b/.github/workflows/qwen-hunt-nightly.yml @@ -0,0 +1,189 @@ +# The qwen story's pmat bug-hunt manifest, in its own job (#4875). +# +# The hunt's coverage gaps read coverage-nightly's coverage JSON, and only when +# that file was measured on the commit being hunted (scripts/lib_story_pmat.sh). +# qwen-story-daily starts from the nightly pick, as coverage-nightly does, and +# coverage takes 60-70 min against the story job's 30, so inside the story the +# gaps read not_measured on most nights. This workflow starts when Coverage +# Nightly finishes instead, downloads THAT run's coverage-json artifact, and +# hunts the night's C: the same eight hunts the story's beats name +# (STORY_HUNTS), via scripts/story_pmat_hunt.sh. The story itself runs with +# PMAT_HUNT=0 now; a dispatch can still turn its hunts on. +# +# One hop from the pick, never two: Coverage Nightly chains from "Nightly pick", +# and scripts/release/check_producers_chain.sh (HOPS) holds both links. The C +# step reads the COVERAGE run's created_at and conclusion, so a coverage run +# that did not succeed makes this run not_measured. + +name: qwen-hunt-nightly + +on: + workflow_run: + # The workflow NAME (coverage-nightly.yml `name:`), not the file name. + workflows: ["Coverage Nightly"] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + file_issue: + description: 'File a tracking issue if the hunt fails or the manifest grows' + required: false + default: 'true' + +permissions: + contents: read + issues: write # to open/comment on the tracking issue + actions: read # to download coverage-nightly's coverage-json artifact + +concurrency: + group: qwen-hunt-nightly + cancel-in-progress: false + +jobs: + hunt: + name: Qwen story pmat hunt + # pmat and the checkout only: no apr, GPU or model, so the clean-room + # pool, as guards-nightly, and never a perf host. + runs-on: [self-hosted, Linux, X64, clean-room] + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 1 + - name: Measure the night's C, not main's head (T44) + if: github.event_name == 'workflow_run' + env: + NIGHTLY_PICK_AT: ${{ github.event.workflow_run.created_at }} + NIGHTLY_PICK_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + run: bash scripts/release/nightly_c_checkout.sh --at "$NIGHTLY_PICK_AT" --pick-conclusion "$NIGHTLY_PICK_CONCLUSION" + + - name: pmat must match the pinned fleet versions (never install) + run: | + export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" + setsid --wait bash scripts/check_tool_versions.sh + + # The triggering coverage run's own artifact, matched on the coverage.sha it + # records, never on that run's head_sha: a merge between the pick's start and + # its finish moves head_sha off C while coverage measured C. lib_story_pmat.sh + # uses the file only when coverage.sha is this checkout's HEAD; any other + # file, or none, makes the gaps not_measured and the hunt red. + - name: Fetch the triggering coverage run's coverage JSON + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # workflow_run: the Coverage Nightly run that started this one. + # workflow_dispatch: empty, so look one up for the checked-out commit. + COV_RUN_ID: ${{ github.event.workflow_run.id }} + run: | + set -uo pipefail + dir="$RUNNER_TEMP/coverage-json" + rm -rf "${dir:?}" + run_id="$COV_RUN_ID" + if [ -z "$run_id" ]; then + run_id=$(gh run list -R "$GITHUB_REPOSITORY" -w coverage-nightly.yml --commit "$(git rev-parse HEAD)" \ + --limit 5 --json databaseId,status --jq '[.[] | select(.status == "completed")][0].databaseId // empty') || run_id="" + fi + if [ -z "$run_id" ]; then + echo "::warning::no completed coverage-nightly run for $(git rev-parse HEAD) - pmat hunt coverage gaps will be not_measured" + exit 0 + fi + if ! gh run download "$run_id" -R "$GITHUB_REPOSITORY" -n coverage-json -D "$dir"; then + echo "::warning::coverage-nightly run $run_id has no coverage-json artifact - pmat hunt coverage gaps will be not_measured" + exit 0 + fi + echo "STORY_COVERAGE_FILE=$dir/coverage.json" >> "$GITHUB_ENV" + echo "STORY_COVERAGE_SHA=$(cat "$dir/coverage.sha" 2>/dev/null)" >> "$GITHUB_ENV" + echo "coverage JSON from coverage-nightly run $run_id, measured on $(cat "$dir/coverage.sha" 2>/dev/null || echo '(none recorded)'); HEAD is $(git rev-parse HEAD)" + + # ${PIPESTATUS[0]}: the hunt's status, not tee's (FALSIFY-QWEN-STORY-009). + - name: Run scripts/story_pmat_hunt.sh + id: hunt + continue-on-error: true + shell: bash + run: | + set +e + set -o pipefail + export PATH="$HOME/.cargo/bin:$HOME/.local/bin:$PATH" + bash scripts/story_pmat_hunt.sh 2>&1 | tee /tmp/hunt.log + echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + + - name: Extract pmat bug-hunt manifest + id: manifest + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + # Rows are indented; anchor on leading whitespace, as qwen-story-daily does. + mkdir -p /tmp/qwen-hunt-artifacts + grep -E '^[[:space:]]+(gap|churn|fault)[[:space:]]' /tmp/hunt.log \ + > /tmp/qwen-hunt-artifacts/pmat-manifest.txt || true + LINES=$(wc -l < /tmp/qwen-hunt-artifacts/pmat-manifest.txt) + echo "manifest_lines=$LINES" >> "$GITHUB_OUTPUT" + PREV_LINES=$(gh run list --workflow=qwen-hunt-nightly.yml --status=success --limit=1 \ + --json databaseId --jq '.[0].databaseId' 2>/dev/null \ + | xargs -I {} gh run download {} -n pmat-manifest -D /tmp/prev 2>/dev/null \ + && wc -l < /tmp/prev/pmat-manifest.txt 2>/dev/null || echo 0) + GROWTH=$((LINES - PREV_LINES)) + echo "growth=$GROWTH" >> "$GITHUB_OUTPUT" + echo "Previous manifest: $PREV_LINES lines, current: $LINES lines, growth: $GROWTH" + + - name: Upload pmat manifest artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: pmat-manifest + path: /tmp/qwen-hunt-artifacts/pmat-manifest.txt + retention-days: 90 + + - name: Upload hunt log artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: hunt-log + path: /tmp/hunt.log + retention-days: 30 + + # Notifies, never judges (continue-on-error), as in qwen-story-daily: a + # notification failure must not mask or fake the hunt's own result. + - name: File / update tracking issue on failure + continue-on-error: true + if: | + (steps.hunt.outputs.exit_code != '0' || + fromJson(steps.manifest.outputs.growth || '0') > 5) && + (github.event.inputs.file_issue != 'false') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + set -uo pipefail + for L in qwen-story-daily regression; do + gh label create "$L" --description "Filed by qwen-story-daily" --force >/dev/null 2>&1 || true + done + TITLE="qwen-hunt-nightly: hunt exit=${{ steps.hunt.outputs.exit_code }} manifest_growth=${{ steps.manifest.outputs.growth }}" + EXISTING=$(gh issue list --label qwen-story-daily --state open --limit 1 \ + --json number --jq '.[0].number // empty') + { + echo "## Qwen story pmat hunt — $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "" + echo "- Hunt exit code: \`${{ steps.hunt.outputs.exit_code }}\`" + echo "- Pmat manifest lines: \`${{ steps.manifest.outputs.manifest_lines }}\` (growth: ${{ steps.manifest.outputs.growth }})" + echo "- Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "" + echo "### Hunt log (tail)" + echo '```' + tail -50 /tmp/hunt.log + echo '```' + } > /tmp/issue-body.md + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --body-file /tmp/issue-body.md + else + gh issue create --title "$TITLE" --label qwen-story-daily,regression --body-file /tmp/issue-body.md + fi + + # always(): without a status function GitHub prepends success(), and any + # earlier failure would skip the verdict. + - name: Fail the job if the hunt failed + if: always() && steps.hunt.outputs.exit_code != '0' + run: | + echo "::error::pmat hunt exited ${{ steps.hunt.outputs.exit_code }} - see the hunt-log artifact" + exit 1 diff --git a/.github/workflows/qwen-story-daily.yml b/.github/workflows/qwen-story-daily.yml index aca60e7f13..162590e578 100644 --- a/.github/workflows/qwen-story-daily.yml +++ b/.github/workflows/qwen-story-daily.yml @@ -1,4 +1,4 @@ -# Nightly Qwen-story regression run (after coverage-nightly) + pmat bug-hunt manifest. +# Nightly Qwen-story regression run (chained from the nightly pick); its pmat bug-hunt runs in qwen-hunt-nightly.yml. # # Contract: contracts/qwen-story-v1.yaml § FALSIFY-QWEN-STORY-006. # Runs `scripts/qwen-story.sh` against the canonical Qwen model registry, @@ -16,10 +16,10 @@ on: # Chained, not scheduled (T44, #4686): this producer measures the night's # C that "Nightly pick" pinned at nightly/, never main's head at # whatever hour GitHub got round to creating a scheduled run. - # #4715 chained it from Coverage Nightly so the pmat hunt finds coverage's - # artifact for the same sha. Coverage now starts from the same pick and takes - # 60-70 min, past this job's 30, so the hunt reads not_measured on most nights - # until it moves into its own job chained from Coverage Nightly. + # The pmat hunt runs in qwen-hunt-nightly.yml (#4875), chained from Coverage + # Nightly so it reads that run's coverage of the same C. Coverage starts from the + # same pick and takes 60-70 min, past this job's 30, so here PMAT_HUNT defaults + # to 0; a dispatch with pmat_hunt=1 still hunts inside the story. workflow_run: workflows: ["Nightly pick"] types: [completed] @@ -27,9 +27,9 @@ on: workflow_dispatch: inputs: pmat_hunt: - description: 'Run pmat bug-hunt manifest (1 = yes, 0 = skip for speed)' + description: 'Also run the pmat bug-hunt here (1 = yes; nightly it runs in qwen-hunt-nightly)' required: false - default: '1' + default: '0' file_issue: description: 'File a tracking issue if manifest grows' required: false @@ -73,7 +73,7 @@ jobs: # Default to the runner user's $HOME/models (the models live there, e.g. # /home/noah/models). The old hardcoded /home/runner/models was the retired # runner's path → every beat SKIPped (missing model) even once scheduled. - PMAT_HUNT: ${{ github.event.inputs.pmat_hunt || '1' }} + PMAT_HUNT: ${{ github.event.inputs.pmat_hunt || '0' }} RUST_BACKTRACE: '1' steps: - uses: actions/checkout@v7 diff --git a/contracts/nightly-pick-v1.yaml b/contracts/nightly-pick-v1.yaml index f57d34baa2..57d415de55 100644 --- a/contracts/nightly-pick-v1.yaml +++ b/contracts/nightly-pick-v1.yaml @@ -22,7 +22,7 @@ five_whys: equations: night: formula: "night(t) = UTC date of (t − 12 h)" - domain: "t = the pick run's created_at in UTC, which a rerun keeps and every chained producer reads (check_producers_chain P7)" + domain: "t = the pick run's created_at in UTC, which a rerun keeps and every chained producer reads (check_producers_chain P7); a one-hop producer (HOPS) reads its upstream run's created_at, the same night unless that upstream is rerun the next evening, which its same-sha rule reads as not_measured" codomain: "YYYY-MM-DD" invariants: - "a night runs from 12:00Z to 12:00Z, so a pick that fires hours late still lands on the night it was meant for" diff --git a/contracts/qwen-story-v1.yaml b/contracts/qwen-story-v1.yaml index 56db2a6e45..99d6d48421 100644 --- a/contracts/qwen-story-v1.yaml +++ b/contracts/qwen-story-v1.yaml @@ -79,7 +79,7 @@ proof_obligations: - id: QSA60C-INV-003 type: invariant property: "pmat audit runs against the right modules" - formal: "Each beat's pmat_hunt call references the apr command-handler files it just exercised" + formal: "Each beat's line of STORY_HUNTS (scripts/lib_story_pmat.sh), run by story_hunt N, names the apr command-handler files it just exercised" applies_to: pmat_audit_per_beat falsification_tests: @@ -107,8 +107,8 @@ falsification_tests: - id: FALSIFY-QWEN-STORY-004 rule: "PMAT audit is wired per beat" - prediction: "Each beat function calls pmat_hunt at the end" - test: "[ \"$(grep -cE 'pmat_hunt ' scripts/qwen-story.sh)\" -ge '8' ]" + prediction: "Each beat function ends with story_hunt N, its line of the shared STORY_HUNTS table (#4875)" + test: "[ \"$(grep -cE '^ story_hunt [1-8]$' scripts/qwen-story.sh)\" -ge '8' ]" if_fails: "Bug-hunting layer disconnected from the story" - id: FALSIFY-QWEN-STORY-005 diff --git a/scripts/check_no_pipe_into_grep_q.sh b/scripts/check_no_pipe_into_grep_q.sh index fe39a6ccf8..08d6476f3d 100755 --- a/scripts/check_no_pipe_into_grep_q.sh +++ b/scripts/check_no_pipe_into_grep_q.sh @@ -38,7 +38,7 @@ SELF="scripts/check_no_pipe_into_grep_q.sh" # it is NOT fed by a pipe, so `cmd || grep -q PAT file` has no producer to kill # and no SIGPIPE to misread. The shipped regex matched the SECOND bar of `||` # and flagged 2 safe sites in this tree (dogfood.sh:1477 greps a FILE, -# qwen-story.sh:281 greps a HERE-STRING) plus every new one written since. The +# qwen-story.sh:275 greps a HERE-STRING) plus every new one written since. The # case table below now has a row for it in both directions, because a regex # without a discriminating row is how all five previous versions of this pattern # shipped wrong. diff --git a/scripts/check_story_pmat_hunt.sh b/scripts/check_story_pmat_hunt.sh index 424944e43b..b079cd41ae 100755 --- a/scripts/check_story_pmat_hunt.sh +++ b/scripts/check_story_pmat_hunt.sh @@ -256,23 +256,51 @@ fi # -- 8. Every path the story hunts still exists ----------------------------- # Cause 3. Static, because a path can rot without anyone running the nightly. -if [ -f "$STORY" ]; then - missing="" - checked=0 - while read -r p; do - checked=$((checked + 1)) - [ -e "$p" ] || missing="$missing $p" - done < <(grep -oE '(crates|src)/[A-Za-z0-9_./-]+' "$STORY" \ - | grep -E '\.rs$|/serve$' | sort -u) - want "every source path hunted by qwen-story.sh exists" "" "$missing" - # Fail closed: a discovery step that found nothing must not report success. - if [ "$checked" -ge 15 ]; then - ok "path discovery found $checked hunted paths" - else - bad "path discovery found enough hunted paths" ">= 15" "$checked" - fi +# The paths live in one table, STORY_HUNTS (lib_story_pmat.sh, #4875), read by +# both the story's beats and the hunt-only nightly. +missing="" +checked=0 +while read -r p; do + checked=$((checked + 1)) + [ -e "$p" ] || missing="$missing $p" +done < <(sed -e 's/^[^|]*|//' <<< "$STORY_HUNTS" | tr ' ' '\n' | grep -v '^$' | sort -u) +want "every source path hunted by the story exists" "" "$missing" +# Fail closed: a discovery step that found nothing must not report success. +if [ "$checked" -ge 15 ]; then + ok "path discovery found $checked hunted paths" +else + bad "path discovery found enough hunted paths" ">= 15" "$checked" fi +# -- 10. One table, two readers (#4875) -------------------------------------- +# The story runs line N at the end of beat N; story_pmat_hunt.sh runs all eight. +# A beat that stops calling its line, or a line no beat calls, is a hunt the +# nightly no longer runs where the story said it would. +want "STORY_HUNTS has one line per beat" "8" "$(grep -c '|' <<< "$STORY_HUNTS")" +want "qwen-story.sh calls story_hunt 1..8 once each, in beat order" "1 2 3 4 5 6 7 8" \ + "$(sed -n -e 's/^ story_hunt \([0-9][0-9]*\)$/\1/p' "$STORY" | tr '\n' ' ' | sed -e 's/ $//')" +want "qwen-story.sh names no hunt path itself" "" "$(grep -n -E '^[[:space:]]*pmat_hunt ' "$STORY")" +: > "$FAILLOG" +story_hunt 9 >/dev/null 2>&1; rc=$? +want "a line STORY_HUNTS lacks returns 1" "1" "$rc" +want "a line STORY_HUNTS lacks tallies a failure" "pmat-hunt beat 9 :: STORY_HUNTS has no line 9" "$(cat "$FAILLOG")" +: > "$FAILLOG" +out=$(STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" PMAT_HUNT=1 story_hunt 2); rc=$? +want "story_hunt 2 hunts beat 2's label" "1" "$(grep -c -F -e '-- pmat bug-hunt manifest (qa validate lint) --' <<< "$out")" +want "story_hunt 2 returns 0 with rows" "0" "$rc" + +# The hunt-only nightly, end to end against the stub: eight headers and rc 0 on +# a coverage file of HEAD; rc 2 (never 0) when that file is absent; rc 1 when +# pmat is missing, because the library's silent skip would otherwise pass. +HUNT="scripts/story_pmat_hunt.sh" +out=$(STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" PMAT_HUNT=0 bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: coverage of HEAD returns 0 (PMAT_HUNT=0 in its env is overridden)" "0" "$rc" +want "story_pmat_hunt.sh: runs all eight hunts" "8" "$(grep -c -e '-- pmat bug-hunt manifest (' <<< "$out")" +out=$(env -u STORY_COVERAGE_FILE -u STORY_COVERAGE_SHA bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: no coverage file returns 2" "2" "$rc" +out=$(PMAT_BIN="$TMP/bin/no-such-pmat" STORY_COVERAGE_FILE="$COVFILE" STORY_COVERAGE_SHA="$HEAD_SHA" bash "$HUNT" 2>&1); rc=$? +want "story_pmat_hunt.sh: no pmat returns 1" "1" "$rc" + echo if [ "$fails" -eq 0 ]; then echo "check_story_pmat_hunt: OK - the manifest produces rows, and an empty one fails the beat" diff --git a/scripts/lib_story_pmat.sh b/scripts/lib_story_pmat.sh index ec3129ed9f..d99ab0c94f 100755 --- a/scripts/lib_story_pmat.sh +++ b/scripts/lib_story_pmat.sh @@ -215,3 +215,30 @@ pmat_hunt() { fi return 0 } + +# The eight beats' hunts, one line per beat in beat order: "