diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51c0eb097f..408fa9a820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,43 @@ jobs: printf '%s\n' "$out" printf '%s\n' "$out" | grep -E '^(x86|det|cite|pr_head)=' >> "$GITHUB_OUTPUT" + # T43b (#5884, Noah 16:28Z #3, extends mg-reuse): on merge_group, workspace-test-shard is skipped ONLY when the queue + # entry's tree sha == the tree sha of a green PR run's head (K1-K5: queue head has one parent, the base; the base + # has one parent; the base is an ancestor of the head; M^{tree} == H^{tree}; the head's newest ci.yml pull_request + # run is complete and every job is that run's) AND that run's `workspace-test` verdict succeeded. Any difference + # (changed main, another tree, a lookup that failed) leaves wt=0 and the full shards run. x86-main and determinism + # are NOT read here: they stay un-reused on the queue (20X r2), so x86-main still never waits for this job. + mg-reuse-wt: + if: github.event_name == 'merge_group' + # same lane as the shards it gates (mq on a queue entry); the job only runs on merge_group, the clean-room arm just names the pool for the label guard + runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + pull-requests: read + outputs: + wt: ${{ steps.key.outputs.wt }} + wt_cite: ${{ steps.key.outputs.wt_cite }} + steps: + - uses: actions/checkout@v7 + with: + # full history: K3 (merge-base --is-ancestor) walks the PR head back to the base; the mq runner keeps a warm clone + fetch-depth: 0 + - name: Read the workspace-test reuse key for this queue entry (3 API calls) + id: key + env: + GH_TOKEN: ${{ github.token }} + QREF: ${{ github.event.merge_group.head_ref }} + QHEAD: ${{ github.event.merge_group.head_sha }} + QBASE: ${{ github.event.merge_group.base_sha }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + out=$(bash scripts/ci_mg_reuse.sh resolve --event merge_group --ref "$QREF" --queue-head "$QHEAD" --base "$QBASE" --repo "$REPO") + printf '%s\n' "$out" + printf '%s\n' "$out" | grep -E '^(wt|wt_cite)=' >> "$GITHUB_OUTPUT" + # The same case table on every event, so a change that breaks the key is RED on # its own PR, not first in the queue. gate requires it. mg-reuse-self-test: @@ -475,8 +512,22 @@ jobs: # The workspace tests, one shard per job (nextest hash partition N/12 plus the # explicit runner's round-robin N/12), in parallel on the X64 clean-room pool. # The section body is ci/sections.yml `workspace-test-shard`, unchanged. + # mg-reuse-wt may cite a PR run only if that run really tested the tree. A receipt-reuse run (#5850, tier=reuse) + # runs no test step, so its shards are green over zero tests. This marker needs the shards and runs only after + # they SUCCEEDED, so a run whose shards were skipped or failed never has it; ci_mg_reuse.sh refuses wt=1 unless it + # succeeded (R:k6-wt-ran). Once #5850's receipt-judge job exists, ci_mg_reuse.sh --self-test + # (mgshape) is RED until this job needs it and keys its if on receipt-judge.outputs.ok. + workspace-test-ran: + needs: [workspace-test-shard] + if: ${{ !cancelled() && needs.workspace-test-shard.result == 'success' && github.event_name == 'pull_request' && github.event.pull_request.draft != true }} + runs-on: [self-hosted, Linux, clean-room] + timeout-minutes: 5 + steps: + - run: echo "this run owes and runs the test suite" workspace-test-shard: - if: ${{ github.event.pull_request.draft != true }} + # T43b: a merge_group entry whose tree equals a green PR run's skips the shards (mg-reuse-wt); anything else runs them + needs: [mg-reuse-wt] + if: ${{ !cancelled() && needs.mg-reuse-wt.outputs.wt != '1' && github.event.pull_request.draft != true }} # same lanes as x86-main: clean-room for pull_request/push, mq for a merge_group entry runs-on: ${{ github.event_name == 'merge_group' && fromJSON('["self-hosted","Linux","X64","mq"]') || fromJSON('["self-hosted","Linux","X64","clean-room"]') }} timeout-minutes: 180 @@ -528,22 +579,31 @@ jobs: workspace-test: runs-on: [self-hosted, Linux, clean-room] timeout-minutes: 10 - needs: [workspace-test-shard] + needs: [workspace-test-shard, mg-reuse-wt] if: always() steps: - - name: Every shard passed + - name: Every shard passed (or the whole set was reused from an identical-tree green PR run) env: SHARD_RESULT: ${{ needs.workspace-test-shard.result }} + WT_REUSE: ${{ needs.mg-reuse-wt.outputs.wt }} + WT_CITE: ${{ needs.mg-reuse-wt.outputs.wt_cite }} DRAFT: ${{ github.event.pull_request.draft }} run: | set -euo pipefail [ "${DRAFT:-}" != true ] || { echo "draft PR: the shards are not owed until ready_for_review"; exit 0; } - echo "workspace-test-shard matrix result: $SHARD_RESULT" + echo "workspace-test-shard matrix result: $SHARD_RESULT (reuse key wt='${WT_REUSE:-}')" + if [ "${WT_REUSE:-}" = 1 ]; then + # T43b: the shards are skipped ONLY under the key; a shard that ran anyway is a contradiction, never a pass + [ "$SHARD_RESULT" = skipped ] || { echo "::error::workspace-test: reuse key wt=1 but the shard matrix result is '$SHARD_RESULT'"; exit 1; } + [ -n "${WT_CITE:-}" ] || { echo "::error::workspace-test: reuse key wt=1 without a citation"; exit 1; } + echo "workspace-test reused: merge-group tree == tree of a green PR run (${WT_CITE})" + exit 0 + fi [ "$SHARD_RESULT" = success ] || { echo "::error::workspace-test: shard matrix result is '$SHARD_RESULT' (a cancelled or skipped shard is not a pass)"; exit 1; } - uses: actions/checkout@v7 if: ${{ github.event.pull_request.draft != true }} - uses: actions/download-artifact@v4 - if: ${{ github.event.pull_request.draft != true }} + if: ${{ needs.mg-reuse-wt.outputs.wt != '1' && github.event.pull_request.draft != true }} with: pattern: sigma-shard-* path: ${{ runner.temp }}/sigma @@ -552,7 +612,7 @@ jobs: # ids is compared with the universe shard 1 listed. A shard whose ids did # not arrive is RED, never a smaller universe. - name: "Σ-executed: the union of the shards is exactly the owed set (#4433)" - if: ${{ github.event.pull_request.draft != true }} + if: ${{ needs.mg-reuse-wt.outputs.wt != '1' && github.event.pull_request.draft != true }} run: | set -euo pipefail d="$RUNNER_TEMP/sigma" diff --git a/crates/aprender-core/tests/monorepo_invariants.rs b/crates/aprender-core/tests/monorepo_invariants.rs index 2579950bad..7ca5d9987d 100644 --- a/crates/aprender-core/tests/monorepo_invariants.rs +++ b/crates/aprender-core/tests/monorepo_invariants.rs @@ -1152,7 +1152,10 @@ fn draft_gate_canon() -> Vec<(&'static str, String)> { ("x86-main", format!("${{{{ !cancelled() && needs.mg-reuse.outputs.x86 != '1' && {DF} }}}}")), ("x86-main-advisories", format!("${{{{ !cancelled() && needs.mg-reuse.outputs.x86 == '1' && {DF} }}}}")), ("determinism", "${{ !cancelled() && needs.mg-reuse.outputs.det != '1' && github.event_name != 'pull_request' }}".into()), - ("workspace-test-shard", format!("${{{{ {DF} }}}}")), + // merge_group-only: a draft never reaches the queue, so there is nothing to skip. + ("mg-reuse-wt", "github.event_name == 'merge_group'".into()), + ("workspace-test-shard", format!("${{{{ !cancelled() && needs.mg-reuse-wt.outputs.wt != '1' && {DF} }}}}")), + ("workspace-test-ran", format!("${{{{ !cancelled() && needs.workspace-test-shard.result == 'success' && github.event_name == 'pull_request' && {DF} }}}}")), ("workspace-test", "always()".into()), ("ci-gate", "always()".into()), ("gate", "always()".into()), diff --git a/docs/roadmaps/entries/PMAT-5884.yaml b/docs/roadmaps/entries/PMAT-5884.yaml new file mode 100644 index 0000000000..100b0c6db1 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-5884.yaml @@ -0,0 +1,20 @@ +- id: PMAT-5884 + github_issue: 5884 + item_type: task + title: 'mg-reuse: merge_group skips workspace-test-shard when its tree == a green PR run''s tree (Noah 16:28Z #3)' + status: planned + priority: high + assigned_to: null + created: 2026-10-21T17:05:00Z + updated: 2026-10-21T17:05:00Z + spec: null + acceptance_criteria: + - 'merge_group skips workspace-test-shard only when K1-K5 hold and the PR head run workspace-test succeeded; any difference runs the full shards' + - 'ci_mg_reuse.sh --self-test falsifier: changed main/tree -> wt=0, identical tree -> wt=1, override carries wt only when exactly 1' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 277fbb376d..abbeef5d40 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -25436,6 +25436,26 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 5510 +- id: PMAT-5884 + github_issue: 5884 + item_type: task + title: 'mg-reuse: merge_group skips workspace-test-shard when its tree == a green PR run''s tree (Noah 16:28Z #3)' + status: planned + priority: high + assigned_to: null + created: 2026-10-21T17:05:00Z + updated: 2026-10-21T17:05:00Z + spec: null + acceptance_criteria: + - 'merge_group skips workspace-test-shard only when K1-K5 hold and the PR head run workspace-test succeeded; any difference runs the full shards' + - 'ci_mg_reuse.sh --self-test falsifier: changed main/tree -> wt=0, identical tree -> wt=1, override carries wt only when exactly 1' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 5510 - id: PMAT-5888 github_issue: 5888 item_type: task diff --git a/scripts/check_ci_draft_skip.sh b/scripts/check_ci_draft_skip.sh index 8db62fc5ac..60250eeb2e 100755 --- a/scripts/check_ci_draft_skip.sh +++ b/scripts/check_ci_draft_skip.sh @@ -48,7 +48,7 @@ check() { # FILE -> prints one FAIL line per violated condition; rc 1 when any fi done blk="$(job_block "$f" workspace-test)" - if [ "$(printf '%s\n' "$blk" | /usr/bin/grep -c 'if: ${{ github.event.pull_request.draft != true }}')" -lt 3 ]; then + if [ "$(printf '%s\n' "$blk" | /usr/bin/grep -c -E 'if: [$][{][{] (needs[.]mg-reuse-wt[.]outputs[.]wt != .1. && )?github[.]event[.]pull_request[.]draft != true [}][}]')" -lt 3 ]; then echo "FAIL workspace-test: checkout, download-artifact and sigma steps are not all skipped on a draft"; bad=1 fi return "$bad" diff --git a/scripts/check_ci_push_reuse.sh b/scripts/check_ci_push_reuse.sh index 0532c5d4e3..303a3e6544 100755 --- a/scripts/check_ci_push_reuse.sh +++ b/scripts/check_ci_push_reuse.sh @@ -137,10 +137,14 @@ OTHER=2222222222222222222222222222222222222222 QUEUE=gh-readonly-queue/main/pr-4700-0000000000000000000000000000000000000000 suite() { printf '{"id":%s,"head_branch":"%s","app":{"slug":"%s"}}' "$1" "$2" "${3:-github-actions}"; } crun() { printf '{"status":"%s","conclusion":%s,"head_sha":"%s","completed_at":"%s","check_suite":{"id":%s},"details_url":"https://github.com/o/r/actions/runs/%s/job/9"}' "$1" "$2" "$3" "$4" "$5" "$6"; } +sjobs() { # sjobs ... -> a jobs body whose workspace-test-shard (n, 4) jobs all succeeded (the queue run really tested) + local n; { for n in "$@"; do printf '{"name":"workspace-test-shard (%s, 4)","conclusion":"success"}\n' "$n"; done; printf '{"name":"gate","conclusion":"success"}\n'; } | jq -s '{jobs: .}' +} canned() { # canned - rm -f "${T:?}/canned/suites.FAIL" "${T:?}/canned/runs.FAIL" "${T:?}/canned/calls" + rm -f "${T:?}/canned/suites.FAIL" "${T:?}/canned/runs.FAIL" "${T:?}/canned/mgjobs.FAIL" "${T:?}/canned/calls" printf '{"check_suites":[%s]}' "$1" >"$T/canned/suites.json" printf '{"check_runs":[%s]}' "$2" >"$T/canned/runs.json" + sjobs 1 2 3 4 >"$T/canned/mgjobs.json" } look() { OUT="$(bash "$LOOKUP" o/r "${1:-$SHA}" 2>"$T/err")"; RC=$?; ERR="$(cat "$T/err")"; } flat() { tr '\n' ' ' <<<"$1" | sed 's/ $//'; } @@ -154,7 +158,7 @@ look want "L1 a merge-queue workspace-test success on the sha -> found, exit 0" 0 "$RC" want "L1 prints the run, the sha the API reports, the conclusion (the push suite's later failure is not the queue's)" \ "--mg-run 7001 --mg-sha $SHA --mg-conclusion success" "$(flat "$OUT")" -want "L1 costs exactly 2 API calls" 2 "$(wc -l <"$T/canned/calls" | tr -d ' ')" +want "L1 costs exactly 3 API calls (suites, check-runs, the run's jobs)" 3 "$(wc -l <"$T/canned/calls" | tr -d ' ')" canned "$(suite 50 "$QUEUE")" "$(crun completed '"failure"' $SHA 2026-10-04T10:00:00Z 50 7003)" look @@ -213,6 +217,36 @@ canned "$(suite 50 "$QUEUE")" "$(crun completed '"success"' abc 2026-10-04T10:00 look want "L13 the check run reports a head_sha that is not a sha -> nothing, exit 1" "1:" "$RC:$OUT" +# L14-L18: a queue run whose green `workspace-test` did not run its own shards is a citation, not a measurement (finding: transitive reuse) +canned "$(suite 50 "$QUEUE")" "$(crun completed '"success"' $SHA 2026-10-04T10:00:00Z 50 7015)" +sjobs 1 2 | jq '.jobs += [{"name":"workspace-test-shard (3, 4)","conclusion":"skipped"},{"name":"workspace-test-shard (4, 4)","conclusion":"skipped"}]' >"$T/canned/mgjobs.json" +look +want "L14 the queue run skipped its shards (it cited a PR run) -> nothing, exit 1" "1:" "$RC:$OUT" +has "L14 says why" "did not run and pass its own workspace-test shards" "$ERR" + +canned "$(suite 50 "$QUEUE")" "$(crun completed '"success"' $SHA 2026-10-04T10:00:00Z 50 7016)" +sjobs 1 2 3 | jq '.jobs += [{"name":"workspace-test-shard (4, 4)","conclusion":"failure"}]' >"$T/canned/mgjobs.json" +look +want "L15 one shard failed under a green verdict -> nothing, exit 1" "1:" "$RC:$OUT" + +canned "$(suite 50 "$QUEUE")" "$(crun completed '"success"' $SHA 2026-10-04T10:00:00Z 50 7017)" +sjobs >"$T/canned/mgjobs.json" +look +want "L16 the queue run has no shard job at all -> nothing, exit 1" "1:" "$RC:$OUT" + +canned "$(suite 50 "$QUEUE")" "$(crun completed '"success"' $SHA 2026-10-04T10:00:00Z 50 7018)" +: >"$T/canned/mgjobs.FAIL" +look +want "L17 the jobs call fails -> nothing, exit 1" "1:" "$RC:$OUT" +has "L17 names the failed call" "jobs lookup failed" "$ERR" + +canned "$(suite 50 "$QUEUE")" "$(crun completed '"failure"' $SHA 2026-10-04T10:00:00Z 50 7019)" +sjobs >"$T/canned/mgjobs.json" +look +want "L18 a failed verdict is still printed without a jobs call (only a success is checked for citation)" \ + "--mg-run 7019 --mg-sha $SHA --mg-conclusion failure" "$(flat "$OUT")" +want "L18 and it cost 2 API calls" 2 "$(wc -l <"$T/canned/calls" | tr -d ' ')" + # -- D: the decision ------------------------------------------------------------------------------- mkrepo() { # mkrepo -> a two-commit repo whose push diff is docs-only (today's tier: none, no cargo) git init -q -b main "$1" diff --git a/scripts/ci_mg_reuse.sh b/scripts/ci_mg_reuse.sh index 10f010aa3c..45f685ac24 100755 --- a/scripts/ci_mg_reuse.sh +++ b/scripts/ci_mg_reuse.sh @@ -49,7 +49,7 @@ set -uo pipefail CI_WORKFLOW_PATH=".github/workflows/ci.yml" PRH="" -emit() { printf 'x86=%s\ndet=%s\nmac=%s\ncite=%s\npr_head=%s\nreason=%s\n' "$1" "$2" "$3" "$4" "$5" "$6"; } +emit() { printf 'x86=%s\ndet=%s\nmac=%s\ncite=%s\npr_head=%s\nreason=%s\nwt=%s\nwt_cite=%s\n' "$1" "$2" "$3" "$4" "$5" "$6" "${7:-0}" "${8:-}"; } refuse() { emit 0 0 0 "" "$PRH" "$1"; } usage() { echo "usage: ci_mg_reuse.sh decide|resolve|decide-push|resolve-push [opts] | --self-test [--workflow FILE]" >&2; exit 2; } @@ -67,6 +67,14 @@ concl() { | if length == 1 then (.[0].conclusion // "none") else "count=\(length)" end' "$1" 2> /dev/null } +# The matrix jobs `workspace-test-shard (N, M)` in jobs file $1: prints " ", M = the shard total every +# name agrees on (0 if they disagree). A draft PR run (or a reused one) has green `workspace-test` and NO such jobs (#5885 lane-b). +shards() { + jq -r '[.jobs[]? | select(.name | test("^workspace-test-shard \\([0-9]+, [0-9]+\\)$"))] as $s + | ($s | map(.name | capture(", (?[0-9]+)\\)$").m | tonumber) | unique) as $m + | "\($s | map(.name) | unique | length) \($s | map(select(.conclusion != "success")) | length) \(if ($m | length) == 1 then ($m | first) else 0 end)"' "$1" 2> /dev/null || echo "0 1 0" +} + decide() { local ev="" m="" b="" h="" runs="" jobs="" root="." while [ $# -gt 0 ]; do @@ -112,17 +120,39 @@ decide() { [ "$(concl "$jobs" 'ci / gate')" = success ] || why="${why:+$why; }ci / gate is $(concl "$jobs" 'ci / gate')" # R:k6-cigate x=1 [ -z "$why" ] || x=0 + # T43b (#5884, operator 16:28Z #3): workspace-test is its own unit. K1-K5 held, so the queue entry's tree IS the tree + # the head's run tested; reuse that run's `workspace-test` verdict (it is the Σ-executed fan-in over every shard). + # Not tied to x86-main/determinism: a red x86-main on the head does not make a green workspace-test stale. + wt=0 + # wt=1 needs the verdict green AND >=1 shard job, all success: a draft run is green with zero shards (#5885 lane-b). + read -r nsh nbad msh <<< "$(shards "$jobs")" + wcite="run $rid attempt $att on ${h:0:10}" + # ...and EVERY shard of the matrix is present: (N, M) names must agree on M and number M (a partial list is not a run). + [ "$(concl "$jobs" workspace-test)" = success ] && [ "${msh:-0}" -ge 1 ] 2> /dev/null && [ "${nsh:-0}" = "${msh:-0}" ] && [ "${nbad:-1}" = 0 ] && wt=1 # R:k6-wt-all-shards + # ...and the run owed and ran a test suite: a receipt-reuse PR run (#5850 tier=reuse) skips the test step, so its 12/12 shards are green over ZERO tests; + # `workspace-test-ran` runs only when the tier is not reuse, so only a run that tested the tree may be cited. + [ "$(concl "$jobs" workspace-test-ran)" = success ] || wt=0 # R:k6-wt-ran + # a reuse with nothing to cite is not a reuse + [[ "$wcite" =~ ^run\ [0-9]+\ attempt\ [0-9]+\ on\ [0-9a-f]{10}$ ]] || wt=0 # R:k6-wt-cite mc=1 [ "$(concl "$jobs" mac-check)" = success ] || mc=0 # R:k6-mac emit "$x" "$x" "$mc" "run $rid attempt $att on ${h:0:10}" "$h" \ - "K1-K5 hold; x86-main+determinism: $([ "$x" = 1 ] && echo reuse || echo "run ($why)"); mac-check: $([ "$mc" = 1 ] && echo reuse || echo "run ($(concl "$jobs" mac-check))")" + "K1-K5 hold; x86-main+determinism: $([ "$x" = 1 ] && echo reuse || echo "run ($why)"); mac-check: $([ "$mc" = 1 ] && echo reuse || echo "run ($(concl "$jobs" mac-check))"); workspace-test: $([ "$wt" = 1 ] && echo reuse || echo "run ($(concl "$jobs" workspace-test))")" \ + "$wt" "$([ "$wt" = 1 ] && echo "$wcite")" } # The queue-entry override, FORCED: drop whatever decide said for these keys (or omitted) and write # the values explicitly, so a decide that leaves a key out cannot fail open (r6). force_no_reuse() { - { grep -v -E '^(x86|det|mac|cite|pr_head)=' || true; } - printf 'x86=0\ndet=0\nmac=0\ncite=\npr_head=\n' + local in wt=0 wc="" + in=$(cat) + # T43b (#5884): workspace-test reuse crosses the override ONLY as exactly wt=1 with its citation; anything else is 0. + if [ "$(printf '%s\n' "$in" | sed -n 's/^wt=//p' | tail -n 1)" = 1 ]; then + wt=1 + wc=$(printf '%s\n' "$in" | sed -n 's/^wt_cite=//p' | tail -n 1) + fi + { printf '%s\n' "$in" | grep -v -E '^(x86|det|mac|cite|pr_head|wt|wt_cite)=' || true; } + printf 'x86=0\ndet=0\nmac=0\ncite=\npr_head=\nwt=%s\nwt_cite=%s\n' "$wt" "$wc" } resolve() { @@ -268,10 +298,11 @@ resolve_push() { } # ---------------------------------------------------------------- case table -# The jobs of run 37040341824, #4655's only PR-head run (db86acf021), verbatim -# from the API (two GPU-host leg job names replaced; the key never reads them): -# x86-main failed while gate and ci / gate passed. -PLANTED_4655_JOBS=$'gpu-touched\tsuccess\nmutants-table-scope\tsuccess\ndeterminism\tsuccess\nworkspace-test-shard (1)\tsuccess\nmac-check\tsuccess\nworkspace-test-shard (3)\tsuccess\nworkspace-test-shard (2)\tsuccess\nx86-main\tfailure\nmutants-table\tskipped\nmutants-shard\tskipped\ngpu-host-leg-a\tsuccess\ngpu-host-leg-b\tsuccess\nworkspace-test\tsuccess\nci / gate\tsuccess\ngate\tsuccess' +# The jobs of run 37040341824, #4655's only PR-head run (db86acf021), from the API: two GPU-host leg job names +# replaced (the key never reads them), and the workspace-test-shard names rewritten from the run's old names to the +# live matrix form "workspace-test-shard (N, M)" (MG run 39666213701), so this is NOT verbatim. The verdict shape is +# the planted one: x86-main failed while gate and ci / gate passed. +PLANTED_4655_JOBS=$'gpu-touched\tsuccess\nmutants-table-scope\tsuccess\ndeterminism\tsuccess\nworkspace-test-shard (1, 4)\tsuccess\nmac-check\tsuccess\nworkspace-test-shard (3, 4)\tsuccess\nworkspace-test-shard (2, 4)\tsuccess\nx86-main\tfailure\nmutants-table\tskipped\nmutants-shard\tskipped\ngpu-host-leg-a\tsuccess\ngpu-host-leg-b\tsuccess\nworkspace-test\tsuccess\nci / gate\tsuccess\ngate\tsuccess' GREEN_JOBS=$'x86-main\tsuccess\ndeterminism\tsuccess\nmac-check\tsuccess\nworkspace-test\tsuccess\nci / gate\tsuccess\ngate\tsuccess' # jobs_json HEAD RUN_ID EXTRA_TOTAL < "nameconclusion" lines @@ -382,6 +413,57 @@ self_test() { dr k6-x86-main-twice "0 0 1" merge_group "$M" "$B" "$H" runs.ok jobs.dup-x86 dr k6-mac-failed "1 1 0" merge_group "$M" "$B" "$H" runs.ok jobs.fail-mac-check + # T43b (#5884, Noah 16:28Z #3): workspace-test reuse. RED->GREEN falsifier: a changed main must run, an identical tree may skip. + sed '/^workspace-test /d' <<< "$GREEN_JOBS" | jobs_json "$H" 1001 0 > "$F/jobs.no-wt" + # shard job names are the real matrix names, "workspace-test-shard (N, M)" (MG run 39666213701); sh4 N... = those N of 4, success + sh4() { local n; for n in "$@"; do printf 'workspace-test-shard (%s, 4)\tsuccess\n' "$n"; done; } + ran() { printf 'workspace-test-ran\t%s\n' "$1"; } + { cat <<< "$GREEN_JOBS"; sh4 1 2 3 4; ran success; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-ok" + # the same fully-tested jobs on the other head, so a row whose reason is the K1-K5 key reaches wt=0 for that reason alone + { cat <<< "$GREEN_JOBS"; sh4 1 2 3 4; ran success; } | jobs_json "$H2" 1001 0 > "$F/jobs.wt-ok-h2" + { cat <<< "$GREEN_JOBS"; sh4 1 2 3 4; ran skipped; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-receipt" + { cat <<< "$GREEN_JOBS"; sh4 1 2 3 4; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-noran" + sed "s|^determinism\tsuccess\$|determinism\tfailure|" <<< "$GREEN_JOBS" | { cat; sh4 1 2 3 4; ran success; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-x86fail" + { cat <<< "$GREEN_JOBS"; sh4 1 3 4; printf 'workspace-test-shard (2, 4)\tfailure\n'; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-shard-failed" + { cat <<< "$GREEN_JOBS"; sh4 1 2 3; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-partial" + { cat <<< "$GREEN_JOBS"; sh4 1 2 3 4; printf 'workspace-test-shard (5, 12)\tsuccess\n'; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-mixed-m" + run_obj 1001 "$H" completed | jq '.run_attempt = ""' | runs_json 0 > "$F/runs.no-attempt" + { cat <<< "$GREEN_JOBS"; printf 'workspace-test-shard\tskipped\n'; } | jobs_json "$H" 1001 0 > "$F/jobs.wt-draft" + sed 's|^workspace-test success$|workspace-test failure|' <<< "$GREEN_JOBS" | jobs_json "$H" 1001 0 > "$F/jobs.fail-wt" + wrow() { # wrow NAME WANT decide-args... + local n="$1" want="$2" out; shift 2 + out=$(decide "$@" --repo-root "$R") + if [ "$(kv wt "$out")" = "$want" ]; then pass=$((pass + 1)); else + fail=$((fail + 1)); echo "FAIL wt-$n: want wt=$want got '$(kv wt "$out")' -- $(kv reason "$out")"; fi + } + wd() { local n="$1" want="$2" m="$3" b="$4" h="$5" r="$6" j="$7" + wrow "$n" "$want" --event merge_group --queue-head "$m" --base "$b" --pr-head "$h" --runs "$F/$r" --jobs "$F/$j"; } + wd identical-tree-skips 1 "$M" "$B" "$H" runs.ok jobs.wt-ok + wd draft-run-no-shards-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-draft + wd receipt-reuse-12of12-green-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-receipt + wd ran-job-missing-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-noran + wd greenverdict-zero-shards 0 "$M" "$B" "$H" runs.ok jobs.ok + wd one-shard-failed-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-shard-failed + wd partial-shards-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-partial + wd mixed-shard-totals-runs 0 "$M" "$B" "$H" runs.ok jobs.wt-mixed-m + wd empty-attempt-no-cite-runs 0 "$M" "$B" "$H" runs.no-attempt jobs.wt-ok + wd changed-tree-runs 0 "$M_TREE" "$B" "$H" runs.ok jobs.wt-ok + wd changed-main-not-ancestor 0 "$M_K3" "$B2" "$H" runs.ok jobs.wt-ok + wd base-is-a-merge-runs 0 "$M_K2" "$BM" "$H2" runs.h2 jobs.wt-ok-h2 + wd head-wt-failed-runs 0 "$M" "$B" "$H" runs.ok jobs.fail-wt + wd head-wt-missing-runs 0 "$M" "$B" "$H" runs.ok jobs.no-wt + wd run-in-progress-runs 0 "$M" "$B" "$H" runs.newer-running jobs.wt-ok + wd head-x86-failed-still-skips 1 "$M" "$B" "$H" runs.ok jobs.wt-x86fail + wrow push-never-skips 0 --event push --queue-head "$M" --base "$B" --pr-head "$H" --runs "$F/runs.ok" --jobs "$F/jobs.ok" + # the override passes wt only as exactly 1, with its citation + local wo + wo=$(force_no_reuse <<< $'wt=1\nwt_cite=run 7\nx86=1') + [ "$(kv wt "$wo")|$(kv wt_cite "$wo")|$(kv x86 "$wo")" = "1|run 7|0" ] && pass=$((pass + 1)) || { fail=$((fail + 1)); echo "FAIL wt-override-pass: $wo"; } + for wo in $'wt=2' $'wt=' $'wt=0\nwt_cite=run 7' $'reason=x'; do + wo=$(force_no_reuse <<< "$wo") + [ "$(kv wt "$wo")|$(kv wt_cite "$wo")" = "0|" ] && pass=$((pass + 1)) || { fail=$((fail + 1)); echo "FAIL wt-override-closed: $wo"; } + done + # resolve, end to end, through a stub gh serving canned JSON. A failed lookup never reuses. local SB="$T/bin"; mkdir -p "$SB" cat > "$SB/gh" <<'STUB' @@ -564,12 +646,21 @@ STUB awk -v tok='(^|[^A-Za-z0-9_-])mg-reuse($|[^A-Za-z0-9_-])' -v want="if: github.event_name == 'push' && github.ref == 'refs/heads/main'" ' /^ [A-Za-z0-9_-]+:$/ { j = $1; sub(/:$/, "", j); seen[j] = 1; next } /^ if:/ { jif[j] = $0; sub(/^ /, "", jif[j]) } + /^ runs-on:/ { jrun[j] = $0 } + /^ needs:/ || (inn && /^ /) { jneeds[j] = jneeds[j] $0 } /^ needs:/ { inn = 1; if ($0 ~ tok) dep[j] = 1; next } inn && /^ / { if ($0 ~ tok) dep[j] = 1; next } /^ [A-Za-z]/ { inn = 0 } END { if (jif["mg-reuse"] != want) print "mg-reuse job-level if is not the push-only key: [" jif["mg-reuse"] "]" if (jif["mg-reuse"] ~ /merge_group/) print "mg-reuse if names merge_group: a queue entry could reuse" + # once the receipt judge exists, the test-ran marker must be keyed on its verdict, else a receipt run reads as tested + if (("receipt-judge" in seen) && (jneeds["workspace-test-ran"] !~ /receipt-judge/ || jif["workspace-test-ran"] !~ /receipt-judge\.outputs\.ok/)) print "workspace-test-ran ignores receipt-judge: a receipt-reuse run would read as tested" + # the marker must prove the shards ran: it needs them and its if requires their success (else a run with skipped shards owns it) + if (jneeds["workspace-test-ran"] !~ /workspace-test-shard/) print "workspace-test-ran does not need workspace-test-shard: the marker no longer proves the shards ran" + if (jif["workspace-test-ran"] !~ /needs\.workspace-test-shard\.result == .success./) print "workspace-test-ran if lacks the shard success clause: skipped or failed shards would still own the marker" + # the reuse lookup gates every queue entry shard, so it shares their lane (mq), never the PR clean-room pool + if (jrun["mg-reuse-wt"] !~ /"mq"/) print "mg-reuse-wt is not on the mq lane: it would queue behind PR work ahead of every shard" n = split("x86-main determinism x86-main-advisories ci-gate gate", must, " ") for (i = 1; i <= n; i++) { if (!(must[i] in seen)) print "job " must[i] " is missing from the workflow" @@ -597,6 +688,12 @@ STUB mgmut x86-main-needs-lookalike "job x86-main no longer needs mg-reuse" '/^ x86-main:$/,/^ if:/{s/^ needs: \[mg-reuse\]$/ needs: [mg-reuse-self-test]/}' mgmut x86-main-drops-needs "job x86-main no longer needs mg-reuse" '/^ x86-main:$/,/^ if:/{/^ needs: \[mg-reuse\]$/d}' mgmut gate-if-loses-always "job gate needs mg-reuse but its if lacks" "/^ gate:\$/,/^ steps:/{s/^ if: always\(\)/ if: success()/}" + # once a receipt-judge job exists (#5850), the marker must need it and key its if on the verdict + mgmut ran-marker-ignores-receipt-judge "workspace-test-ran ignores receipt-judge" 's#^ workspace-test-ran:$# receipt-judge:\n runs-on: x\n workspace-test-ran:#' + mgmut ran-marker-needs-judge-but-if-ignores-it "workspace-test-ran ignores receipt-judge" 's#^ workspace-test-ran:$# receipt-judge:\n runs-on: x\n workspace-test-ran:\n needs: [receipt-judge]#' + mgmut ran-marker-drops-shard-needs "workspace-test-ran does not need workspace-test-shard" '/^ workspace-test-ran:$/,/^ runs-on:/{/^ needs: \[workspace-test-shard\]$/d}' + mgmut ran-marker-if-drops-shard-result "workspace-test-ran if lacks the shard success clause" "/^ workspace-test-ran:\$/,/^ runs-on:/{s/ \&\& needs.workspace-test-shard.result == 'success'//}" + mgmut mg-reuse-wt-leaves-mq-lane "mg-reuse-wt is not on the mq lane" '/^ mg-reuse-wt:$/,/^ timeout-minutes:/{s/,"mq"\]/,"clean-room"]/}' # Positive control for wire itself: one job of more than 64 KiB (a pipe buffer) whose # wanted line comes first. A pipe from job into grep -q takes SIGPIPE on this every diff --git a/scripts/ci_mg_workspace_result.sh b/scripts/ci_mg_workspace_result.sh index 2493b96e70..844d7c6bd0 100644 --- a/scripts/ci_mg_workspace_result.sh +++ b/scripts/ci_mg_workspace_result.sh @@ -13,7 +13,10 @@ # Actions suite of a merge-queue branch (gh-readonly-queue/...). It reports what it found, a # failure included; judging it is ci_test_tier.sh's job. # -# Exactly two API calls, both on the checks endpoints (the ones the merge_group arm already reads). +# Three API calls: two on the checks endpoints (the ones the merge_group arm already reads) and, for a success, one +# on the found run's jobs. A queue run that SKIPPED its workspace-test shards (it cited a PR run, mg-reuse-wt #5884) +# also reports a green `workspace-test`; reusing that on the push would cite a citation, so the lookup refuses +# unless every `workspace-test-shard (N, M)` job of that run succeeded. # Anything that is not a clean answer prints NOTHING on stdout, one reason on stderr, and exits 1: # the caller then decides the tier from the push's own diff, as before T36. # @@ -50,4 +53,14 @@ done <<<"$runs" read -r hsha concl url <<<"$found" [[ $hsha =~ ^[0-9a-f]{40}$ ]] || no "the workspace-test check run reports no sha" # R-HEADSHA [[ $url =~ /actions/runs/([0-9]+)(/|$) ]] || no "the workspace-test details_url names no run id: $url" # R-RUNID -printf -- '--mg-run\n%s\n--mg-sha\n%s\n--mg-conclusion\n%s\n' "${BASH_REMATCH[1]}" "$hsha" "$concl" +rid=${BASH_REMATCH[1]} + +# Call 3 (a success only): the shards of that run. A skipped, failed or absent shard means the green +# `workspace-test` was a citation of another run (or tested nothing), never this tree's own measurement. +if [ "$concl" = success ]; then + jobs=$(gh api "repos/$repo/actions/runs/$rid/jobs?per_page=100&filter=latest"); rc=$? + [ "$rc" -eq 0 ] || no "the jobs lookup failed for run $rid" # R-JOBSAPI + shards=$(jq -r '[.jobs[]? | select(.name | test("^workspace-test-shard \\([0-9]+, [0-9]+\\)$"))] | if length == 0 then "none" elif all(.conclusion == "success") then "ok" else "no" end' <<<"$jobs" 2>/dev/null) + [ "$shards" = ok ] || no "run $rid did not run and pass its own workspace-test shards (${shards:-unreadable}): its green verdict may be a citation" # R-SHARDS +fi +printf -- '--mg-run\n%s\n--mg-sha\n%s\n--mg-conclusion\n%s\n' "$rid" "$hsha" "$concl"