From aec00654083fc95282253d9a22152efb93599cb9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 10:44:22 +0200 Subject: [PATCH 01/14] fix(mutants-nightly): read the workspace-test excludes from ci/sections.yml, where the line now lives The nightly derived its --exclude-crate list by grepping ci.yml for the workspace-test nextest line. #4655 moved that line into ci/sections.yml (ci.yml's PR shard already reads it there), so the grep matched nothing and the step failed closed every night: run 37183004032, 'no --exclude list found on the workspace-test nextest line'. Same derivation, same fail-closed check, same exclusions (aprender-gpu, aprender-cuda-edge, aprender-compute); only the file read changes. Proof: old path -> empty, new path -> the three crates. Refs #4681 Agent: aprender-wmainred Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5dccaa4bf62816e0fb3cd81175c638a2b908f972) --- .github/workflows/mutants-nightly.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mutants-nightly.yml b/.github/workflows/mutants-nightly.yml index a5d12a8c98..2c9e251686 100644 --- a/.github/workflows/mutants-nightly.yml +++ b/.github/workflows/mutants-nightly.yml @@ -104,8 +104,9 @@ jobs: } fi cargo mutants --version || { echo "::error::cargo-mutants not runnable after install"; exit 1; } - # the crates ci.yml workspace-test --excludes, derived from its nextest line (see ci.yml mutants) - EXCL=$(grep -m1 -E "^[[:space:]]+cargo nextest run --profile ci --workspace --lib" .github/workflows/ci.yml \ + # the crates workspace-test --excludes, derived from its nextest line. That line moved from + # ci.yml into ci/sections.yml (#4655); reading the old file found nothing and failed every night. + EXCL=$(grep -m1 -E "^[[:space:]]+cargo nextest run --profile ci --workspace --lib" ci/sections.yml \ | grep -oE -- "--exclude [a-z0-9_-]+" | sed "s/^--exclude /--exclude-crate /" | tr "\n" " ") [ -n "$EXCL" ] || { echo "::error::no --exclude list found on the workspace-test nextest line"; exit 1; } # shellcheck disable=SC2086 From e6225f28c5bfcff90b01603c2de7e06201ae0eeb Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 10:45:15 +0200 Subject: [PATCH 02/14] fix(nightly): darwin leg built into the mini's redirected target dir, so --version found no apr The mini runner's env sets CARGO_TARGET_DIR=~/data/cargo-target. cargo built there and the next line, "target/$T/release/apr" --version, exited 127 (run 37171118510, job 'apr aarch64-apple-darwin on mini'). Package reads the same ./target path. Step-local export of CARGO_TARGET_DIR=$GITHUB_WORKSPACE/target, the fix #4660 already made for binary-release.yml's mini lane. No check changes. Refs #4681 Agent: aprender-wmainred Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 25ce0227a19418d14d120bb03a8e948f8eefb5ac) --- .github/workflows/nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1f34b0e156..69d79c63ee 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -259,6 +259,11 @@ jobs: - name: Build apr (host-native, pinned toolchain) run: | set -euo pipefail + # Every later step reads ./target. The mini's runner env redirects cargo's + # target dir (CARGO_TARGET_DIR=~/data/cargo-target), so cargo built + # elsewhere and the --version line found no apr (run 37171118510, exit + # 127). Same fix as binary-release.yml's mini lane (#4660). + export CARGO_TARGET_DIR="$GITHUB_WORKSPACE/target" rustup show active-toolchain cargo build --release --locked -p apr-cli --target "$T" "target/$T/release/apr" --version From c356ec874c65803fd57906033af5574fe0cf4bd9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 10:46:43 +0200 Subject: [PATCH 03/14] fix(nightly): publish read one page of release assets; staged leftovers past it made every upload 422 The nightly release holds 234 assets, 117 of them staged.* leftovers from runs that died mid-publish. publish() listed assets with per_page=100 and no paging, so it saw 100. The leftovers past page one were never deleted, and staging the same name again failed HTTP 422 already_exists (run 37171118510, job publish). Since the cleanup can't see them, the failure repeats every night. release_assets() reads every page. Two self-test rows: 234 over 3 pages, and an exact multiple of 100. Mutant 'page one only' fails both rows. Local complexity hook skipped under C205(3) as extended: hook errors on scripts/nightly_manifest.py 4 before -> 4 after (publish cognitive 192 -> 181; self_test cyclomatic 30 = base). SATD 4 <= 5 and the docs check were run by hand, green. Logged in controls-ledger.md. Refs #4681 Agent: aprender-wmainred Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 99ac4c10b183e043815fb2d8cf51db3a6825bec3) --- scripts/nightly_manifest.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/nightly_manifest.py b/scripts/nightly_manifest.py index 206b97fce7..05c06cd215 100755 --- a/scripts/nightly_manifest.py +++ b/scripts/nightly_manifest.py @@ -327,6 +327,19 @@ def publish_plan(manifest, sha, dist_files): return names + [MANIFEST_ASSET] +def release_assets(A, rel_id, token, get=api): + """{name: id} for EVERY asset on the release, all pages. GitHub caps a page at 100; + page one alone missed 134 of 234 assets, so the staged leftovers past it were never + deleted and the next upload of the same name died 422 already_exists (run 37171118510).""" + out, page = {}, 1 + while True: + batch = get("GET", f"{A}/releases/{rel_id}/assets?per_page=100&page={page}", token) + out.update({a["name"]: a["id"] for a in batch}) + if len(batch) < 100: + return out + page += 1 + + def publish(manifest_path, dist, sha, repo, token): with open(manifest_path) as f: man = json.load(f) @@ -349,7 +362,7 @@ def publish(manifest_path, dist, sha, repo, token): rel = api("POST", f"{A}/releases", token, json.dumps({ "tag_name": "nightly", "target_commitish": sha, "name": "Nightly Build", "prerelease": True, "make_latest": "false"}).encode()) - existing = {a["name"]: a["id"] for a in api("GET", f"{A}/releases/{rel['id']}/assets?per_page=100", token)} + existing = release_assets(A, rel["id"], token) for n, i in existing.items(): # a staged upload an earlier run died holding if n.startswith(STAGED): api("DELETE", f"{A}/releases/assets/{i}", token) @@ -379,6 +392,18 @@ def publish(manifest_path, dist, sha, repo, token): # ---------------------------------------------------------------- self-test +def _self_test_release_assets(check): + print("release assets:") + rows = [{"name": f"staged.a{i}" if i % 2 else f"a{i}", "id": i} for i in range(234)] + + def pages(of): + return lambda m, url, tok: of[(int(url.rsplit("page=", 1)[1]) - 1) * 100:][:100] + got = release_assets("A", 1, None, pages(rows)) + check("every page is read: 234 assets over 3 pages, all 117 staged leftovers seen", + (len(got), sum(n.startswith(STAGED) for n in got)), (234, 117)) + check("an exact multiple of 100 ends on the empty page", len(release_assets("A", 1, None, pages(rows[:200]))), 200) + + def self_test(): fails = [] @@ -608,6 +633,7 @@ def pkg(name, bins, root=False, feats=None): except ValueError: got = "error" check("a manifest naming an absent asset refuses to publish", got, "error") + _self_test_release_assets(check) print(f"\n{'PASS' if not fails else 'FAIL'}: {len(fails)} failing row(s)") return 1 if fails else 0 From e795817d843aba10148593b9f32e39cc075ace4a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 10:48:03 +0200 Subject: [PATCH 04/14] =?UTF-8?q?fix(ci):=20conleche-nightly=20cache=20pat?= =?UTF-8?q?h=20is=20lambda-only=20=E2=80=94=20mkdir=20denied=20on=20clean-?= =?UTF-8?q?room,=20job=20never=20ran?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI_CACHE was hard-coded to /mnt/nvme-raid0/ci-cache/pvl-conleche. The job runs on [self-hosted, Linux, X64, clean-room], which lands on intel, where /mnt/nvme-raid0 is root-owned: `mkdir: cannot create directory '/mnt/nvme-raid0/ci-cache': Permission denied` (runs 37082537768, 37163229207). The workflow has never gone green since #4655. The cache now lives beside the runner's _work dir and is exported via GITHUB_ENV, so the con-leche step's PVL_CONLECHE_CACHE="$CI_CACHE/pins" is unchanged. No check changes. Refs #4681 Agent: aprender-wmainred Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 344d386fec396e2910c4ca3726bd7f373da00bdd) --- .github/workflows/conleche-nightly.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conleche-nightly.yml b/.github/workflows/conleche-nightly.yml index 63285936a7..767e167d96 100644 --- a/.github/workflows/conleche-nightly.yml +++ b/.github/workflows/conleche-nightly.yml @@ -42,7 +42,6 @@ jobs: # 2.6 min) -> T = max(15, 68, 65) = 68, rounded up for a shared host. The first three runs replace this basis. timeout-minutes: 90 env: - CI_CACHE: /mnt/nvme-raid0/ci-cache/pvl-conleche ELAN_VERSION: v4.2.4 ELAN_SHA256: 42b94d4244e8353142c456ec0e4ca6528fd898a6c604d4059f494e706e431f63 LEAN_NUM_THREADS: '8' @@ -55,6 +54,11 @@ jobs: - name: Pinned elan in the job's own directory (sha256-checked, never a host install) run: | set -euo pipefail + # The cache lives beside the runner's _work dir, as nightly.yml's cache-nightly does. The old + # /mnt/nvme-raid0/ci-cache path exists only on lambda; on the clean-room hosts /mnt/nvme-raid0 is + # root-owned, so mkdir failed and the job never ran (runs 37082537768, 37163229207). + CI_CACHE="$(cd "$GITHUB_WORKSPACE/../.." && pwd)/cache-conleche" + echo "CI_CACHE=$CI_CACHE" >> "$GITHUB_ENV" export ELAN_HOME="$CI_CACHE/elan" mkdir -p "$CI_CACHE" if [ ! -x "$ELAN_HOME/bin/elan" ] || ! "$ELAN_HOME/bin/elan" --version | grep -q "${ELAN_VERSION#v}"; then From c6336fffd741e76681889a21e62640b43cb6592b Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 4 Oct 2026 10:59:32 +0200 Subject: [PATCH 05/14] =?UTF-8?q?fix(ci):=20CUDA=20Nightly=20G0=20step=20a?= =?UTF-8?q?borted=20on=20its=20own=20RED=20control=20=E2=80=94=20bash=20-e?= =?UTF-8?q?=20killed=20it=20before=20r=3D$=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The G0 step (aprender#4250) runs under GitHub's default `bash -e {0}`. Its `set -uo pipefail` does not turn -e off, so the per-token RED control's expected exit 101 aborted the step at `; r=$?`, before the assertions ran. Run 37161229337: the GREEN arm passed, then the step failed 101 with no ::error:: line. Turn -e off with `set +e`, as the perf041 step in this workflow already does. Every failure path is guarded explicitly (`|| exit 1` and the GREEN/RED checks), so the step accepts exactly what it was written to accept. Repro: bash -e -c 'set -uo pipefail; (exit 101); r=$?; echo reached' prints nothing, rc=101. Refs #4681 Agent: aprender-wmainred Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0ae57c033f518bc1e0cea034ac121fa2c9210393) --- .github/workflows/cuda-nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cuda-nightly.yml b/.github/workflows/cuda-nightly.yml index 53dafd909d..c33e72b56d 100644 --- a/.github/workflows/cuda-nightly.yml +++ b/.github/workflows/cuda-nightly.yml @@ -707,6 +707,11 @@ jobs: if: ${{ !cancelled() && steps.decide.outputs.proceed == 'true' }} run: | set -uo pipefail + # -e OFF, as in the perf041 step: the default shell is `bash -e {0}`, so the + # RED control's expected exit 101 aborted the step at `; r=$?` before its + # assertions ran (run 37161229337: GREEN passed, step failed 101, no ::error::). + # Every failure path below is guarded explicitly. + set +e t=api::openai_handlers::qwen35_serve_tests::gpu_every_serve_route_prefills_through_the_batched_prefill model="$HOME/models/Qwen3.5-0.8B-Q4_K_M.gguf" [ -f "$model" ] || { echo "::error::#4250 UNMEASURABLE on ${{ matrix.name }} (provisioning): $model is absent"; exit 1; } From 2bbf15664890e3495a9dc07f75cee8c4a1200a5c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 08:21:59 +0200 Subject: [PATCH 06/14] chore(roadmap): PMAT-4715 + GH-4718 fragments, one aggregate regen for the B2 fold The five B2 nightly fixes above carried the same PMAT-4715 fragment each; it lands once here. GH-4718's fragment moves here too, so the queue holds one roadmap.yaml regen. make roadmap-aggregate-check: ok, 299 fragments, idempotent. Pmat-Ticket: PMAT-4715 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- docs/roadmaps/entries/GH-4718.yaml | 17 +++++++++++++ docs/roadmaps/entries/PMAT-4715.yaml | 19 +++++++++++++++ docs/roadmaps/roadmap.yaml | 36 ++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 docs/roadmaps/entries/GH-4718.yaml create mode 100644 docs/roadmaps/entries/PMAT-4715.yaml diff --git a/docs/roadmaps/entries/GH-4718.yaml b/docs/roadmaps/entries/GH-4718.yaml new file mode 100644 index 0000000000..715d659fbd --- /dev/null +++ b/docs/roadmaps/entries/GH-4718.yaml @@ -0,0 +1,17 @@ +- id: GH-4718 + github_issue: 4718 + item_type: task + title: '0702-P1: nightly producers for the deep lanes (deep-doctests, deep-nodefault, deep-bins-smoke)' + status: inprogress + priority: critical + assigned_to: null + created: 2026-10-04T09:17:34.904751550+00:00 + updated: 2026-10-04T09:17:34.904751550+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + notes: null diff --git a/docs/roadmaps/entries/PMAT-4715.yaml b/docs/roadmaps/entries/PMAT-4715.yaml new file mode 100644 index 0000000000..398c99140b --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4715.yaml @@ -0,0 +1,19 @@ +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 846e9565f3..dd03ac0a42 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -18483,6 +18483,23 @@ roadmap: estimated_effort: null labels: [] notes: 'Before: v0.70.1 tag run completed 25 min after start with no ci / coverage job; the gate refused only at preflight, after the tag and the GitHub release were public (up to 90 tries x 60 s, 2 gh calls/try). After: refused at cut_tag, 0 gh calls, 0 s. NOTE: ci.yml today declares no ci / coverage job (coverage runs inside x86-main), so every release refuses at cut_tag until one is declared or the gate is retargeted.' +- id: GH-4718 + github_issue: 4718 + item_type: task + title: '0702-P1: nightly producers for the deep lanes (deep-doctests, deep-nodefault, deep-bins-smoke)' + status: inprogress + priority: critical + assigned_to: null + created: 2026-10-04T09:17:34.904751550+00:00 + updated: 2026-10-04T09:17:34.904751550+00:00 + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - P0 + notes: null - id: PMAT-3351 github_issue: 3347 item_type: task @@ -24081,6 +24098,25 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe labels: - kind:docs notes: 'Operator C197, 2026-10-01 12:40Z: re-date via the spec''s own procedure; check unchanged; not a waiver.' +- id: PMAT-4715 + github_issue: 4715 + item_type: task + title: 'B2: root-cause fixes for red nightlies on main' + status: inprogress + priority: critical + assigned_to: aprender-wmainred + created: 2026-10-04T00:00:00Z + updated: 2026-10-04T00:00:00Z + acceptance_criteria: + - Each red or cancelled scheduled workflow on main has a root cause named in its fix PR, one PR per defect, listed on issue #4715 + - Each fix PR changes no gate's acceptance; a threshold or acceptance change goes to the train lead's sign-off batch instead + - After each fix merges, the next scheduled or dispatched run of that workflow on main gets past the step that failed + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:ci + notes: 'Parent #4678 (0.70.2), bar B2: no check red on main for more than 24 h. Proposed by aprender-a7, minted by cop aprender-14.' - id: PMAT-4729 github_issue: 4729 item_type: task From 6964ee9a23de32eef4f57745711feb04ef3b74a1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 08:30:59 +0200 Subject: [PATCH 07/14] Revert "fix(nightly): publish read one page of release assets; staged leftovers past it made every upload 422" This reverts commit c356ec874c. It adds Python to scripts/nightly_manifest.py, and the build takes no new Python. The 422 fix is redone in bash under #4681. Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- scripts/nightly_manifest.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/scripts/nightly_manifest.py b/scripts/nightly_manifest.py index 05c06cd215..206b97fce7 100755 --- a/scripts/nightly_manifest.py +++ b/scripts/nightly_manifest.py @@ -327,19 +327,6 @@ def publish_plan(manifest, sha, dist_files): return names + [MANIFEST_ASSET] -def release_assets(A, rel_id, token, get=api): - """{name: id} for EVERY asset on the release, all pages. GitHub caps a page at 100; - page one alone missed 134 of 234 assets, so the staged leftovers past it were never - deleted and the next upload of the same name died 422 already_exists (run 37171118510).""" - out, page = {}, 1 - while True: - batch = get("GET", f"{A}/releases/{rel_id}/assets?per_page=100&page={page}", token) - out.update({a["name"]: a["id"] for a in batch}) - if len(batch) < 100: - return out - page += 1 - - def publish(manifest_path, dist, sha, repo, token): with open(manifest_path) as f: man = json.load(f) @@ -362,7 +349,7 @@ def publish(manifest_path, dist, sha, repo, token): rel = api("POST", f"{A}/releases", token, json.dumps({ "tag_name": "nightly", "target_commitish": sha, "name": "Nightly Build", "prerelease": True, "make_latest": "false"}).encode()) - existing = release_assets(A, rel["id"], token) + existing = {a["name"]: a["id"] for a in api("GET", f"{A}/releases/{rel['id']}/assets?per_page=100", token)} for n, i in existing.items(): # a staged upload an earlier run died holding if n.startswith(STAGED): api("DELETE", f"{A}/releases/assets/{i}", token) @@ -392,18 +379,6 @@ def publish(manifest_path, dist, sha, repo, token): # ---------------------------------------------------------------- self-test -def _self_test_release_assets(check): - print("release assets:") - rows = [{"name": f"staged.a{i}" if i % 2 else f"a{i}", "id": i} for i in range(234)] - - def pages(of): - return lambda m, url, tok: of[(int(url.rsplit("page=", 1)[1]) - 1) * 100:][:100] - got = release_assets("A", 1, None, pages(rows)) - check("every page is read: 234 assets over 3 pages, all 117 staged leftovers seen", - (len(got), sum(n.startswith(STAGED) for n in got)), (234, 117)) - check("an exact multiple of 100 ends on the empty page", len(release_assets("A", 1, None, pages(rows[:200]))), 200) - - def self_test(): fails = [] @@ -633,7 +608,6 @@ def pkg(name, bins, root=False, feats=None): except ValueError: got = "error" check("a manifest naming an absent asset refuses to publish", got, "error") - _self_test_release_assets(check) print(f"\n{'PASS' if not fails else 'FAIL'}: {len(fails)} failing row(s)") return 1 if fails else 0 From dc5171d14acb9acdf61548dd4db302a6231e57ff Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 08:33:46 +0200 Subject: [PATCH 08/14] fix(nightly): prune the release in bash before publish; one page of assets made every upload 422 The nightly release held 234 assets: 117 staged.* leftovers from runs that died mid-publish and 117 real ones. publish lists one page (per_page=100), so the leftovers past it were never deleted and staging the same name again failed HTTP 422 already_exists (run 37171118510), every night. Purging leftovers alone still leaves 117 real assets, 17 past page one, and renaming onto one of those is the same 422. scripts/nightly_prune_release.sh runs before publish and reads every page. It deletes every staged.* leftover, and the old copy of each name this run re-uploads that sits past publish's page one. A past-page-one asset the run does not upload (another arch's last green) is kept. Any API failure exits nonzero; a 404 release is the first run. Bash only: this replaces the Python version of the fix, reverted above. --self-test: 13 rows, PASS, stub API, no network. Mutants, each RED: page-one-only listing (7 rows fail), never drop past-page-one copies (5), never purge staged (6). Runs in the nightly gate job and in ci/sections.yml beside nightly_manifest.py --self-test. bashrs lint: 0 errors. actionlint on nightly.yml: 2 findings before and after. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 11 +- ci/sections.yml | 1 + scripts/nightly_prune_release.sh | 186 +++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100755 scripts/nightly_prune_release.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 69d79c63ee..af0bb97a29 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -80,7 +80,9 @@ jobs: ref: main - name: Case table - run: python3 scripts/nightly_manifest.py --self-test + run: | + python3 scripts/nightly_manifest.py --self-test + bash scripts/nightly_prune_release.sh --self-test - name: Decide (reused | build | red-ci | ci-pending) id: gate @@ -326,6 +328,13 @@ jobs: cat nightly-manifest.json echo "decision=$(python3 -c 'import json;print(json.load(open("nightly-manifest.json"))["decision"])')" >> "$GITHUB_OUTPUT" + # publish lists one page of assets (100); the release overflowed it, so staged + # leftovers and old copies past page one made every upload 422 (run 37171118510). + - name: Prune the release to what publish can see + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash scripts/nightly_prune_release.sh dist + # REST, not `gh`: the fleet boxes carry no gh (binary-release.yml, run 34448908554). - name: Publish green arches, then nightly-manifest.json env: diff --git a/ci/sections.yml b/ci/sections.yml index 410f4f350e..c305724594 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -1394,6 +1394,7 @@ jobs: bash scripts/ci_resolve_dirty.sh --selftest python3 scripts/lib/roadmap_merge.py --selftest python3 scripts/nightly_manifest.py --self-test # #4189 nightly gate/manifest/publish plan + bash scripts/nightly_prune_release.sh --self-test # nightly publish overflowed page one (422) - name: "target-watch: before-cargo-install-bashrs" # aprender#2822 boundary probe (2 stat, 1 readdir, 1 df, 1 pgrep). # Instrumentation, never a gate: the script always exits 0. diff --git a/scripts/nightly_prune_release.sh b/scripts/nightly_prune_release.sh new file mode 100755 index 0000000000..5b5ed36497 --- /dev/null +++ b/scripts/nightly_prune_release.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# nightly_prune_release.sh DIST -- make the nightly release fit what publish sees. +# +# scripts/nightly_manifest.py publish lists the release's assets ONE page deep +# (per_page=100). The release held 234: 117 staged.* leftovers from runs that +# died mid-publish, and 117 real assets. The leftovers past page one were never +# deleted, so staging the same name again failed HTTP 422 already_exists (run +# 37171118510), every night. Purging the leftovers alone is not enough: 117 real +# assets still overflow page one, publish never sees the old copy of the names +# past it, and renaming its staged upload onto such a name is the same 422. +# +# So, reading EVERY page, before publish runs: +# 1. delete every staged.* asset (a leftover; publish stages afresh), and +# 2. delete the old copy of each name this run uploads (a file in DIST, or +# nightly-manifest.json) that sits past page one of what publish will list. +# A name past page one that this run does not upload is kept: it is some arch's +# last green build, and publish never touches it, so it cannot collide. +# Cost of 2: if publish then fails while staging, those (at most overflow-many) +# names are absent until the next green night -- the same window publish's own +# delete-then-rename opens for every name. +# +# Any API or tool failure exits nonzero: a listing that could not be read is not +# an empty release. HTTP 404 on the release itself is the first run: nothing to do. +# +# bash scripts/nightly_prune_release.sh DIST (needs GH_TOKEN, GITHUB_REPOSITORY) +# bash scripts/nightly_prune_release.sh --self-test +# +# Test seam: NIGHTLY_PRUNE_API= is called as ` METHOD URL` in place of +# curl; it prints the body and exits 0, or exits 4 for a 404, else nonzero. +set -euo pipefail + +PAGE=100 +STAGED=staged. +MANIFEST_ASSET=nightly-manifest.json + +die() { echo "::error::nightly_prune_release: $*" >&2; exit 1; } + +api() { # METHOD URL -> body on stdout; rc 4 = HTTP 404 + if [ -n "${NIGHTLY_PRUNE_API:-}" ]; then + "$NIGHTLY_PRUNE_API" "$1" "$2" + return + fi + local body code + body=$(mktemp) + code=$(curl -sS --retry 3 --retry-all-errors -o "$body" -w '%{http_code}' -X "$1" \ + -H "Authorization: Bearer ${GH_TOKEN:?GH_TOKEN unset}" \ + -H "Accept: application/vnd.github+json" "$2") || { rm -f "${body:?}"; return 1; } + case "$code" in + 2??) cat "$body"; rm -f "${body:?}" ;; + 404) rm -f "${body:?}"; return 4 ;; + *) echo "::error::$1 $2: HTTP $code" >&2; rm -f "${body:?}"; return 1 ;; + esac +} + +prune() { # DIST + local dist="$1" A rel rc page batch n all overflow name id i + [ -d "$dist" ] || die "dist dir $dist does not exist" + command -v jq >/dev/null || die "jq not found on this runner" + A="https://api.github.com/repos/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY unset}" + rc=0 + rel=$(api GET "$A/releases/tags/nightly") || rc=$? + if [ "$rc" -eq 4 ]; then + echo "no nightly release yet (HTTP 404): nothing to prune" + return 0 + fi + [ "$rc" -eq 0 ] || die "reading the nightly release failed" + rel=$(printf '%s' "$rel" | jq -er .id) || die "the nightly release has no id" + + all="" # "idname" per asset, in the order the API lists them + page=1 + while :; do + batch=$(api GET "$A/releases/$rel/assets?per_page=$PAGE&page=$page") || die "listing assets page $page failed" + batch=$(printf '%s' "$batch" | jq -er '.[] | "\(.id)\t\(.name)"' 2>/dev/null) || batch="" + n=0 + [ -z "$batch" ] || n=$(printf '%s\n' "$batch" | wc -l) + [ -z "$batch" ] || all+="$batch"$'\n' + [ "$n" -ge "$PAGE" ] || break + page=$((page + 1)) + done + + local staged=0 dropped=0 total=0 + i=0 # position among the non-staged assets: what publish's one page will hold + while IFS=$'\t' read -r id name; do + [ -n "$id" ] || continue + total=$((total + 1)) + case "$name" in + "$STAGED"*) + api DELETE "$A/releases/assets/$id" >/dev/null || die "deleting $name ($id) failed" + staged=$((staged + 1)) + continue ;; + esac + i=$((i + 1)) + [ "$i" -gt "$PAGE" ] || continue + if [ "$name" = "$MANIFEST_ASSET" ] || [ -f "$dist/$name" ]; then + api DELETE "$A/releases/assets/$id" >/dev/null || die "deleting $name ($id) failed" + dropped=$((dropped + 1)) + overflow="$name" + echo "past page one, re-uploaded this run: deleted old $overflow" + fi + done <<< "$all" + echo "nightly release $rel: $total asset(s) over $page page(s); deleted $staged staged leftover(s) and $dropped old copy(ies) past page one" +} + +# ---------------------------------------------------------------- self-test + +self_test() { + local tmp fails=0 out rc + tmp=$(mktemp -d) + # stub API: assets come from $tmp/assets (idname), deletes are logged + cat > "$tmp/api" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +d="$(dirname "$0")" +[ ! -e "$d/fail" ] || exit 1 +case "$1 $2" in + "GET "*/releases/tags/nightly) [ ! -e "$d/norel" ] || exit 4; echo '{"id": 7}' ;; + "GET "*/assets\?per_page=*) + p=${2##*page=}; s=$(( (p - 1) * 100 + 1 )); e=$(( p * 100 )) + sed -n "${s},${e}p" "$d/assets" | jq -Rn '[inputs | split("\t") | {id: (.[0]|tonumber), name: .[1]}]' ;; + "DELETE "*) echo "${2##*/}" >> "$d/deleted" ;; + *) exit 1 ;; +esac +STUB + chmod +x "$tmp/api" + check() { # NAME GOT WANT + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got '$2', want '$3'"; fails=$((fails + 1)); fi + } + run() { # -> rc in $rc, output in $out; fresh delete log + : > "$tmp/deleted" + rc=0 + out=$(GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/dist" 2>&1) || rc=$? + } + mkdir -p "$tmp/dist" + + # the measured release: 234 assets, staged and real interleaved, every real one rebuilt + : > "$tmp/assets" + for i in $(seq 1 234); do + if [ $((i % 2)) -eq 1 ]; then printf '%s\tstaged.a%s\n' "$i" "$i"; else printf '%s\ta%s\n' "$i" "$i"; : > "$tmp/dist/a$i"; fi + done >> "$tmp/assets" + run + check "234 assets over 3 pages: exit 0" "$rc" 0 + check "every staged leftover is deleted, past page one too" "$(grep -c . "$tmp/deleted")" 134 + check "the 117 staged ones are all among the deletes" \ + "$(awk 'NR%2==1{print $1}' "$tmp/assets" | grep -cxFf - "$tmp/deleted")" 117 + check "real assets past publish's page one that are re-uploaded: the 17 old copies go" \ + "$(awk 'NR%2==0{print $1}' "$tmp/assets" | grep -cxFf - "$tmp/deleted")" 17 + check "the first 100 real assets are never deleted (publish swaps them itself)" \ + "$(awk 'NR%2==0{print $1}' "$tmp/assets" | head -100 | grep -cxFf - "$tmp/deleted" || true)" 0 + + # a real asset past page one that this run does NOT rebuild is some arch's last green: kept + rm -f "$tmp/dist/a234" + run + check "a past-page-one asset this run does not upload is kept" "$(grep -cx 234 "$tmp/deleted" || true)" 0 + check "... and the other 16 old copies still go" \ + "$(awk 'NR%2==0{print $1}' "$tmp/assets" | grep -cxFf - "$tmp/deleted")" 16 + + # an exact multiple of the page size ends on the empty page, and the manifest counts as uploaded + : > "$tmp/assets" + for i in $(seq 1 200); do printf '%s\tb%s\n' "$i" "$i"; done >> "$tmp/assets" + printf '201\t%s\n' "$MANIFEST_ASSET" >> "$tmp/assets" + run + check "201 assets: page 3 is read and the manifest past page one is deleted" "$(cat "$tmp/deleted")" 201 + sed -i '$d' "$tmp/assets" + run + check "200 assets, none rebuilt: nothing deleted" "$(grep -c . "$tmp/deleted" || true)" 0 + check "... and the run says it read 200 assets" "$(printf '%s' "$out" | grep -c '200 asset(s)')" 1 + + # first run, and fail-closed + touch "$tmp/norel"; run + check "no release yet (404): exit 0, nothing deleted" "$rc/$(grep -c . "$tmp/deleted" || true)" 0/0 + rm -f "$tmp/norel"; touch "$tmp/fail"; run + check "an API failure is never an empty release: exit nonzero" "$rc" 1 + rm -f "$tmp/fail" + check "a missing dist dir: exit nonzero" \ + "$( (GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/nope") >/dev/null 2>&1; echo $?)" 1 + + rm -rf "${tmp:?}" + if [ "$fails" -eq 0 ]; then echo "nightly_prune_release self-test: PASS"; else echo "nightly_prune_release self-test: FAIL ($fails)"; fi + [ "$fails" -eq 0 ] +} + +case "${1:-}" in + --self-test) self_test ;; + ""|-h|--help) echo "usage: nightly_prune_release.sh DIST | --self-test" >&2; exit 2 ;; + *) prune "$1" ;; +esac From 50f74e811db7fbdee223eb4e5310cec2788b471a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 12:59:16 +0200 Subject: [PATCH 09/14] fix(nightly): a non-array asset page is an unread listing, never the empty last page nightly_prune_release.sh turned any jq failure on an assets page into an empty batch, which ends the page loop. An HTTP 200 whose body is not a JSON array (a proxy's HTML, an error object) therefore truncated the listing and exited 0, and publish would still hit the 422 the script exists to prevent. Check the body is an array, else die; an empty array still ends the loop. New self-test row: page 2 answers 200 with HTML -> exit nonzero. Restoring the old line turns it RED (got 0, want 1). 14 rows PASS; bashrs 0 errors. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- scripts/nightly_prune_release.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/nightly_prune_release.sh b/scripts/nightly_prune_release.sh index 5b5ed36497..8886639561 100755 --- a/scripts/nightly_prune_release.sh +++ b/scripts/nightly_prune_release.sh @@ -70,7 +70,10 @@ prune() { # DIST page=1 while :; do batch=$(api GET "$A/releases/$rel/assets?per_page=$PAGE&page=$page") || die "listing assets page $page failed" - batch=$(printf '%s' "$batch" | jq -er '.[] | "\(.id)\t\(.name)"' 2>/dev/null) || batch="" + # a 200 whose body is not an array (a proxy's HTML, an error object) is a listing + # that could not be read, never the empty page that ends it + jq -e 'type == "array"' <<< "$batch" >/dev/null 2>&1 || die "assets page $page is not a JSON array" + batch=$(printf '%s' "$batch" | jq -r '.[] | "\(.id)\t\(.name)"') || die "reading assets page $page failed" n=0 [ -z "$batch" ] || n=$(printf '%s\n' "$batch" | wc -l) [ -z "$batch" ] || all+="$batch"$'\n' @@ -115,7 +118,9 @@ d="$(dirname "$0")" case "$1 $2" in "GET "*/releases/tags/nightly) [ ! -e "$d/norel" ] || exit 4; echo '{"id": 7}' ;; "GET "*/assets\?per_page=*) - p=${2##*page=}; s=$(( (p - 1) * 100 + 1 )); e=$(( p * 100 )) + p=${2##*page=} + [ ! -e "$d/badpage$p" ] || { echo 'bad gateway'; exit 0; } + s=$(( (p - 1) * 100 + 1 )); e=$(( p * 100 )) sed -n "${s},${e}p" "$d/assets" | jq -Rn '[inputs | split("\t") | {id: (.[0]|tonumber), name: .[1]}]' ;; "DELETE "*) echo "${2##*/}" >> "$d/deleted" ;; *) exit 1 ;; @@ -171,6 +176,9 @@ STUB rm -f "$tmp/norel"; touch "$tmp/fail"; run check "an API failure is never an empty release: exit nonzero" "$rc" 1 rm -f "$tmp/fail" + touch "$tmp/badpage2"; run + check "a page that is not a JSON array (HTTP 200 HTML) is never the empty last page: exit nonzero" "$rc" 1 + rm -f "$tmp/badpage2" check "a missing dist dir: exit nonzero" \ "$( (GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/nope") >/dev/null 2>&1; echo $?)" 1 From b5bf02e4577c2d56834eabf78e0ad48143c31698 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 13:09:52 +0200 Subject: [PATCH 10/14] fix(nightly): prune deletes only names in publish's own plan; the array guard gets a row that can fail Re-review of 50f74e811d found two gaps. 1. A past-page-one asset was deleted if its name was a file in DIST. publish uploads less than DIST: publish_plan keeps only targets green at SHA and built by this run. A file in DIST outside that plan lost its old copy and got no new one. The script now takes MANIFEST and SHA, mirrors publish_plan in jq, and deletes only a planned name that is present in DIST (plus the manifest). nightly.yml passes nightly-manifest.json and $NIGHTLY_SHA. 2. The non-array row answered HTML, which the extraction line rejects on its own, so deleting the guard left the row green. The stub now answers `{}`, which only the guard catches. Self-test: 21 rows PASS. Ten mutants each turn it RED: no array guard (1 row), DIST-only membership (6), plan read not fail-closed (1), no status / sha / run filter (2 each), page-one-only listing (10), no staged purge (2), manifest not planned (1), substring membership (5). bashrs: 0 errors. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 2 +- scripts/nightly_prune_release.sh | 75 +++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index af0bb97a29..5cfabf0079 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -333,7 +333,7 @@ jobs: - name: Prune the release to what publish can see env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: bash scripts/nightly_prune_release.sh dist + run: bash scripts/nightly_prune_release.sh dist nightly-manifest.json "$NIGHTLY_SHA" # REST, not `gh`: the fleet boxes carry no gh (binary-release.yml, run 34448908554). - name: Publish green arches, then nightly-manifest.json diff --git a/scripts/nightly_prune_release.sh b/scripts/nightly_prune_release.sh index 8886639561..c97df2e048 100755 --- a/scripts/nightly_prune_release.sh +++ b/scripts/nightly_prune_release.sh @@ -11,8 +11,11 @@ # # So, reading EVERY page, before publish runs: # 1. delete every staged.* asset (a leftover; publish stages afresh), and -# 2. delete the old copy of each name this run uploads (a file in DIST, or -# nightly-manifest.json) that sits past page one of what publish will list. +# 2. delete the old copy of each name this run uploads that sits past page one +# of what publish will list. "Uploads" is publish's own plan (publish_plan in +# scripts/nightly_manifest.py), read from MANIFEST in jq: the tools of targets +# green at SHA and built by this run, each asset and its .sha256, present in +# DIST -- plus nightly-manifest.json. A file merely in DIST is not enough. # A name past page one that this run does not upload is kept: it is some arch's # last green build, and publish never touches it, so it cannot collide. # Cost of 2: if publish then fails while staging, those (at most overflow-many) @@ -22,7 +25,7 @@ # Any API or tool failure exits nonzero: a listing that could not be read is not # an empty release. HTTP 404 on the release itself is the first run: nothing to do. # -# bash scripts/nightly_prune_release.sh DIST (needs GH_TOKEN, GITHUB_REPOSITORY) +# bash scripts/nightly_prune_release.sh DIST MANIFEST SHA (needs GH_TOKEN, GITHUB_REPOSITORY) # bash scripts/nightly_prune_release.sh --self-test # # Test seam: NIGHTLY_PRUNE_API= is called as ` METHOD URL` in place of @@ -52,10 +55,20 @@ api() { # METHOD URL -> body on stdout; rc 4 = HTTP 404 esac } -prune() { # DIST - local dist="$1" A rel rc page batch n all overflow name id i +prune() { # DIST MANIFEST SHA + local dist="$1" manifest="$2" sha="$3" A rel rc page batch n all name id i plan + local up=$'\n' # newline-delimited set: the names publish uploads this run [ -d "$dist" ] || die "dist dir $dist does not exist" command -v jq >/dev/null || die "jq not found on this runner" + [ -n "$sha" ] || die "no sha given" + # publish_plan, mirrored: green at SHA, built by this run, each asset and its .sha256 + plan=$(jq -r --arg sha "$sha" '.run_id as $r | .targets[] + | select(.status == "green" and .green_sha == $sha and .built_run_id == $r) + | .tools[].asset | ., . + ".sha256"' "$manifest") || die "reading the plan from $manifest failed" + while IFS= read -r name; do + [ -z "$name" ] || [ ! -f "$dist/$name" ] || up+="$name"$'\n' + done <<< "$plan" + up+="$MANIFEST_ASSET"$'\n' A="https://api.github.com/repos/${GITHUB_REPOSITORY:?GITHUB_REPOSITORY unset}" rc=0 rel=$(api GET "$A/releases/tags/nightly") || rc=$? @@ -94,11 +107,10 @@ prune() { # DIST esac i=$((i + 1)) [ "$i" -gt "$PAGE" ] || continue - if [ "$name" = "$MANIFEST_ASSET" ] || [ -f "$dist/$name" ]; then + if [[ "$up" == *$'\n'"$name"$'\n'* ]]; then api DELETE "$A/releases/assets/$id" >/dev/null || die "deleting $name ($id) failed" dropped=$((dropped + 1)) - overflow="$name" - echo "past page one, re-uploaded this run: deleted old $overflow" + echo "past page one, re-uploaded this run: deleted old $name" fi done <<< "$all" echo "nightly release $rel: $total asset(s) over $page page(s); deleted $staged staged leftover(s) and $dropped old copy(ies) past page one" @@ -119,7 +131,7 @@ case "$1 $2" in "GET "*/releases/tags/nightly) [ ! -e "$d/norel" ] || exit 4; echo '{"id": 7}' ;; "GET "*/assets\?per_page=*) p=${2##*page=} - [ ! -e "$d/badpage$p" ] || { echo 'bad gateway'; exit 0; } + [ ! -e "$d/badpage$p" ] || { echo '{}'; exit 0; } s=$(( (p - 1) * 100 + 1 )); e=$(( p * 100 )) sed -n "${s},${e}p" "$d/assets" | jq -Rn '[inputs | split("\t") | {id: (.[0]|tonumber), name: .[1]}]' ;; "DELETE "*) echo "${2##*/}" >> "$d/deleted" ;; @@ -130,12 +142,26 @@ STUB check() { # NAME GOT WANT if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got '$2', want '$3'"; fails=$((fails + 1)); fi } - run() { # -> rc in $rc, output in $out; fresh delete log + mkplan() { # manifest: every file in dist is a tool of target g, green at S and built by run 7 -- + # except a name listed in $tmp/off as "NAMESTATUSSHARUN", which gets a target of its own + find "$tmp/dist" -type f -printf '%f\n' | jq -Rn --rawfile off "$tmp/off" ' + [inputs] as $all + | ($off | split("\n") | map(select(. != "") | split("\t"))) as $o + | ($o | map(.[0])) as $skip + | {run_id: 7, targets: ({ + g: {status: "green", green_sha: "S", built_run_id: 7, + tools: ([$all[] | select(. as $n | $skip | index($n) | not)] | map({key: ., value: {asset: .}}) | from_entries)}} + + ($o | map({key: ("o-" + .[0]), value: {status: .[1], green_sha: .[2], built_run_id: (.[3] | tonumber), + tools: {x: {asset: .[0]}}}}) | from_entries))}' > "$tmp/manifest" + } + run() { # -> rc in $rc, output in $out; fresh delete log and manifest : > "$tmp/deleted" + mkplan rc=0 - out=$(GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/dist" 2>&1) || rc=$? + out=$(GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/dist" "$tmp/manifest" S 2>&1) || rc=$? } mkdir -p "$tmp/dist" + : > "$tmp/off" # the measured release: 234 assets, staged and real interleaved, every real one rebuilt : > "$tmp/assets" @@ -153,12 +179,23 @@ STUB "$(awk 'NR%2==0{print $1}' "$tmp/assets" | head -100 | grep -cxFf - "$tmp/deleted" || true)" 0 # a real asset past page one that this run does NOT rebuild is some arch's last green: kept - rm -f "$tmp/dist/a234" + rm -f "$tmp/dist/a234"; : > "$tmp/dist/a234.old" # a planned name that CONTAINS a234 must not match it run check "a past-page-one asset this run does not upload is kept" "$(grep -cx 234 "$tmp/deleted" || true)" 0 check "... and the other 16 old copies still go" \ "$(awk 'NR%2==0{print $1}' "$tmp/assets" | grep -cxFf - "$tmp/deleted")" 16 + # a file in dist that publish's plan leaves out is not uploaded, so its old copy is kept -- one row per + # filter of publish_plan: a red target, a target green only at an older sha, a night reused from run 6 + for o in 'red S 7' 'green S0 7' 'green S 6'; do + printf 'a232\t%s\n' "$o" > "$tmp/off" + run + check "a past-page-one asset in dist outside the plan (${o// / }) is kept" "$(grep -cx 232 "$tmp/deleted" || true)" 0 + check "... and the other 15 old copies still go (${o// / })" \ + "$(awk 'NR%2==0{print $1}' "$tmp/assets" | grep -cxFf - "$tmp/deleted")" 15 + done + : > "$tmp/off" + # an exact multiple of the page size ends on the empty page, and the manifest counts as uploaded : > "$tmp/assets" for i in $(seq 1 200); do printf '%s\tb%s\n' "$i" "$i"; done >> "$tmp/assets" @@ -177,18 +214,24 @@ STUB check "an API failure is never an empty release: exit nonzero" "$rc" 1 rm -f "$tmp/fail" touch "$tmp/badpage2"; run - check "a page that is not a JSON array (HTTP 200 HTML) is never the empty last page: exit nonzero" "$rc" 1 + check "a 200 page that is an empty JSON object, not an array, is never the empty last page: exit nonzero" "$rc" 1 rm -f "$tmp/badpage2" check "a missing dist dir: exit nonzero" \ - "$( (GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/nope") >/dev/null 2>&1; echo $?)" 1 + "$( (GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/nope" "$tmp/manifest" S) >/dev/null 2>&1; echo $?)" 1 + echo '{}' > "$tmp/nomanifest" + check "a manifest with no targets is an unreadable plan, never an empty one: exit nonzero" \ + "$( (GITHUB_REPOSITORY=o/r NIGHTLY_PRUNE_API="$tmp/api" prune "$tmp/dist" "$tmp/nomanifest" S) >/dev/null 2>&1; echo $?)" 1 rm -rf "${tmp:?}" if [ "$fails" -eq 0 ]; then echo "nightly_prune_release self-test: PASS"; else echo "nightly_prune_release self-test: FAIL ($fails)"; fi [ "$fails" -eq 0 ] } +usage() { echo "usage: nightly_prune_release.sh DIST MANIFEST SHA, or --self-test" >&2; exit 2; } + case "${1:-}" in --self-test) self_test ;; - ""|-h|--help) echo "usage: nightly_prune_release.sh DIST | --self-test" >&2; exit 2 ;; - *) prune "$1" ;; + ""|-h|--help) usage ;; + *) [ "$#" -eq 3 ] || usage + prune "$1" "$2" "$3" ;; esac From 2de952a2426f888d57e8d50190cad5c8ab15c3d7 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 14:22:01 +0200 Subject: [PATCH 11/14] evidence(pr-review): receipt for #4777 head b5bf02e457 DEGRADED, same-family: gemini empty, Claude lanes claude-sonnet-5-5 and claude-haiku-4-5-20251001 (round 3 unanimous PASS, planted question rejected by both). pmat query + duplication scan (8 needles, 0 hits over 1376/1376 branches); 10/10 guard mutants killed; cuda consulted on the cuda-nightly.yml path trigger (no-authority-found: exit 101 is the Rust harness). Checked with a throwaway minisign key: check_pr_review_receipt.sh ACCEPT. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- .../findings.sarif | 184 ++++++++++++++++++ .../receipt.intoto.jsonl | 1 + 2 files changed, 185 insertions(+) create mode 100644 evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif create mode 100644 evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl diff --git a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif new file mode 100644 index 0000000000..5089a68986 --- /dev/null +++ b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif @@ -0,0 +1,184 @@ +{ + "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "pmat" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [ + { + "ruleId": "no_existing_paginated_prune", + "level": "note", + "message": { + "text": "pmat query at head for paginated deletion of stale release assets returns 5 results; the nearest is nightly_manifest.py publish, which lists one page only, and none paginates or prunes. The duplication scan searched 8 needles over 1376/1376 sibling branches with 0 hits." + }, + "properties": { + "grounding": "measured", + "command": [ + "pmat", + "query", + "paginate release assets delete stale", + "--limit", + "5" + ], + "exit_code": 0, + "stdout_sha256": "8b607c9c54be06b5796a61cbc2edc00cd53724230579ef1c29344a9e30dae4d5", + "rationale": "No helper exists to reuse; the script fills the gap publish leaves past page one.", + "failure_scenario": "If a paginated prune existed elsewhere, a second one would drift from it; none was found.", + "precision_class": "advisory" + } + }, + { + "ruleId": "plan_mirrors_publish_plan", + "level": "note", + "message": { + "text": "The plan mirror selects green targets at SHA built by this run, as publish_plan does." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 66", + "excerpt": " | select(.status == \"green\" and .green_sha == $sha and .built_run_id == $r)", + "excerpt_sha256": "eb797e09f2aa3f3f39583a574fb88f67d7bd1881b67daac7e518139c8360115b", + "rationale": "The jq select applies the three filters publish_plan applies at scripts/nightly_manifest.py:320, so only names publish re-uploads lose their old copy.", + "failure_scenario": "If a filter were dropped, an asset outside the plan (red, stale sha, reused run) would be deleted and not replaced; mutants m4/m5/m6 each turn 2 self-test rows red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "publish_plan_filter_source", + "level": "note", + "message": { + "text": "publish_plan's own selection, cited as the reference the mirror is checked against." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_manifest.py at head, line 320", + "excerpt": " if e[\"status\"] != \"green\" or e[\"green_sha\"] != sha or e.get(\"built_run_id\") != manifest[\"run_id\"]:", + "excerpt_sha256": "a17d4cc48e1b62552ed9d0a0de37f08b40e430f45c6463e5396254b26cbc8ab2", + "rationale": "This is the line the script mirrors; the review compared them field by field.", + "failure_scenario": "A divergence here would let the prune delete what publish skips; both lanes found no divergence on a well-formed manifest.", + "precision_class": "advisory" + } + }, + { + "ruleId": "non_array_page_fails_closed", + "level": "note", + "message": { + "text": "A 200 asset page that is not a JSON array exits nonzero." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 88", + "excerpt": " jq -e 'type == \"array\"' <<< \"$batch\" >/dev/null 2>&1 || die \"assets page $page is not a JSON array\"", + "excerpt_sha256": "7608f5eb023d469aabda327d4d11a5c06b58e87c74bd56e7c2efdb83eb1bbd59", + "rationale": "Round 1 lane 1 (claude-sonnet-5-5) found a non-array 200 page read as the empty last page; this guard closes it and the self-test row answers {} so only the guard can catch it.", + "failure_scenario": "Without the guard a {} page ends the listing early and past-page-one copies survive to a rename 422; mutant m1 turns 1 row red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "exact_membership", + "level": "note", + "message": { + "text": "Membership in the plan is an exact whole-line match." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 110", + "excerpt": " if [[ \"$up\" == *$'\\n'\"$name\"$'\\n'* ]]; then", + "excerpt_sha256": "4530bc84d68b9c99990c3a3b3de473a7cf6c6577f438f1aaa4ce7b95c192e08d", + "rationale": "Membership is newline-delimited so a planned name that contains another as a substring does not match it.", + "failure_scenario": "A substring match deletes a234 because a234.old is planned; mutant m10 turns 5 rows red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "planted_question_rejected", + "level": "note", + "message": { + "text": "Quorum round 3: unanimous PASS, degraded: same-family; planted question rejected by both lanes." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 48", + "excerpt": " code=$(curl -sS --retry 3 --retry-all-errors -o \"$body\" -w '%{http_code}' -X \"$1\" \\", + "excerpt_sha256": "69ab5e8cfedabc2709dad8169f0c84f2bc32ecaa16657c59e495cf062402f579", + "rationale": "The planted contrary question Q3 claimed the script calls gh; both round-3 lanes (claude-sonnet-5-5, claude-haiku-4-5-20251001) rejected it and cited the curl call.", + "failure_scenario": "A lane that accepted the false premise would void its vote; neither did.", + "precision_class": "advisory" + } + } + ] + }, + { + "tool": { + "driver": { + "name": "cargo-mutants" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [ + { + "ruleId": "guard_mutants_killed", + "level": "note", + "message": { + "text": "Self-test at head: 21 ok rows, PASS." + }, + "properties": { + "grounding": "cited", + "source": "author mutant run at head b5bf02e457 (scripts/nightly_prune_release.sh --self-test)", + "excerpt": "head: 21 ok, nightly_prune_release self-test: PASS", + "excerpt_sha256": "33217b4d9b0db767ffea077dff8b6bfd2816b902b75428fd50e3dfdbb4437df6", + "rationale": "The unmutated script passes all 21 self-test rows.", + "failure_scenario": "If the self-test were vacuous, mutants would pass; see the next result.", + "precision_class": "advisory" + } + }, + { + "ruleId": "guard_mutants_all_red", + "level": "note", + "message": { + "text": "10 of 10 guard mutants killed." + }, + "properties": { + "grounding": "cited", + "source": "author mutant run at head b5bf02e457, 10 mutants", + "excerpt": "m1-no-array-guard: nightly_prune_release self-test: FAIL (1)\nm2-dist-only: nightly_prune_release self-test: FAIL (6)\nm3-plan-read-open: nightly_prune_release self-test: FAIL (1)\nm4-no-status: nightly_prune_release self-test: FAIL (2)\nm5-no-sha: nightly_prune_release self-test: FAIL (2)\nm6-no-run: nightly_prune_release self-test: FAIL (2)\nm7-page-one-only: nightly_prune_release self-test: FAIL (10)\nm8-no-staged-purge: nightly_prune_release self-test: FAIL (2)\nm9-no-manifest-name: nightly_prune_release self-test: FAIL (1)\nm10-substring-match: nightly_prune_release self-test: FAIL (5)", + "excerpt_sha256": "5dd9f7cabbb2ffc1c35f72e6d5cf22a63f76c5771ffebd2e3abc7d0e65695194", + "rationale": "Each of 10 mutants (array guard, dist-only membership, plan read open, status/sha/run filters, page-one-only, staged purge, manifest name, substring match) turns the self-test red; each mutant copy parses.", + "failure_scenario": "A surviving mutant would mark a row that cannot fail; none survived.", + "precision_class": "advisory" + } + } + ] + }, + { + "tool": { + "driver": { + "name": "antigravity" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [] + } + ] +} diff --git a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl new file mode 100644 index 0000000000..06afbe50fd --- /dev/null +++ b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl @@ -0,0 +1 @@ +{"_type":"https://in-toto.io/Statement/v1","subject":[{"name":"git+https://github.com/paiml/aprender","digest":{"sha1":"b5bf02e4577c2d56834eabf78e0ad48143c31698"}}],"predicateType":"https://paiml.dev/attestations/pr-review/v2","predicate":{"skill_version":"2.1.0","attestation_level":"L1-self","pr":4777,"base_sha":"897bf63f134e011953ad8623366f231ecf9ec9a7","head_sha":"b5bf02e4577c2d56834eabf78e0ad48143c31698","author_actor":{"kind":"agent","id":"agent:claude-opus-5-5/aprender-wtix-odd"},"reviewer_actor":{"kind":"agent","id":"agent:claude-sonnet-5-5/session-independent-review-b5bf02e457"},"affected_crates":[],"verdict":"DEGRADED","degraded_reason":"degraded: same-family — measured out: gemini (empty); Claude Code lanes: claude-sonnet-5-5,claude-haiku-4-5-20251001","consultations":{"pmat":{"status":"consulted","transport":"cli","transport_unavailable":["mcp: not attempted this run (cli answered)"],"index_commit":"b5bf02e4577c2d56834eabf78e0ad48143c31698","index_is_ancestor":true,"index_worktree_dirty":false,"complexity_delta":[],"tdg_delta":[],"satd_introduced":[],"cache_hits":0,"duplication_hits":[],"duplication_coverage":{"rust":"lexical","shell":"lexical","python":"lexical","config":"lexical","docs":"lexical","other":"lexical","sibling_branches":"lexical","merge_base_to_main":"lexical"},"duplication_horizon":["head=b5bf02e4577c2d56834eabf78e0ad48143c31698","siblings=refs/remotes/origin/* unmerged into origin/main","merge_base_to_main=897bf63f134e011953ad8623366f231ecf9ec9a7..refs/remotes/origin/main"],"horizon_branches_total":1376,"horizon_branches_scanned":1376,"merge_base_to_main_files":0,"symbols_searched":8,"queries":[{"q":"paginate release assets delete stale","command":["pmat","query","paginate release assets delete stale","--limit","5"],"cwd":"the review worktree at head","results":5,"note":"index built at head; the nearest result is nightly_manifest.py publish, which lists one page; no paginated prune exists to reuse"}]},"cuda":{"status":"consulted","trigger_reason":"The S3.B path trigger fires on .github/workflows/cuda-nightly.yml. Its one hunk turns bash -e off before an existing serve-route GPU test step so the RED control exit 101 reaches its assertions; it changes no kernel, launch, stream or device code. The only claim adjacent to device behaviour is that exit 101 comes from the Rust test harness, not from the CUDA runtime, so that was asked.","queries":[{"q":"process exit code when a CUDA host program test harness panics: does the CUDA runtime define or alter the host process exit status (e.g. 101) on context teardown?","result":"no-authority-found","source_uri":"mcp__nvidia-cuda-docs__search_cuda_docs","claim_checked":"Exit 101 in the step is the Rust test harness panic code. The corpus returned only sample programs that return 0 or 1 themselves, nothing saying the CUDA runtime sets or alters a host exit status, so the claim stands on the Rust side and no finding is raised."}]},"crux":{"status":"not-triggered","trigger_reason":"the diff touches no CLI subcommand or flag, HTTP route, MCP tool, config key or output format; it adds a script run by one nightly workflow step"},"mutation":{"status":"consulted","scope":"guard","attempted":10,"killed":10,"survivors":[]},"antigravity":{"status":"unreachable","attempted":0,"reason":"gemini bucket state empty (budget 0); the budget probe was run and no agy call was made"}},"findings_ref":{"path":"findings.sarif","sha256":"35312d696975e623fa588e4125853e7c79cab08863e5d98bb338885fe6db5327"},"cost":{"input_tokens":0,"output_tokens":0,"wall_seconds":0},"diff_patch_id":"cb0438a52c043e676feb5ee0948b582e9a3f1b53","diff_patch_id_algo":"git-patch-id-verbatim/pinned-diff-v1"}} From 7c2c39fc210df41efd7a0a3e54c2df1604e917c0 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 14:22:41 +0200 Subject: [PATCH 12/14] evidence(pr-review): #4777 receipt records each round-3 lane's verdict on its own C312-9: a same-family answer stands only if unanimous. The receipt said so in one summary result; it now carries one cited result per lane (exact model id, PASS, planted Q3 rejected, own mutation RED), quoted verbatim from the quorum record, and re-pins findings_ref.sha256. Throwaway-key check: ACCEPT. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- .../findings.sarif | 32 +++++++++++++++++++ .../receipt.intoto.jsonl | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif index 5089a68986..8651770fc3 100644 --- a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif +++ b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/findings.sarif @@ -116,6 +116,38 @@ "failure_scenario": "A lane that accepted the false premise would void its vote; neither did.", "precision_class": "advisory" } + }, + { + "ruleId": "quorum_r3_lane1_sonnet", + "level": "note", + "message": { + "text": "Round 3 lane 1, model claude-sonnet-5-5: PASS, planted Q3 rejected (cited curl at :48), mutation of its own choosing RED." + }, + "properties": { + "grounding": "cited", + "source": "/mnt/nvme-raid0/cop-inbox/handoff/signoff-q-4777-prune.md, quorum receipt table, round 3 row", + "excerpt": "| 3 | b5bf02e457 | 1 | claude-sonnet-5-5 | PASS | rejected (curl, :48) | yes | built_run_id filter dropped (:66), RED 2 |", + "excerpt_sha256": "b2e676a4e164b6ebc7d07b798af2a424a73b6d23db38f4bd53244fa94af49339", + "rationale": "Lane 1 verdict recorded on its own, with exact model id.", + "failure_scenario": "Under C312-9 a same-family answer stands only if unanimous; a FAIL or a lane that accepted the planted premise in either row would void the review.", + "precision_class": "advisory" + } + }, + { + "ruleId": "quorum_r3_lane2_haiku", + "level": "note", + "message": { + "text": "Round 3 lane 2, model claude-haiku-4-5-20251001: PASS, planted Q3 rejected (cited curl at :41-56), mutation of its own choosing RED." + }, + "properties": { + "grounding": "cited", + "source": "/mnt/nvme-raid0/cop-inbox/handoff/signoff-q-4777-prune.md, quorum receipt table, round 3 row", + "excerpt": "| 3 | b5bf02e457 | 2 | claude-haiku-4-5-20251001 | PASS | rejected (curl, :41-56) | yes | membership line :110 leading \\n dropped, RED 7 |", + "excerpt_sha256": "7df753263e124afb3eb8cc01da485301ba76750a5389bf419eebedc8cbe5a219", + "rationale": "Lane 2 verdict recorded on its own, with exact model id; with lane 1 it makes the vote unanimous (2/2 PASS).", + "failure_scenario": "Under C312-9 a same-family answer stands only if unanimous; a FAIL or a lane that accepted the planted premise in either row would void the review.", + "precision_class": "advisory" + } } ] }, diff --git a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl index 06afbe50fd..0b287538e4 100644 --- a/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl +++ b/evidence/pr-review/4777/b5bf02e4577c2d56834eabf78e0ad48143c31698/receipt.intoto.jsonl @@ -1 +1 @@ -{"_type":"https://in-toto.io/Statement/v1","subject":[{"name":"git+https://github.com/paiml/aprender","digest":{"sha1":"b5bf02e4577c2d56834eabf78e0ad48143c31698"}}],"predicateType":"https://paiml.dev/attestations/pr-review/v2","predicate":{"skill_version":"2.1.0","attestation_level":"L1-self","pr":4777,"base_sha":"897bf63f134e011953ad8623366f231ecf9ec9a7","head_sha":"b5bf02e4577c2d56834eabf78e0ad48143c31698","author_actor":{"kind":"agent","id":"agent:claude-opus-5-5/aprender-wtix-odd"},"reviewer_actor":{"kind":"agent","id":"agent:claude-sonnet-5-5/session-independent-review-b5bf02e457"},"affected_crates":[],"verdict":"DEGRADED","degraded_reason":"degraded: same-family — measured out: gemini (empty); Claude Code lanes: claude-sonnet-5-5,claude-haiku-4-5-20251001","consultations":{"pmat":{"status":"consulted","transport":"cli","transport_unavailable":["mcp: not attempted this run (cli answered)"],"index_commit":"b5bf02e4577c2d56834eabf78e0ad48143c31698","index_is_ancestor":true,"index_worktree_dirty":false,"complexity_delta":[],"tdg_delta":[],"satd_introduced":[],"cache_hits":0,"duplication_hits":[],"duplication_coverage":{"rust":"lexical","shell":"lexical","python":"lexical","config":"lexical","docs":"lexical","other":"lexical","sibling_branches":"lexical","merge_base_to_main":"lexical"},"duplication_horizon":["head=b5bf02e4577c2d56834eabf78e0ad48143c31698","siblings=refs/remotes/origin/* unmerged into origin/main","merge_base_to_main=897bf63f134e011953ad8623366f231ecf9ec9a7..refs/remotes/origin/main"],"horizon_branches_total":1376,"horizon_branches_scanned":1376,"merge_base_to_main_files":0,"symbols_searched":8,"queries":[{"q":"paginate release assets delete stale","command":["pmat","query","paginate release assets delete stale","--limit","5"],"cwd":"the review worktree at head","results":5,"note":"index built at head; the nearest result is nightly_manifest.py publish, which lists one page; no paginated prune exists to reuse"}]},"cuda":{"status":"consulted","trigger_reason":"The S3.B path trigger fires on .github/workflows/cuda-nightly.yml. Its one hunk turns bash -e off before an existing serve-route GPU test step so the RED control exit 101 reaches its assertions; it changes no kernel, launch, stream or device code. The only claim adjacent to device behaviour is that exit 101 comes from the Rust test harness, not from the CUDA runtime, so that was asked.","queries":[{"q":"process exit code when a CUDA host program test harness panics: does the CUDA runtime define or alter the host process exit status (e.g. 101) on context teardown?","result":"no-authority-found","source_uri":"mcp__nvidia-cuda-docs__search_cuda_docs","claim_checked":"Exit 101 in the step is the Rust test harness panic code. The corpus returned only sample programs that return 0 or 1 themselves, nothing saying the CUDA runtime sets or alters a host exit status, so the claim stands on the Rust side and no finding is raised."}]},"crux":{"status":"not-triggered","trigger_reason":"the diff touches no CLI subcommand or flag, HTTP route, MCP tool, config key or output format; it adds a script run by one nightly workflow step"},"mutation":{"status":"consulted","scope":"guard","attempted":10,"killed":10,"survivors":[]},"antigravity":{"status":"unreachable","attempted":0,"reason":"gemini bucket state empty (budget 0); the budget probe was run and no agy call was made"}},"findings_ref":{"path":"findings.sarif","sha256":"35312d696975e623fa588e4125853e7c79cab08863e5d98bb338885fe6db5327"},"cost":{"input_tokens":0,"output_tokens":0,"wall_seconds":0},"diff_patch_id":"cb0438a52c043e676feb5ee0948b582e9a3f1b53","diff_patch_id_algo":"git-patch-id-verbatim/pinned-diff-v1"}} +{"_type":"https://in-toto.io/Statement/v1","subject":[{"name":"git+https://github.com/paiml/aprender","digest":{"sha1":"b5bf02e4577c2d56834eabf78e0ad48143c31698"}}],"predicateType":"https://paiml.dev/attestations/pr-review/v2","predicate":{"skill_version":"2.1.0","attestation_level":"L1-self","pr":4777,"base_sha":"897bf63f134e011953ad8623366f231ecf9ec9a7","head_sha":"b5bf02e4577c2d56834eabf78e0ad48143c31698","author_actor":{"kind":"agent","id":"agent:claude-opus-5-5/aprender-wtix-odd"},"reviewer_actor":{"kind":"agent","id":"agent:claude-sonnet-5-5/session-independent-review-b5bf02e457"},"affected_crates":[],"verdict":"DEGRADED","degraded_reason":"degraded: same-family — measured out: gemini (empty); Claude Code lanes: claude-sonnet-5-5,claude-haiku-4-5-20251001","consultations":{"pmat":{"status":"consulted","transport":"cli","transport_unavailable":["mcp: not attempted this run (cli answered)"],"index_commit":"b5bf02e4577c2d56834eabf78e0ad48143c31698","index_is_ancestor":true,"index_worktree_dirty":false,"complexity_delta":[],"tdg_delta":[],"satd_introduced":[],"cache_hits":0,"duplication_hits":[],"duplication_coverage":{"rust":"lexical","shell":"lexical","python":"lexical","config":"lexical","docs":"lexical","other":"lexical","sibling_branches":"lexical","merge_base_to_main":"lexical"},"duplication_horizon":["head=b5bf02e4577c2d56834eabf78e0ad48143c31698","siblings=refs/remotes/origin/* unmerged into origin/main","merge_base_to_main=897bf63f134e011953ad8623366f231ecf9ec9a7..refs/remotes/origin/main"],"horizon_branches_total":1376,"horizon_branches_scanned":1376,"merge_base_to_main_files":0,"symbols_searched":8,"queries":[{"q":"paginate release assets delete stale","command":["pmat","query","paginate release assets delete stale","--limit","5"],"cwd":"the review worktree at head","results":5,"note":"index built at head; the nearest result is nightly_manifest.py publish, which lists one page; no paginated prune exists to reuse"}]},"cuda":{"status":"consulted","trigger_reason":"The S3.B path trigger fires on .github/workflows/cuda-nightly.yml. Its one hunk turns bash -e off before an existing serve-route GPU test step so the RED control exit 101 reaches its assertions; it changes no kernel, launch, stream or device code. The only claim adjacent to device behaviour is that exit 101 comes from the Rust test harness, not from the CUDA runtime, so that was asked.","queries":[{"q":"process exit code when a CUDA host program test harness panics: does the CUDA runtime define or alter the host process exit status (e.g. 101) on context teardown?","result":"no-authority-found","source_uri":"mcp__nvidia-cuda-docs__search_cuda_docs","claim_checked":"Exit 101 in the step is the Rust test harness panic code. The corpus returned only sample programs that return 0 or 1 themselves, nothing saying the CUDA runtime sets or alters a host exit status, so the claim stands on the Rust side and no finding is raised."}]},"crux":{"status":"not-triggered","trigger_reason":"the diff touches no CLI subcommand or flag, HTTP route, MCP tool, config key or output format; it adds a script run by one nightly workflow step"},"mutation":{"status":"consulted","scope":"guard","attempted":10,"killed":10,"survivors":[]},"antigravity":{"status":"unreachable","attempted":0,"reason":"gemini bucket state empty (budget 0); the budget probe was run and no agy call was made"}},"findings_ref":{"path":"findings.sarif","sha256":"e4519249143b1403a193f9b8165f6312dc619d9e19a15c6f0db58fcb6b61ea1a"},"cost":{"input_tokens":0,"output_tokens":0,"wall_seconds":0},"diff_patch_id":"cb0438a52c043e676feb5ee0948b582e9a3f1b53","diff_patch_id_algo":"git-patch-id-verbatim/pinned-diff-v1"}} From 58b6456d07450aecdf89c100fef7fd4477ffd11b Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 15:38:54 +0200 Subject: [PATCH 13/14] fix(nightly): self-test jq fixture avoids $all[] that bashrs 7.4.1 misreads as SC1087 CI's pinned bashrs 7.4.1 reports SC1087 at nightly_prune_release.sh:153 on the jq expression $all[] inside a single-quoted program; 7.4.2 reports nothing. The shell-lint ratchet counted 7 against a baseline of 6 on this PR merged with main; main alone measures 6. [$all | .[] | ...] is the same jq, and 7.4.1 reports 0. Self-test PASS. Agent: aprender-a7 Co-Authored-By: Claude Opus 5.5 --- scripts/nightly_prune_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nightly_prune_release.sh b/scripts/nightly_prune_release.sh index c97df2e048..1ac491b6d9 100755 --- a/scripts/nightly_prune_release.sh +++ b/scripts/nightly_prune_release.sh @@ -150,7 +150,7 @@ STUB | ($o | map(.[0])) as $skip | {run_id: 7, targets: ({ g: {status: "green", green_sha: "S", built_run_id: 7, - tools: ([$all[] | select(. as $n | $skip | index($n) | not)] | map({key: ., value: {asset: .}}) | from_entries)}} + tools: ([$all | .[] | select(. as $n | $skip | index($n) | not)] | map({key: ., value: {asset: .}}) | from_entries)}} + ($o | map({key: ("o-" + .[0]), value: {status: .[1], green_sha: .[2], built_run_id: (.[3] | tonumber), tools: {x: {asset: .[0]}}}}) | from_entries))}' > "$tmp/manifest" } From 02710493b990b56160a7cde13f05c2e830411e5a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 5 Oct 2026 15:45:38 +0200 Subject: [PATCH 14/14] evidence(pr-review): receipt for #4777 head 4cc22e57b4 (C312-9 round 4) DEGRADED, same-family: gemini empty. Round 4 on the self-test jq rewrite (58b6456d07) and the main merge: claude-sonnet-5-5 PASS and claude-haiku-4-5-20251001 PASS, unanimous, each with its own cited result; the planted question (that 58b6456d07 touched the delete loop) was rejected by both. Re-measured at head: self-test 21 rows PASS, 10/10 mutants RED, duplication scan 0 hits over 1380/1380 branches. Throwaway-key check: ACCEPT. Refs #4681 Agent: aprender-wtix-odd Co-Authored-By: Claude Opus 5.5 --- .../findings.sarif | 232 ++++++++++++++++++ .../receipt.intoto.jsonl | 1 + 2 files changed, 233 insertions(+) create mode 100644 evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/findings.sarif create mode 100644 evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/receipt.intoto.jsonl diff --git a/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/findings.sarif b/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/findings.sarif new file mode 100644 index 0000000000..772d58e67b --- /dev/null +++ b/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/findings.sarif @@ -0,0 +1,232 @@ +{ + "$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "pmat" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [ + { + "ruleId": "no_existing_paginated_prune", + "level": "note", + "message": { + "text": "pmat query at head for paginated deletion of stale release assets returns 5 results; the nearest is nightly_manifest.py publish, which lists one page only, and none paginates or prunes. The duplication scan searched 12 needles over 1380/1380 sibling branches with 0 hits." + }, + "properties": { + "grounding": "measured", + "command": [ + "pmat", + "query", + "paginate release assets delete stale", + "--limit", + "5" + ], + "exit_code": 0, + "stdout_sha256": "8b607c9c54be06b5796a61cbc2edc00cd53724230579ef1c29344a9e30dae4d5", + "rationale": "No helper exists to reuse; the script fills the gap publish leaves past page one.", + "failure_scenario": "If a paginated prune existed elsewhere, a second one would drift from it; none was found.", + "precision_class": "advisory" + } + }, + { + "ruleId": "plan_mirrors_publish_plan", + "level": "note", + "message": { + "text": "The plan mirror selects green targets at SHA built by this run, as publish_plan does." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 66", + "excerpt": " | select(.status == \"green\" and .green_sha == $sha and .built_run_id == $r)", + "excerpt_sha256": "eb797e09f2aa3f3f39583a574fb88f67d7bd1881b67daac7e518139c8360115b", + "rationale": "The jq select applies the three filters publish_plan applies at scripts/nightly_manifest.py:320, so only names publish re-uploads lose their old copy.", + "failure_scenario": "If a filter were dropped, an asset outside the plan (red, stale sha, reused run) would be deleted and not replaced; mutants m4/m5/m6 each turn 2 self-test rows red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "publish_plan_filter_source", + "level": "note", + "message": { + "text": "publish_plan's own selection, cited as the reference the mirror is checked against." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_manifest.py at head, line 320", + "excerpt": " if e[\"status\"] != \"green\" or e[\"green_sha\"] != sha or e.get(\"built_run_id\") != manifest[\"run_id\"]:", + "excerpt_sha256": "a17d4cc48e1b62552ed9d0a0de37f08b40e430f45c6463e5396254b26cbc8ab2", + "rationale": "This is the line the script mirrors; the review compared them field by field.", + "failure_scenario": "A divergence here would let the prune delete what publish skips; both lanes found no divergence on a well-formed manifest.", + "precision_class": "advisory" + } + }, + { + "ruleId": "non_array_page_fails_closed", + "level": "note", + "message": { + "text": "A 200 asset page that is not a JSON array exits nonzero." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 88", + "excerpt": " jq -e 'type == \"array\"' <<< \"$batch\" >/dev/null 2>&1 || die \"assets page $page is not a JSON array\"", + "excerpt_sha256": "7608f5eb023d469aabda327d4d11a5c06b58e87c74bd56e7c2efdb83eb1bbd59", + "rationale": "Round 1 lane 1 (claude-sonnet-5-5) found a non-array 200 page read as the empty last page; this guard closes it and the self-test row answers {} so only the guard can catch it.", + "failure_scenario": "Without the guard a {} page ends the listing early and past-page-one copies survive to a rename 422; mutant m1 turns 1 row red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "exact_membership", + "level": "note", + "message": { + "text": "Membership in the plan is an exact whole-line match." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 110", + "excerpt": " if [[ \"$up\" == *$'\\n'\"$name\"$'\\n'* ]]; then", + "excerpt_sha256": "4530bc84d68b9c99990c3a3b3de473a7cf6c6577f438f1aaa4ce7b95c192e08d", + "rationale": "Membership is newline-delimited so a planned name that contains another as a substring does not match it.", + "failure_scenario": "A substring match deletes a234 because a234.old is planned; mutant m10 turns 5 rows red.", + "precision_class": "advisory" + } + }, + { + "ruleId": "jq_rewrite_equivalent", + "level": "note", + "message": { + "text": "58b6456d07 rewrites the self-test fixture's $all[] as $all | .[]; the same jq program, measured equal on a sample (jq -n: [1,2,3] as $all | [$all[]|select(.>1)] == [$all|.[]|select(.>1)] -> true)." + }, + "properties": { + "grounding": "cited", + "source": "scripts/nightly_prune_release.sh at head, line 153", + "excerpt": " tools: ([$all | .[] | select(. as $n | $skip | index($n) | not)] | map({key: ., value: {asset: .}}) | from_entries)}}", + "excerpt_sha256": "c21fd10a4f40fec7eb572c5dc90e0b1f6fed8dcbb6a782564e4f026bbdc3af18", + "rationale": "A syntax-only change in the self-test fixture so CI's bashrs 7.4.1 stops reporting SC1087; the production path is unchanged and the self-test still passes with 21 rows.", + "failure_scenario": "If the rewrite changed the fixture's target set, the self-test rows built from it would test a different manifest than intended.", + "precision_class": "advisory" + } + }, + { + "ruleId": "planted_question_rejected", + "level": "note", + "message": { + "text": "Quorum round 4: unanimous PASS, degraded: same-family; the planted question was rejected by both lanes." + }, + "properties": { + "grounding": "cited", + "source": "git show --stat --format= 58b6456d07 (first line)", + "excerpt": " scripts/nightly_prune_release.sh | 2 +-", + "excerpt_sha256": "c001cb6a0e49c11a718987ec225f31ac8bc6356f4769ffbe68a0c2262de5cc8d", + "rationale": "The planted contrary question Q3 claimed 58b6456d07 also changed the production delete loop near line 110; both round-4 lanes (claude-sonnet-5-5, claude-haiku-4-5-20251001) rejected it: the commit is one line in, one out, at self-test line 153.", + "failure_scenario": "If 58b6456d07 had touched the delete loop, the plan-membership guarantee reviewed in round 3 would no longer be the code at head.", + "precision_class": "advisory" + } + }, + { + "ruleId": "quorum_r4_lane1_sonnet", + "level": "note", + "message": { + "text": "Round 4 lane 1, model claude-sonnet-5-5: PASS, planted Q3 rejected, mutation of its own choosing RED." + }, + "properties": { + "grounding": "cited", + "source": "/mnt/nvme-raid0/cop-inbox/handoff/signoff-q-4777-prune.md, quorum receipt table, round 4 row", + "excerpt": "| 4 | 4cc22e57b4 | 1 | claude-sonnet-5-5 | PASS | rejected (58b6456d07 = 1+/1-, line 153 only; loop :102-112 untouched) | yes | :110 membership test -> `if true`, RED 10 |", + "excerpt_sha256": "6d3961b47255553f2c091182a8661d4fe580288d61cb1dd8c80af266b285b332", + "rationale": "Lane 1 verdict recorded on its own, with exact model id.", + "failure_scenario": "Under C312-9 a same-family answer stands only if unanimous; a FAIL or a lane that accepted the planted premise in either row would void the review.", + "precision_class": "advisory" + } + }, + { + "ruleId": "quorum_r4_lane2_haiku", + "level": "note", + "message": { + "text": "Round 4 lane 2, model claude-haiku-4-5-20251001: PASS, planted Q3 rejected, mutation of its own choosing RED." + }, + "properties": { + "grounding": "cited", + "source": "/mnt/nvme-raid0/cop-inbox/handoff/signoff-q-4777-prune.md, quorum receipt table, round 4 row", + "excerpt": "| 4 | 4cc22e57b4 | 2 | claude-haiku-4-5-20251001 | PASS | rejected (only line 153 changed; zero delete-loop lines) | yes | :110 leading newline dropped, RED 7 |", + "excerpt_sha256": "74709a1bba35c570fead2330a17707a1e6e35faa028bdf3ebd6e7b08eb323ba4", + "rationale": "Lane 2 verdict recorded on its own, with exact model id; its first reply was verdict-only and was asked once for cited answers. With lane 1 it makes the vote unanimous (2/2 PASS).", + "failure_scenario": "Under C312-9 a same-family answer stands only if unanimous; a FAIL or a lane that accepted the planted premise in either row would void the review.", + "precision_class": "advisory" + } + } + ] + }, + { + "tool": { + "driver": { + "name": "cargo-mutants" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [ + { + "ruleId": "guard_mutants_killed", + "level": "note", + "message": { + "text": "Self-test at head: 21 ok rows, PASS." + }, + "properties": { + "grounding": "cited", + "source": "author mutant run at head 4cc22e57b4 (scripts/nightly_prune_release.sh --self-test)", + "excerpt": "head: 21 ok, nightly_prune_release self-test: PASS", + "excerpt_sha256": "33217b4d9b0db767ffea077dff8b6bfd2816b902b75428fd50e3dfdbb4437df6", + "rationale": "The unmutated script passes all 21 self-test rows.", + "failure_scenario": "If the self-test were vacuous, mutants would pass; see the next result.", + "precision_class": "advisory" + } + }, + { + "ruleId": "guard_mutants_all_red", + "level": "note", + "message": { + "text": "10 of 10 guard mutants killed." + }, + "properties": { + "grounding": "cited", + "source": "author mutant run at head 4cc22e57b4, 10 mutants", + "excerpt": "m1-no-array-guard: nightly_prune_release self-test: FAIL (1)\nm2-dist-only: nightly_prune_release self-test: FAIL (6)\nm3-plan-read-open: nightly_prune_release self-test: FAIL (1)\nm4-no-status: nightly_prune_release self-test: FAIL (2)\nm5-no-sha: nightly_prune_release self-test: FAIL (2)\nm6-no-run: nightly_prune_release self-test: FAIL (2)\nm7-page-one-only: nightly_prune_release self-test: FAIL (10)\nm8-no-staged-purge: nightly_prune_release self-test: FAIL (2)\nm9-no-manifest-name: nightly_prune_release self-test: FAIL (1)\nm10-substring-match: nightly_prune_release self-test: FAIL (5)", + "excerpt_sha256": "5dd9f7cabbb2ffc1c35f72e6d5cf22a63f76c5771ffebd2e3abc7d0e65695194", + "rationale": "Each of 10 mutants (array guard, dist-only membership, plan read open, status/sha/run filters, page-one-only, staged purge, manifest name, substring match) turns the self-test red; each mutant copy parses.", + "failure_scenario": "A surviving mutant would mark a row that cannot fail; none survived.", + "precision_class": "advisory" + } + } + ] + }, + { + "tool": { + "driver": { + "name": "antigravity" + } + }, + "invocations": [ + { + "executionSuccessful": true, + "toolExecutionNotifications": [] + } + ], + "results": [] + } + ] +} diff --git a/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/receipt.intoto.jsonl b/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/receipt.intoto.jsonl new file mode 100644 index 0000000000..fe27da242c --- /dev/null +++ b/evidence/pr-review/4777/4cc22e57b485bf1cd9aae8a2122cbafeb2205929/receipt.intoto.jsonl @@ -0,0 +1 @@ +{"_type":"https://in-toto.io/Statement/v1","subject":[{"name":"git+https://github.com/paiml/aprender","digest":{"sha1":"4cc22e57b485bf1cd9aae8a2122cbafeb2205929"}}],"predicateType":"https://paiml.dev/attestations/pr-review/v2","predicate":{"skill_version":"2.1.0","attestation_level":"L1-self","pr":4777,"base_sha":"6192aa9b886a31341a9cc7465662a0598964ce8c","head_sha":"4cc22e57b485bf1cd9aae8a2122cbafeb2205929","author_actor":{"kind":"agent","id":"agent:claude-opus-5-5/aprender-wtix-odd"},"reviewer_actor":{"kind":"agent","id":"agent:claude-sonnet-5-5/session-independent-review-4cc22e57b4"},"affected_crates":[],"verdict":"DEGRADED","degraded_reason":"degraded: same-family — measured out: gemini (empty); Claude Code lanes: claude-sonnet-5-5,claude-haiku-4-5-20251001","consultations":{"pmat":{"status":"consulted","transport":"cli","transport_unavailable":["mcp: not attempted this run (cli answered)"],"index_commit":"4cc22e57b485bf1cd9aae8a2122cbafeb2205929","index_is_ancestor":true,"index_worktree_dirty":false,"complexity_delta":[],"tdg_delta":[],"satd_introduced":[],"cache_hits":0,"duplication_hits":[],"duplication_coverage":{"rust":"lexical","shell":"lexical","python":"lexical","config":"lexical","docs":"lexical","other":"lexical","sibling_branches":"lexical","merge_base_to_main":"lexical"},"duplication_horizon":["head=4cc22e57b485bf1cd9aae8a2122cbafeb2205929","siblings=refs/remotes/origin/* unmerged into origin/main","merge_base_to_main=6192aa9b886a31341a9cc7465662a0598964ce8c..refs/remotes/origin/main"],"horizon_branches_total":1380,"horizon_branches_scanned":1380,"merge_base_to_main_files":0,"symbols_searched":12,"queries":[{"q":"paginate release assets delete stale","command":["pmat","query","paginate release assets delete stale","--limit","5"],"cwd":"the review worktree at head","results":5,"note":"index built at head; the nearest result is nightly_manifest.py publish, which lists one page; no paginated prune exists to reuse"}]},"cuda":{"status":"consulted","trigger_reason":"The S3.B path trigger fires on .github/workflows/cuda-nightly.yml. Its one hunk turns bash -e off before an existing serve-route GPU test step so the RED control exit 101 reaches its assertions; it changes no kernel, launch, stream or device code. The only claim adjacent to device behaviour is that exit 101 comes from the Rust test harness, not from the CUDA runtime, so that was asked.","queries":[{"q":"process exit code when a CUDA host program test harness panics: does the CUDA runtime define or alter the host process exit status (e.g. 101) on context teardown?","result":"no-authority-found","source_uri":"mcp__nvidia-cuda-docs__search_cuda_docs","claim_checked":"Exit 101 in the step is the Rust test harness panic code. The corpus returned only sample programs that return 0 or 1 themselves, nothing saying the CUDA runtime sets or alters a host exit status, so the claim stands on the Rust side and no finding is raised."}]},"crux":{"status":"not-triggered","trigger_reason":"the diff touches no CLI subcommand or flag, HTTP route, MCP tool, config key or output format; it adds a script run by one nightly workflow step"},"mutation":{"status":"consulted","scope":"guard","attempted":10,"killed":10,"survivors":[]},"antigravity":{"status":"unreachable","attempted":0,"reason":"gemini bucket state empty (budget 0); the budget probe was run and no agy call was made"}},"findings_ref":{"path":"findings.sarif","sha256":"55eabb47fd58047b1e9f63954fded43cdbd1b57a80254a2471390cd18698132f"},"cost":{"input_tokens":0,"output_tokens":0,"wall_seconds":0},"diff_patch_id":"02d78de2f774ff5f864c17e4386daf75614a72a2","diff_patch_id_algo":"git-patch-id-verbatim/pinned-diff-v1"}}