From 786c9f659081392b22688c5f957c22b13380a758 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 13:58:45 +0200 Subject: [PATCH 1/5] fix(pv-pin): a pv --version that did not answer read as STALE, a runner fault named as a code cause (#5755) pv_bin_assert_fresh swallowed a nonzero exit into an empty version and printed STALE pv BINARY. A failed probe is now COULD NOT CHECK, rc 2, propagated to the caller. Case row 4o: a pv stub that exits 137 with no output was RED (STALE) before the fix, green after. Refs #5755 Co-Authored-By: Claude Sonnet 5.5 --- scripts/check_pv_bin_resolution.sh | 10 ++++++++++ scripts/pv_bin.sh | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/check_pv_bin_resolution.sh b/scripts/check_pv_bin_resolution.sh index 4d42f6d3d8..e3dd691397 100755 --- a/scripts/check_pv_bin_resolution.sh +++ b/scripts/check_pv_bin_resolution.sh @@ -439,6 +439,13 @@ stage_pv() { # bashrs reads the parens of "(aprender ...)" as an unescaped test # expression (SC1028) when a string holding them shares a line with a test. printf '#!/usr/bin/env bash\n' > "$sp_path" + if [ "$sp_want" = "dead" ]; then + # A probe that dies without answering (SIGKILL under load): no output, rc 137. + printf 'exit 137\n' >> "$sp_path" + chmod +x "$sp_path" + printf '%s: %s/crates/aprender-contracts-cli/src/main.rs\n' "$sp_path" "$sp_dir" > "$sp_path.d" + return 0 + fi printf 'printf "pv %s %s\\n"\n' "$sp_ver" "$PV_IDENTITY" >> "$sp_path" chmod +x "$sp_path" @@ -601,6 +608,9 @@ row '4h foreign good release + own good debug' good self good other debug # A foreign candidate is skipped even when the only own one is old, and the # report is about the own one. row '4i foreign good release + own stale debug' stale self good other FAIL 'STALE' +# #5755: a probe that did not ANSWER is not a stale binary. It must read could-not-check +# (rc 2, "COULD NOT CHECK"), never STALE -- a runner fault named as a code cause. +row '4o own debug pv dies without answering' dead self none none FAIL 'COULD NOT CHECK' # --- the tree's own target dir --------------------------------------------- # `cargo metadata` answers for the shell asking. The dev shell's `cargo` diff --git a/scripts/pv_bin.sh b/scripts/pv_bin.sh index 980c465cb2..67e59b2d02 100644 --- a/scripts/pv_bin.sh +++ b/scripts/pv_bin.sh @@ -793,7 +793,20 @@ pv_bin_assert_fresh() { # in scripts/check_pv_version_parse.sh. # ONE invocation, then both reads off the SAME captured text -- the semver # and the identity must describe the same binary and the same run. - pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers="" + pv_bin_vers_rc=0 + pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers_rc=$? + # #5755: a probe that did not ANSWER (killed under load, ETXTBSY) is a failed + # measurement, not a stale binary. Reading its empty output as a version made + # a runner fault look like a code cause. Could-not-check is rc 2, named so. + if [ "$pv_bin_vers_rc" -ne 0 ]; then + { + printf 'COULD NOT CHECK pv VERSION\n' + printf ' resolved : %s\n ' "$pv_bin_b" + printf '"pv --version" exited %s without a usable answer; this is a failed probe (runner\n' "$pv_bin_vers_rc" + printf ' load, kill or exec race), not evidence the binary is stale. Re-run; no verdict.\n' + } >&2 + return 2 + fi pv_bin_actual=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print $2; exit}') || pv_bin_actual="" pv_bin_first=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print}') || pv_bin_first="" if [ "$pv_bin_actual" != "$pv_bin_declared" ]; then @@ -910,7 +923,11 @@ fi if [ "$PV_BIN_RC" -ne 0 ]; then return 1 2>/dev/null || exit 1 fi -pv_bin_assert_fresh "$PV" || return 1 2>/dev/null || exit 1 +PV_BIN_AF_RC=0 +pv_bin_assert_fresh "$PV" || PV_BIN_AF_RC=$? +if [ "$PV_BIN_AF_RC" -ne 0 ]; then + return "$PV_BIN_AF_RC" 2>/dev/null || exit "$PV_BIN_AF_RC" +fi export PV # ONT-5: pv-sat, the untrusted reasoner, is a second bin of the same package, built by the same From afdcd1b32cb4bb7b95aabc54534dcbe2b941c778 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 14:09:35 +0200 Subject: [PATCH 2/5] fix(pv-pin): retry the version probe 3x; still failing = rc 75 (EX_TEMPFAIL), red only on a confirmed stale binary (#5755) Operator semantics: could-not-check never passes and is a distinct infra exit code; STALE stays for a binary that answered with the wrong version. Cases 4o (dies every probe -> rc 75), 4p (dies once, answers -> accepted), 4q (answers stale -> STALE). Refs #5755 Co-Authored-By: Claude Sonnet 5.5 --- scripts/check_pv_bin_resolution.sh | 21 ++++++++++++++++++++- scripts/pv_bin.sh | 17 +++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/scripts/check_pv_bin_resolution.sh b/scripts/check_pv_bin_resolution.sh index e3dd691397..2ef1b227aa 100755 --- a/scripts/check_pv_bin_resolution.sh +++ b/scripts/check_pv_bin_resolution.sh @@ -439,6 +439,11 @@ stage_pv() { # bashrs reads the parens of "(aprender ...)" as an unescaped test # expression (SC1028) when a string holding them shares a line with a test. printf '#!/usr/bin/env bash\n' > "$sp_path" + if [ "$sp_want" = "flaky" ]; then + # Dies on the first probe, answers from the second (a transient kill). + printf 'c="$0.n"; n=$(cat "$c" 2>/dev/null || echo 0); echo $((n+1)) > "$c"\n' >> "$sp_path" + printf '[ "$n" -ge 1 ] || exit 137\n' >> "$sp_path" + fi if [ "$sp_want" = "dead" ]; then # A probe that dies without answering (SIGKILL under load): no output, rc 137. printf 'exit 137\n' >> "$sp_path" @@ -610,7 +615,21 @@ row '4h foreign good release + own good debug' good self good other debug row '4i foreign good release + own stale debug' stale self good other FAIL 'STALE' # #5755: a probe that did not ANSWER is not a stale binary. It must read could-not-check # (rc 2, "COULD NOT CHECK"), never STALE -- a runner fault named as a code cause. -row '4o own debug pv dies without answering' dead self none none FAIL 'COULD NOT CHECK' +row_probe_rc() { + rp_name="$1"; rp_want="$2"; rp_rc_expect="$3"; rp_msg="$4" + rp_dir=$(safe_tmpdir) || { bad "mktemp -d gave an unusable path"; return 0; } + make_fixture "$rp_dir" "$rp_want" none none none + rp_rc=0 + rp_got=$(resolve_in "$rp_dir" "$rp_dir/stderr.log") || rp_rc=$? + if [ "$rp_rc" -ne "$rp_rc_expect" ]; then bad "$rp_name -> rc $rp_rc, expected $rp_rc_expect"; sed 's/^/ /' "$rp_dir/stderr.log" >&2 + elif [ -n "$rp_msg" ] && ! grep -qF -- "$rp_msg" "$rp_dir/stderr.log"; then bad "$rp_name -> never said '$rp_msg'" + elif [ "$rp_rc_expect" -ne 0 ] && grep -q 'STALE' "$rp_dir/stderr.log"; then bad "$rp_name -> a failed probe was called STALE" + else note "$rp_name"; fi + rm -rf "${rp_dir:?refusing to rm an empty path}" +} +row_probe_rc '4o pv dies on every probe -> COULD NOT CHECK, rc 75' dead 75 'COULD NOT CHECK' +row_probe_rc '4p pv dies once, answers on the retry -> accepted' flaky 0 '' +row '4q pv answers a stale version -> still STALE (red only on a confirmed stale binary)' stale self none none FAIL 'STALE' # --- the tree's own target dir --------------------------------------------- # `cargo metadata` answers for the shell asking. The dev shell's `cargo` diff --git a/scripts/pv_bin.sh b/scripts/pv_bin.sh index 67e59b2d02..d5f2db1eab 100644 --- a/scripts/pv_bin.sh +++ b/scripts/pv_bin.sh @@ -793,19 +793,28 @@ pv_bin_assert_fresh() { # in scripts/check_pv_version_parse.sh. # ONE invocation, then both reads off the SAME captured text -- the semver # and the identity must describe the same binary and the same run. + # Up to 3 probes: a transient kill under load must not decide the gate. pv_bin_vers_rc=0 - pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers_rc=$? + pv_bin_try=1 + while :; do + pv_bin_vers_rc=0 + pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers_rc=$? + [ "$pv_bin_vers_rc" -eq 0 ] && break + [ "$pv_bin_try" -ge 3 ] && break + pv_bin_try=$((pv_bin_try + 1)) + done # #5755: a probe that did not ANSWER (killed under load, ETXTBSY) is a failed # measurement, not a stale binary. Reading its empty output as a version made - # a runner fault look like a code cause. Could-not-check is rc 2, named so. + # a runner fault look like a code cause. Still failing after 3 probes = rc 75 (EX_TEMPFAIL), the distinct infra code the + # step runner reruns once; it never passes. if [ "$pv_bin_vers_rc" -ne 0 ]; then { printf 'COULD NOT CHECK pv VERSION\n' printf ' resolved : %s\n ' "$pv_bin_b" - printf '"pv --version" exited %s without a usable answer; this is a failed probe (runner\n' "$pv_bin_vers_rc" + printf '"pv --version" exited %s on 3 probes, no usable answer; this is a failed probe (runner\n' "$pv_bin_vers_rc" printf ' load, kill or exec race), not evidence the binary is stale. Re-run; no verdict.\n' } >&2 - return 2 + return 75 fi pv_bin_actual=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print $2; exit}') || pv_bin_actual="" pv_bin_first=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print}') || pv_bin_first="" From 6c9ebb721de02f9924cc6ba91a7bb8ec8cc4ac07 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 15:14:28 +0200 Subject: [PATCH 3/5] ci(pv-pin): rerun the pv-pin step once on rc 75, COULD-NOT-CHECK if still 75 (Refs #5755) Shell wrapper + case table instead of fat_driver.py (complexity hook, no new Python). check_pv_bin_resolution.sh now propagates pv_bin.sh's 75. Agent: aprender-w5018 Co-Authored-By: Claude Sonnet 5.5 --- ci/sections.yml | 4 +- scripts/check_pv_bin_resolution.sh | 8 +++- scripts/rerun_on_75.sh | 65 ++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100755 scripts/rerun_on_75.sh diff --git a/ci/sections.yml b/ci/sections.yml index e2bf1475da..c70f7f81af 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -2179,7 +2179,9 @@ jobs: # `cargo run` pv WITHOUT pv_bin.sh's features, a second compile on every run # that this comment said did not happen.) - name: pv_bin.sh must resolve the HEAD-built pv and refuse a stale one - run: setsid --wait bash scripts/check_pv_bin_resolution.sh + run: setsid --wait bash scripts/rerun_on_75.sh bash scripts/check_pv_bin_resolution.sh + - name: "rerun-once-on-75 wrapper must still turn RED (case table)" + run: setsid --wait bash scripts/rerun_on_75.sh --self-test # #4475: the contract gate, blocking. `make contracts` stopped after its first step (an # unconditional `exit` under .ONESHELL) and nothing on a PR ran `extract --check`, so a contract # could land without its census / contracts.nt / shapes.ttl regenerated. The shapes step fails diff --git a/scripts/check_pv_bin_resolution.sh b/scripts/check_pv_bin_resolution.sh index 2ef1b227aa..3ce67d9175 100755 --- a/scripts/check_pv_bin_resolution.sh +++ b/scripts/check_pv_bin_resolution.sh @@ -84,9 +84,15 @@ fi resolve_rc=0 ( set -euo pipefail - . scripts/pv_bin.sh || exit 1 + . scripts/pv_bin.sh || exit "$?" test -x "$PV" ) >"$TMP/resolve.log" 2>&1 || resolve_rc=$? +if [ "$resolve_rc" -eq 75 ]; then + # #5755: the probe could not answer. Exit 75 so scripts/rerun_on_75.sh reruns the step once. + sed 's/^/ /' "$TMP/resolve.log" >&2 + echo "COULD NOT CHECK: pv_bin.sh exited 75" >&2 + exit 75 +fi if [ "$resolve_rc" -ne 0 ]; then bad "pv_bin.sh did not resolve an executable pv under 'set -euo pipefail'" sed 's/^/ /' "$TMP/resolve.log" >&2 diff --git a/scripts/rerun_on_75.sh b/scripts/rerun_on_75.sh new file mode 100755 index 0000000000..451053a844 --- /dev/null +++ b/scripts/rerun_on_75.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# rerun_on_75.sh -- run a CI step; exit 75 (EX_TEMPFAIL, "could not check", scripts/pv_bin.sh) +# means the probe failed, not the code, so rerun the step ONCE. A second 75 stays 75 and says +# COULD-NOT-CHECK: a check that could not run never passes (#5755, operator ruling 3). +# +# rerun_on_75.sh CMD [ARG...] run CMD; rerun once on rc 75; exit with the last rc +# rerun_on_75.sh --self-test case table +set -euo pipefail + +INFRA_RC=75 + +run() { + local rc=0 + "$@" || rc=$? + if [ "$rc" -eq "$INFRA_RC" ]; then + echo "::warning::step exited $INFRA_RC (could not check); rerunning once" >&2 + rc=0 + "$@" || rc=$? + if [ "$rc" -eq "$INFRA_RC" ]; then + echo "COULD-NOT-CHECK: step exited $INFRA_RC twice; failing, not passing" >&2 + fi + fi + return "$rc" +} + +self_test() { + local d bad=0 + d=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf -- '${d:?}'" RETURN + # step.sh [rc2 ...]: exits with the Nth rc on its Nth call. + cat >"${d:?}/step.sh" <<'STEP' +#!/usr/bin/env bash +f=$1; shift +n=$(cat "$f" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" >"$f" +idx=$n; [ "$idx" -le "$#" ] || idx=$# +exit "${!idx}" +STEP + case_() { # name want_rc want_calls rc-sequence... + local name=$1 want=$2 calls=$3 rc=0 got + shift 3 + rm -f "${d:?}/n" + bash "$0" bash "$d/step.sh" "$d/n" "$@" >/dev/null 2>&1 || rc=$? + got=$(cat "$d/n") + if [ "$rc" -ne "$want" ] || [ "$got" -ne "$calls" ]; then + echo "SELF-TEST FAIL: $name wanted rc=$want calls=$calls, got rc=$rc calls=$got" >&2 + bad=1 + fi + } + case_ '75 then pass: reran once, passes' 0 2 75 0 + case_ '75 then 75: second 75 fails, not rerun again' 75 2 75 75 + case_ 'wrong version (rc 1) is red and not rerun' 1 1 1 + case_ '75 then red: rerun result wins' 1 2 75 1 + case_ 'a pass is not rerun' 0 1 0 + if [ "$bad" -eq 0 ]; then + echo "rerun_on_75 self-test: 5 cases ok" + fi + return "$bad" +} + +case "${1:-}" in + --self-test) self_test ;; + "") echo "usage: $0 CMD [ARG...] | --self-test" >&2; exit 2 ;; + *) run "$@" ;; +esac From 8ef9b14a4937e270ce6e05e1240f37097a1f515e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 16:14:29 +0200 Subject: [PATCH 4/5] ci(pv-pin): fold the rerun_on_75 self-test into check_pv_bin_resolution.sh, no new gate step (Refs #5755) C324 rule 7: a new blocking CI step is a new gate. The existing step runs the case table. Agent: aprender-w5018 Co-Authored-By: Claude Sonnet 5.5 --- ci/sections.yml | 2 -- scripts/check_pv_bin_resolution.sh | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/sections.yml b/ci/sections.yml index c70f7f81af..2fd4429d2d 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -2180,8 +2180,6 @@ jobs: # that this comment said did not happen.) - name: pv_bin.sh must resolve the HEAD-built pv and refuse a stale one run: setsid --wait bash scripts/rerun_on_75.sh bash scripts/check_pv_bin_resolution.sh - - name: "rerun-once-on-75 wrapper must still turn RED (case table)" - run: setsid --wait bash scripts/rerun_on_75.sh --self-test # #4475: the contract gate, blocking. `make contracts` stopped after its first step (an # unconditional `exit` under .ONESHELL) and nothing on a PR ran `extract --check`, so a contract # could land without its census / contracts.nt / shapes.ttl regenerated. The shapes step fails diff --git a/scripts/check_pv_bin_resolution.sh b/scripts/check_pv_bin_resolution.sh index 3ce67d9175..30024d006d 100755 --- a/scripts/check_pv_bin_resolution.sh +++ b/scripts/check_pv_bin_resolution.sh @@ -637,6 +637,13 @@ row_probe_rc '4o pv dies on every probe -> COULD NOT CHECK, rc 75' dead 75 'COUL row_probe_rc '4p pv dies once, answers on the retry -> accepted' flaky 0 '' row '4q pv answers a stale version -> still STALE (red only on a confirmed stale binary)' stale self none none FAIL 'STALE' +# #5755: the rerun-once-on-75 wrapper rides in this check, not as its own CI step (C324 rule 7: no new gate). +if rerun_out=$(bash scripts/rerun_on_75.sh --self-test 2>&1); then + note "$rerun_out" +else + bad "rerun_on_75.sh self-test: $rerun_out" +fi + # --- the tree's own target dir --------------------------------------------- # `cargo metadata` answers for the shell asking. The dev shell's `cargo` # function moves the target dir; a build launched from a plain bash script does From a70972c4f1f4b6d80e4a36be6cd90a7995e7afef Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 19 Oct 2026 20:24:07 +0200 Subject: [PATCH 5/5] ci: wrapped pv-pin step keeps setsid --wait on the guard invocation (Refs #5755) Agent: aprender-w5018 Co-Authored-By: Claude Sonnet 5.5 --- ci/sections.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/sections.yml b/ci/sections.yml index 58cd6adce2..e517f97d64 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -2233,7 +2233,7 @@ jobs: # `cargo run` pv WITHOUT pv_bin.sh's features, a second compile on every run # that this comment said did not happen.) - name: pv_bin.sh must resolve the HEAD-built pv and refuse a stale one - run: setsid --wait bash scripts/rerun_on_75.sh bash scripts/check_pv_bin_resolution.sh + run: setsid --wait bash scripts/rerun_on_75.sh setsid --wait bash scripts/check_pv_bin_resolution.sh # #4475: the contract gate, blocking. `make contracts` stopped after its first step (an # unconditional `exit` under .ONESHELL) and nothing on a PR ran `extract --check`, so a contract # could land without its census / contracts.nt / shapes.ttl regenerated. The shapes step fails