fix(pv-pin): failed 'pv --version' probe reads COULD NOT CHECK (rc 75), not STALE (#5755)

ci/sections.yml

@@ -2261,7 +2261,7 @@ jobs:
       # `cargo run` pv WITHOUT pv_bin.sh's features, a second compile on every run
       # that this comment said did not happen.)
       - name: pv_bin.sh must resolve the HEAD-built pv and refuse a stale one
-        run: setsid --wait bash scripts/check_pv_bin_resolution.sh
+        run: setsid --wait bash scripts/rerun_on_75.sh setsid --wait bash scripts/check_pv_bin_resolution.sh
       # #4475: the contract gate, blocking. `make contracts` stopped after its first step (an
       # unconditional `exit` under .ONESHELL) and nothing on a PR ran `extract --check`, so a contract
       # could land without its census / contracts.nt / shapes.ttl regenerated. The shapes step fails

scripts/check_pv_bin_resolution.sh

@@ -84,9 +84,15 @@ fi
 resolve_rc=0
 (
     set -euo pipefail
-    . scripts/pv_bin.sh || exit 1
+    . scripts/pv_bin.sh || exit "$?"
     test -x "$PV"
 ) >"$TMP/resolve.log" 2>&1 || resolve_rc=$?
+if [ "$resolve_rc" -eq 75 ]; then
+    # #5755: the probe could not answer. Exit 75 so scripts/rerun_on_75.sh reruns the step once.
+    sed 's/^/      /' "$TMP/resolve.log" >&2
+    echo "COULD NOT CHECK: pv_bin.sh exited 75" >&2
+    exit 75
+fi
 if [ "$resolve_rc" -ne 0 ]; then
     bad "pv_bin.sh did not resolve an executable pv under 'set -euo pipefail'"
     sed 's/^/      /' "$TMP/resolve.log" >&2
@@ -439,6 +445,18 @@ stage_pv() {
     # bashrs reads the parens of "(aprender ...)" as an unescaped test
     # expression (SC1028) when a string holding them shares a line with a test.
     printf '#!/usr/bin/env bash\n' > "$sp_path"
+    if [ "$sp_want" = "flaky" ]; then
+        # Dies on the first probe, answers from the second (a transient kill).
+        printf 'c="$0.n"; n=$(cat "$c" 2>/dev/null || echo 0); echo $((n+1)) > "$c"\n' >> "$sp_path"
+        printf '[ "$n" -ge 1 ] || exit 137\n' >> "$sp_path"
+    fi
+    if [ "$sp_want" = "dead" ]; then
+        # A probe that dies without answering (SIGKILL under load): no output, rc 137.
+        printf 'exit 137\n' >> "$sp_path"
+        chmod +x "$sp_path"
+        printf '%s: %s/crates/aprender-contracts-cli/src/main.rs\n' "$sp_path" "$sp_dir" > "$sp_path.d"
+        return 0
+    fi
     printf 'printf "pv %s %s\\n"\n' "$sp_ver" "$PV_IDENTITY" >> "$sp_path"
     chmod +x "$sp_path"
 
@@ -601,6 +619,30 @@ row '4h foreign good release + own good debug'  good self good other debug
 # A foreign candidate is skipped even when the only own one is old, and the
 # report is about the own one.
 row '4i foreign good release + own stale debug' stale self good other FAIL 'STALE'
+# #5755: a probe that did not ANSWER is not a stale binary. It must read could-not-check
+# (rc 2, "COULD NOT CHECK"), never STALE -- a runner fault named as a code cause.
+row_probe_rc() {
+    rp_name="$1"; rp_want="$2"; rp_rc_expect="$3"; rp_msg="$4"
+    rp_dir=$(safe_tmpdir) || { bad "mktemp -d gave an unusable path"; return 0; }
+    make_fixture "$rp_dir" "$rp_want" none none none
+    rp_rc=0
+    rp_got=$(resolve_in "$rp_dir" "$rp_dir/stderr.log") || rp_rc=$?
+    if [ "$rp_rc" -ne "$rp_rc_expect" ]; then bad "$rp_name -> rc $rp_rc, expected $rp_rc_expect"; sed 's/^/      /' "$rp_dir/stderr.log" >&2
+    elif [ -n "$rp_msg" ] && ! grep -qF -- "$rp_msg" "$rp_dir/stderr.log"; then bad "$rp_name -> never said '$rp_msg'"
+    elif [ "$rp_rc_expect" -ne 0 ] && grep -q 'STALE' "$rp_dir/stderr.log"; then bad "$rp_name -> a failed probe was called STALE"
+    else note "$rp_name"; fi
+    rm -rf "${rp_dir:?refusing to rm an empty path}"
+}
+row_probe_rc '4o pv dies on every probe -> COULD NOT CHECK, rc 75' dead 75 'COULD NOT CHECK'
+row_probe_rc '4p pv dies once, answers on the retry -> accepted' flaky 0 ''
+row '4q pv answers a stale version -> still STALE (red only on a confirmed stale binary)' stale self none none FAIL 'STALE'
+
+# #5755: the rerun-once-on-75 wrapper rides in this check, not as its own CI step (C324 rule 7: no new gate).
+if rerun_out=$(bash scripts/rerun_on_75.sh --self-test 2>&1); then
+    note "$rerun_out"
+else
+    bad "rerun_on_75.sh self-test: $rerun_out"
+fi
 
 # --- the tree's own target dir ---------------------------------------------
 # `cargo metadata` answers for the shell asking. The dev shell's `cargo`

scripts/pv_bin.sh

@@ -793,7 +793,29 @@ pv_bin_assert_fresh() {
     # in scripts/check_pv_version_parse.sh.
     # ONE invocation, then both reads off the SAME captured text -- the semver
     # and the identity must describe the same binary and the same run.
-    pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers=""
+    # Up to 3 probes: a transient kill under load must not decide the gate.
+    pv_bin_vers_rc=0
+    pv_bin_try=1
+    while :; do
+        pv_bin_vers_rc=0
+        pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers_rc=$?
+        [ "$pv_bin_vers_rc" -eq 0 ] && break
+        [ "$pv_bin_try" -ge 3 ] && break
+        pv_bin_try=$((pv_bin_try + 1))
+    done
+    # #5755: a probe that did not ANSWER (killed under load, ETXTBSY) is a failed
+    # measurement, not a stale binary. Reading its empty output as a version made
+    # a runner fault look like a code cause. Still failing after 3 probes = rc 75 (EX_TEMPFAIL), the distinct infra code the
+    # step runner reruns once; it never passes.
+    if [ "$pv_bin_vers_rc" -ne 0 ]; then
+        {
+            printf 'COULD NOT CHECK pv VERSION\n'
+            printf '  resolved : %s\n  ' "$pv_bin_b"
+            printf '"pv --version" exited %s on 3 probes, no usable answer; this is a failed probe (runner\n' "$pv_bin_vers_rc"
+            printf '  load, kill or exec race), not evidence the binary is stale. Re-run; no verdict.\n'
+        } >&2
+        return 75
+    fi
     pv_bin_actual=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print $2; exit}') || pv_bin_actual=""
     pv_bin_first=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print}') || pv_bin_first=""
     if [ "$pv_bin_actual" != "$pv_bin_declared" ]; then
@@ -910,7 +932,11 @@ fi
 if [ "$PV_BIN_RC" -ne 0 ]; then
     return 1 2>/dev/null || exit 1
 fi
-pv_bin_assert_fresh "$PV" || return 1 2>/dev/null || exit 1
+PV_BIN_AF_RC=0
+pv_bin_assert_fresh "$PV" || PV_BIN_AF_RC=$?
+if [ "$PV_BIN_AF_RC" -ne 0 ]; then
+    return "$PV_BIN_AF_RC" 2>/dev/null || exit "$PV_BIN_AF_RC"
+fi
 export PV
 
 # ONT-5: pv-sat, the untrusted reasoner, is a second bin of the same package, built by the same

scripts/rerun_on_75.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# rerun_on_75.sh -- run a CI step; exit 75 (EX_TEMPFAIL, "could not check", scripts/pv_bin.sh)
+# means the probe failed, not the code, so rerun the step ONCE. A second 75 stays 75 and says
+# COULD-NOT-CHECK: a check that could not run never passes (#5755, operator ruling 3).
+#
+#   rerun_on_75.sh CMD [ARG...]   run CMD; rerun once on rc 75; exit with the last rc
+#   rerun_on_75.sh --self-test    case table
+set -euo pipefail
+
+INFRA_RC=75
+
+run() {
+    local rc=0
+    "$@" || rc=$?
+    if [ "$rc" -eq "$INFRA_RC" ]; then
+        echo "::warning::step exited $INFRA_RC (could not check); rerunning once" >&2
+        rc=0
+        "$@" || rc=$?
+        if [ "$rc" -eq "$INFRA_RC" ]; then
+            echo "COULD-NOT-CHECK: step exited $INFRA_RC twice; failing, not passing" >&2
+        fi
+    fi
+    return "$rc"
+}
+
+self_test() {
+    local d bad=0
+    d=$(mktemp -d)
+    # shellcheck disable=SC2064
+    trap "rm -rf -- '${d:?}'" RETURN
+    # step.sh <counter-file> <rc1> [rc2 ...]: exits with the Nth rc on its Nth call.
+    cat >"${d:?}/step.sh" <<'STEP'
+#!/usr/bin/env bash
+f=$1; shift
+n=$(cat "$f" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" >"$f"
+idx=$n; [ "$idx" -le "$#" ] || idx=$#
+exit "${!idx}"
+STEP
+    case_() { # name want_rc want_calls rc-sequence...
+        local name=$1 want=$2 calls=$3 rc=0 got
+        shift 3
+        rm -f "${d:?}/n"
+        bash "$0" bash "$d/step.sh" "$d/n" "$@" >/dev/null 2>&1 || rc=$?
+        got=$(cat "$d/n")
+        if [ "$rc" -ne "$want" ] || [ "$got" -ne "$calls" ]; then
+            echo "SELF-TEST FAIL: $name wanted rc=$want calls=$calls, got rc=$rc calls=$got" >&2
+            bad=1
+        fi
+    }
+    case_ '75 then pass: reran once, passes' 0 2 75 0
+    case_ '75 then 75: second 75 fails, not rerun again' 75 2 75 75
+    case_ 'wrong version (rc 1) is red and not rerun' 1 1 1
+    case_ '75 then red: rerun result wins' 1 2 75 1
+    case_ 'a pass is not rerun' 0 1 0
+    if [ "$bad" -eq 0 ]; then
+        echo "rerun_on_75 self-test: 5 cases ok"
+    fi
+    return "$bad"
+}
+
+case "${1:-}" in
+    --self-test) self_test ;;
+    "") echo "usage: $0 CMD [ARG...] | --self-test" >&2; exit 2 ;;
+    *) run "$@" ;;
+esac