diff --git a/ci/sections.yml b/ci/sections.yml index cdb4bea410..754569791f 100644 --- a/ci/sections.yml +++ b/ci/sections.yml @@ -2261,7 +2261,7 @@ jobs: # `cargo run` pv WITHOUT pv_bin.sh's features, a second compile on every run # that this comment said did not happen.) - name: pv_bin.sh must resolve the HEAD-built pv and refuse a stale one - run: setsid --wait bash scripts/check_pv_bin_resolution.sh + run: setsid --wait bash scripts/rerun_on_75.sh setsid --wait bash scripts/check_pv_bin_resolution.sh # #4475: the contract gate, blocking. `make contracts` stopped after its first step (an # unconditional `exit` under .ONESHELL) and nothing on a PR ran `extract --check`, so a contract # could land without its census / contracts.nt / shapes.ttl regenerated. The shapes step fails diff --git a/scripts/check_pv_bin_resolution.sh b/scripts/check_pv_bin_resolution.sh index 4d42f6d3d8..30024d006d 100755 --- a/scripts/check_pv_bin_resolution.sh +++ b/scripts/check_pv_bin_resolution.sh @@ -84,9 +84,15 @@ fi resolve_rc=0 ( set -euo pipefail - . scripts/pv_bin.sh || exit 1 + . scripts/pv_bin.sh || exit "$?" test -x "$PV" ) >"$TMP/resolve.log" 2>&1 || resolve_rc=$? +if [ "$resolve_rc" -eq 75 ]; then + # #5755: the probe could not answer. Exit 75 so scripts/rerun_on_75.sh reruns the step once. + sed 's/^/ /' "$TMP/resolve.log" >&2 + echo "COULD NOT CHECK: pv_bin.sh exited 75" >&2 + exit 75 +fi if [ "$resolve_rc" -ne 0 ]; then bad "pv_bin.sh did not resolve an executable pv under 'set -euo pipefail'" sed 's/^/ /' "$TMP/resolve.log" >&2 @@ -439,6 +445,18 @@ stage_pv() { # bashrs reads the parens of "(aprender ...)" as an unescaped test # expression (SC1028) when a string holding them shares a line with a test. printf '#!/usr/bin/env bash\n' > "$sp_path" + if [ "$sp_want" = "flaky" ]; then + # Dies on the first probe, answers from the second (a transient kill). + printf 'c="$0.n"; n=$(cat "$c" 2>/dev/null || echo 0); echo $((n+1)) > "$c"\n' >> "$sp_path" + printf '[ "$n" -ge 1 ] || exit 137\n' >> "$sp_path" + fi + if [ "$sp_want" = "dead" ]; then + # A probe that dies without answering (SIGKILL under load): no output, rc 137. + printf 'exit 137\n' >> "$sp_path" + chmod +x "$sp_path" + printf '%s: %s/crates/aprender-contracts-cli/src/main.rs\n' "$sp_path" "$sp_dir" > "$sp_path.d" + return 0 + fi printf 'printf "pv %s %s\\n"\n' "$sp_ver" "$PV_IDENTITY" >> "$sp_path" chmod +x "$sp_path" @@ -601,6 +619,30 @@ row '4h foreign good release + own good debug' good self good other debug # A foreign candidate is skipped even when the only own one is old, and the # report is about the own one. row '4i foreign good release + own stale debug' stale self good other FAIL 'STALE' +# #5755: a probe that did not ANSWER is not a stale binary. It must read could-not-check +# (rc 2, "COULD NOT CHECK"), never STALE -- a runner fault named as a code cause. +row_probe_rc() { + rp_name="$1"; rp_want="$2"; rp_rc_expect="$3"; rp_msg="$4" + rp_dir=$(safe_tmpdir) || { bad "mktemp -d gave an unusable path"; return 0; } + make_fixture "$rp_dir" "$rp_want" none none none + rp_rc=0 + rp_got=$(resolve_in "$rp_dir" "$rp_dir/stderr.log") || rp_rc=$? + if [ "$rp_rc" -ne "$rp_rc_expect" ]; then bad "$rp_name -> rc $rp_rc, expected $rp_rc_expect"; sed 's/^/ /' "$rp_dir/stderr.log" >&2 + elif [ -n "$rp_msg" ] && ! grep -qF -- "$rp_msg" "$rp_dir/stderr.log"; then bad "$rp_name -> never said '$rp_msg'" + elif [ "$rp_rc_expect" -ne 0 ] && grep -q 'STALE' "$rp_dir/stderr.log"; then bad "$rp_name -> a failed probe was called STALE" + else note "$rp_name"; fi + rm -rf "${rp_dir:?refusing to rm an empty path}" +} +row_probe_rc '4o pv dies on every probe -> COULD NOT CHECK, rc 75' dead 75 'COULD NOT CHECK' +row_probe_rc '4p pv dies once, answers on the retry -> accepted' flaky 0 '' +row '4q pv answers a stale version -> still STALE (red only on a confirmed stale binary)' stale self none none FAIL 'STALE' + +# #5755: the rerun-once-on-75 wrapper rides in this check, not as its own CI step (C324 rule 7: no new gate). +if rerun_out=$(bash scripts/rerun_on_75.sh --self-test 2>&1); then + note "$rerun_out" +else + bad "rerun_on_75.sh self-test: $rerun_out" +fi # --- the tree's own target dir --------------------------------------------- # `cargo metadata` answers for the shell asking. The dev shell's `cargo` diff --git a/scripts/pv_bin.sh b/scripts/pv_bin.sh index 980c465cb2..d5f2db1eab 100644 --- a/scripts/pv_bin.sh +++ b/scripts/pv_bin.sh @@ -793,7 +793,29 @@ pv_bin_assert_fresh() { # in scripts/check_pv_version_parse.sh. # ONE invocation, then both reads off the SAME captured text -- the semver # and the identity must describe the same binary and the same run. - pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers="" + # Up to 3 probes: a transient kill under load must not decide the gate. + pv_bin_vers_rc=0 + pv_bin_try=1 + while :; do + pv_bin_vers_rc=0 + pv_bin_vers=$("$pv_bin_b" --version 2>&1) || pv_bin_vers_rc=$? + [ "$pv_bin_vers_rc" -eq 0 ] && break + [ "$pv_bin_try" -ge 3 ] && break + pv_bin_try=$((pv_bin_try + 1)) + done + # #5755: a probe that did not ANSWER (killed under load, ETXTBSY) is a failed + # measurement, not a stale binary. Reading its empty output as a version made + # a runner fault look like a code cause. Still failing after 3 probes = rc 75 (EX_TEMPFAIL), the distinct infra code the + # step runner reruns once; it never passes. + if [ "$pv_bin_vers_rc" -ne 0 ]; then + { + printf 'COULD NOT CHECK pv VERSION\n' + printf ' resolved : %s\n ' "$pv_bin_b" + printf '"pv --version" exited %s on 3 probes, no usable answer; this is a failed probe (runner\n' "$pv_bin_vers_rc" + printf ' load, kill or exec race), not evidence the binary is stale. Re-run; no verdict.\n' + } >&2 + return 75 + fi pv_bin_actual=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print $2; exit}') || pv_bin_actual="" pv_bin_first=$(printf '%s\n' "$pv_bin_vers" | awk 'NR==1{print}') || pv_bin_first="" if [ "$pv_bin_actual" != "$pv_bin_declared" ]; then @@ -910,7 +932,11 @@ fi if [ "$PV_BIN_RC" -ne 0 ]; then return 1 2>/dev/null || exit 1 fi -pv_bin_assert_fresh "$PV" || return 1 2>/dev/null || exit 1 +PV_BIN_AF_RC=0 +pv_bin_assert_fresh "$PV" || PV_BIN_AF_RC=$? +if [ "$PV_BIN_AF_RC" -ne 0 ]; then + return "$PV_BIN_AF_RC" 2>/dev/null || exit "$PV_BIN_AF_RC" +fi export PV # ONT-5: pv-sat, the untrusted reasoner, is a second bin of the same package, built by the same diff --git a/scripts/rerun_on_75.sh b/scripts/rerun_on_75.sh new file mode 100755 index 0000000000..451053a844 --- /dev/null +++ b/scripts/rerun_on_75.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# rerun_on_75.sh -- run a CI step; exit 75 (EX_TEMPFAIL, "could not check", scripts/pv_bin.sh) +# means the probe failed, not the code, so rerun the step ONCE. A second 75 stays 75 and says +# COULD-NOT-CHECK: a check that could not run never passes (#5755, operator ruling 3). +# +# rerun_on_75.sh CMD [ARG...] run CMD; rerun once on rc 75; exit with the last rc +# rerun_on_75.sh --self-test case table +set -euo pipefail + +INFRA_RC=75 + +run() { + local rc=0 + "$@" || rc=$? + if [ "$rc" -eq "$INFRA_RC" ]; then + echo "::warning::step exited $INFRA_RC (could not check); rerunning once" >&2 + rc=0 + "$@" || rc=$? + if [ "$rc" -eq "$INFRA_RC" ]; then + echo "COULD-NOT-CHECK: step exited $INFRA_RC twice; failing, not passing" >&2 + fi + fi + return "$rc" +} + +self_test() { + local d bad=0 + d=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf -- '${d:?}'" RETURN + # step.sh [rc2 ...]: exits with the Nth rc on its Nth call. + cat >"${d:?}/step.sh" <<'STEP' +#!/usr/bin/env bash +f=$1; shift +n=$(cat "$f" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" >"$f" +idx=$n; [ "$idx" -le "$#" ] || idx=$# +exit "${!idx}" +STEP + case_() { # name want_rc want_calls rc-sequence... + local name=$1 want=$2 calls=$3 rc=0 got + shift 3 + rm -f "${d:?}/n" + bash "$0" bash "$d/step.sh" "$d/n" "$@" >/dev/null 2>&1 || rc=$? + got=$(cat "$d/n") + if [ "$rc" -ne "$want" ] || [ "$got" -ne "$calls" ]; then + echo "SELF-TEST FAIL: $name wanted rc=$want calls=$calls, got rc=$rc calls=$got" >&2 + bad=1 + fi + } + case_ '75 then pass: reran once, passes' 0 2 75 0 + case_ '75 then 75: second 75 fails, not rerun again' 75 2 75 75 + case_ 'wrong version (rc 1) is red and not rerun' 1 1 1 + case_ '75 then red: rerun result wins' 1 2 75 1 + case_ 'a pass is not rerun' 0 1 0 + if [ "$bad" -eq 0 ]; then + echo "rerun_on_75 self-test: 5 cases ok" + fi + return "$bad" +} + +case "${1:-}" in + --self-test) self_test ;; + "") echo "usage: $0 CMD [ARG...] | --self-test" >&2; exit 2 ;; + *) run "$@" ;; +esac