From 57730c34d0dbf6e98c77d9d0c4e319eada094470 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 20 Sep 2026 18:54:07 +0200 Subject: [PATCH 1/2] PMAT-3587 step 1 (RED): the falsifier for fail-fast discarding verdicts RED FIXTURE FIRST. Only the case table moves; scripts/ci_run_explicit_test_commands.sh is untouched, so these rows fail against the runner as it stands today. `bash scripts/check_explicit_test_commands.sh --self-test` on THIS commit: rc=1, 6 failing of 72 rows (66 ok -- the fixture discriminates, it does not fail everything): FAIL row 30 a failing command no longer stops the run (rc 1 = ran, measured failure) FAIL row 32 ...and the command AFTER it ran too -- no fail-fast FAIL row 33 the receipt reports ran/skipped/total as NUMBERS, never an absence FAIL row 34 ...a command's own rc is NOT the runner's rc (7 must not leak out) FAIL row 35 --fail-fast stops early and is UNKNOWN (rc 2), not Fail FAIL row 37 all ran, none failed -> rc 0 and skipped=0 This is the ticket's own falsifier: plant a failing command early in the list and require the EXACT count that still ran and the EXACT count skipped. A receipt that reports only the failure fails the control. The exit vocabulary these rows assert -- the runner's rc is its OWN verdict and never a passthrough of a command's rc, because propagating one would let a test exiting 2 mean the same thing as "commands were not run": 0 every command ran, none failed 1 every command ran, F >= 1 FAILED (a measured failure) 2 UNMEASURED commands exist (Unknown, doctrine 4) Row 34 exists specifically to pin that: the fixture's failing command exits 7, and the runner must NOT exit 7. Two defects in my own fixture, found by running it rather than reading it, both worth recording because each produced a wrong count: - a malformed `notfact ... ; test $? -eq 7` aborted the self-test after row 33 under `set -e`, hiding rows 34-37. A fail-fast abort concealing later rows, inside the harness written to fix exactly that -- it reported 3 failures when there were 6. - the all-pass fixture used `true` for both fragments; the runner correctly REFUSED a duplicate command (rc 2). The runner was right and the fixture was wrong. Refs #3587, #3581, #3586, #3561, #3080 Pmat-Ticket: PMAT-3587 Co-Authored-By: Claude Opus 5 (1M context) --- scripts/check_explicit_test_commands.sh | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/check_explicit_test_commands.sh b/scripts/check_explicit_test_commands.sh index 25afe18e3f..c18864ac75 100755 --- a/scripts/check_explicit_test_commands.sh +++ b/scripts/check_explicit_test_commands.sh @@ -190,10 +190,36 @@ self_test() { row 2 "REFUSE a directory named like a fragment" 'not a regular file' bash "$R" --list "$td/name" notfact " ...and a refused tree prints NO commands on stdout" test -n "$(bash "$R" --list "$td/name" 2>/dev/null)" # ── the runner: execution ──────────────────────────────────────────── + # PMAT-3587 -- fail-fast is a VERDICT-DISCARDING default. Measured on #3581: one + # test failing on a missing interpreter produced "fail-fast: the 15 command(s) + # after it did not run", and one shard hid 8,576 tests. Fifteen unknowns became + # an absence, and an absence reads as "nothing to report", which is + # indistinguishable from "passed" to every consumer downstream. + # + # The runner's exit code is its OWN vocabulary and never a passthrough of a + # command's rc -- propagating one would let a test that exits 2 mean the same + # thing as "commands were not run": + # 0 every command ran, none failed + # 1 every command ran, F >= 1 FAILED (a measured failure) + # 2 UNMEASURED commands exist (Unknown, doctrine 4) + # + # Ticket falsifier: plant a failing command early, require the EXACT count that + # still ran and the EXACT count skipped. frags "$td/ff" "010-a.cmd=touch $td/a\n" '020-b.cmd=exit 7\n' "030-c.cmd=touch $td/c\n" - row 7 "FAIL-FAST: the first non-zero command's status is the step's status" 'exit 7' bash "$R" --run "$td/ff" + rm -f "$td/a" "$td/c" + row 1 "a failing command no longer stops the run (rc 1 = ran, measured failure)" 'exit 7' bash "$R" --run "$td/ff" fact " ...the command before it ran" test -e "$td/a" - notfact " ...the command after it did NOT run" test -e "$td/c" + fact " ...and the command AFTER it ran too -- no fail-fast" test -e "$td/c" + row 1 "the receipt reports ran/skipped/total as NUMBERS, never an absence" 'RESULT ran=3 skipped=0 total=3 failed=1' bash "$R" --run "$td/ff" + notfact " ...a command's own rc is NOT the runner's rc (7 must not leak out)" \ + bash -c 'bash "$1" --run "$2" > /dev/null 2>&1; [ $? -eq 7 ]' _ "$R" "$td/ff" + # Opt-in fail-fast still exists, and it is UNKNOWN (2), not Fail: the commands it + # did not run are UNMEASURED, and unmeasured is neither green nor red. + rm -f "$td/c" + row 2 "--fail-fast stops early and is UNKNOWN (rc 2), not Fail" 'RESULT ran=2 skipped=1 total=3 failed=1' bash "$R" --run "$td/ff" --fail-fast + notfact " ...the command after the failure did NOT run" test -e "$td/c" + frags "$td/allok" "010-a.cmd=echo alpha\n" "020-b.cmd=echo beta\n" + row 0 "all ran, none failed -> rc 0 and skipped=0" 'RESULT ran=2 skipped=0 total=2 failed=0' bash "$R" --run "$td/allok" frags "$td/in" "010-a.cmd=cat > $td/swallowed\n" "020-b.cmd=touch $td/after\n" row 0 "a command reading stdin gets /dev/null" '^PASS 2/2' bash "$R" --run "$td/in" fact " ...and the next command still ran" test -e "$td/after" From 1cd876418d65380a06ae253ee53fc3ebc4b421c1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 20 Sep 2026 18:55:27 +0200 Subject: [PATCH 2/2] PMAT-3587 step 2 (GREEN): fail-fast off by default; skipped is a number, not an absence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fixture, unchanged. Only scripts/ci_run_explicit_test_commands.sh moves. `--self-test` now rc=0, 72/72 rows. THE DEFAULT FLIPS, rather than a flag being added for the workflow to pass. The operator's framing is "fail-fast is a verdict-discarding DEFAULT", and a flag would leave the caller choosing, per invocation, whether to discard verdicts -- which is the defect restated, not removed. #3581 is what it cost: one test failing on a missing interpreter produced "fail-fast: the 15 command(s) after it did not run", and one shard hid 8,576 tests. Fifteen unknowns became an absence, and an absence is indistinguishable from "passed" to every consumer downstream. default every command runs; a red run still measures everything else --fail-fast opt-in, restores the old stop-at-first behaviour A RESULT line is printed on BOTH paths, always: RESULT ran=N skipped=S total=T failed=F RESULT first-failure [i/T] exit R: skipped is a NUMBER. That is the whole point: an absence reads as "nothing to report"; a 2 reads as two commands nobody measured. EXIT CODE is the runner's OWN vocabulary and never a passthrough of a command's rc. Propagating one would let a test exiting 2 mean the same thing as "commands were not run" -- two meanings, one spelling, the defect this repo keeps finding: 0 every command ran, none failed 1 every command ran, F >= 1 FAILED -- a measured failure 2 UNMEASURED commands exist -- Unknown, doctrine 4; also every refusal Measured end to end on a 4-command fixture whose 2nd exits 7: default ran=4 skipped=0 total=4 failed=1 rc=1 (NOT 7 -- row 34 pins this) --fail-fast ran=2 skipped=2 total=4 failed=1 rc=2 UNKNOWN Callers checked on origin/main before flipping, not locally: the only invocation is ci.yml:634, a plain `run:` step whose rc is the step's rc, so 1 and 2 both fail the step as before and nothing branches on the value. ci.yml is NOT touched by this PR. Arg parsing takes --shard and --fail-fast in either order; both orders measured. Gates, fresh worktree off origin/main 00e5d2dde: check_explicit_test_commands.sh and --self-test rc=0 (72 rows) check_hardcoded_paths.sh --full-if-capable rc=0 check_bashrs_gate.sh · check_apr_bin_pinned.sh rc=0 check_no_pipe_into_grep_q.sh rc=0 check_shell_lint_ratchet.sh rc=0 check_guards_are_wired.sh rc=0 check_sourced_libs_option_neutral.sh rc=0 Refs #3587, #3581, #3586, #3561, #3080 Pmat-Ticket: PMAT-3587 Co-Authored-By: Claude Opus 5 (1M context) --- scripts/ci_run_explicit_test_commands.sh | 51 +++++++++++++++++++----- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/scripts/ci_run_explicit_test_commands.sh b/scripts/ci_run_explicit_test_commands.sh index 71593875b4..877bee6d83 100755 --- a/scripts/ci_run_explicit_test_commands.sh +++ b/scripts/ci_run_explicit_test_commands.sh @@ -21,15 +21,27 @@ # * a missing or empty directory is refused -- zero commands is broken wiring # Any violation: rc 2, nothing is printed on stdout, nothing runs. # -# Execution (--run), held equal to the old `&&` chain: +# Execution (--run): # * commands in order, each as `bash -c "$cmd"` with stdin from /dev/null (the # old docker run had no -i; the list is fully read before anything runs) # * a ::group:: header per command -# * FAIL-FAST: the first non-zero command ends the run with THAT exit status +# * NO FAIL-FAST (PMAT-3587). Every command runs. A red run still measures +# everything else it was going to measure, which is exactly when the data is +# most wanted. `--fail-fast` restores the old stop-at-first behaviour. +# * a RESULT line always: `RESULT ran=N skipped=S total=T failed=F`. The skipped +# count is a NUMBER, never an absence -- an absence reads as "nothing to +# report", which is indistinguishable from "passed" to every consumer. +# +# EXIT CODE -- the runner's own vocabulary, never a passthrough of a command's rc +# (propagating one would let a test exiting 2 mean "commands were not run"): +# 0 every command ran, none failed +# 1 every command ran, F >= 1 FAILED -- a measured failure +# 2 UNMEASURED commands exist -- Unknown, doctrine 4; also every +# refusal (bad tree, bad shard, vacuity) # # Usage: # scripts/ci_run_explicit_test_commands.sh --list [DIR] -# scripts/ci_run_explicit_test_commands.sh --run [DIR] +# scripts/ci_run_explicit_test_commands.sh --run [DIR] [--shard N/M] [--fail-fast] # DIR defaults to ci/explicit-test-commands.d. The case table for this script # lives in scripts/check_explicit_test_commands.sh --self-test. set -euo pipefail @@ -96,7 +108,7 @@ parse() { # list exactly once between them (PACK-001). N and M are validated; a shard that # selects zero commands is a refusal, not a pass. run() { - local dir=$1 shard=${2:-1/1} out cmd i=0 total rc n m k=0 sel=0 + local dir=$1 shard=${2:-1/1} ff=${3:-0} out cmd i=0 total rc n m k=0 sel=0 failed=0 first_fail="" local -a cmds if ! [[ "$shard" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then printf 'REFUSE --shard must look like N/M, got %s\n' "$shard" >&2; return 2 @@ -130,20 +142,41 @@ run() { bash -c "$cmd" < /dev/null || rc=$? printf '::endgroup::\n' if [ "$rc" -ne 0 ]; then + failed=$((failed + 1)) + [ -n "$first_fail" ] || first_fail="[$i/$total] exit $rc: $cmd" printf 'FAIL [%s/%s] exit %s: %s\n' "$i" "$total" "$rc" "$cmd" >&2 - printf ' fail-fast: the %s command(s) after it did not run\n' "$((total - i))" >&2 - return "$rc" + if [ "$ff" -eq 1 ]; then + printf ' --fail-fast: the %s command(s) after it were NOT RUN -- unmeasured, not passed\n' "$((total - i))" >&2 + break + fi fi done + # Always, on both paths: the counts are numbers a consumer can read. + printf 'RESULT ran=%s skipped=%s total=%s failed=%s\n' "$i" "$((total - i))" "$total" "$failed" + [ -n "$first_fail" ] && printf 'RESULT first-failure %s\n' "$first_fail" + if [ "$i" -lt "$total" ]; then + printf 'UNKNOWN %s of %s command(s) were not run; unmeasured is neither pass nor fail\n' "$((total - i))" "$total" >&2 + return 2 + fi + if [ "$failed" -gt 0 ]; then + printf 'FAIL %s of %s explicit test command(s) failed; all %s ran\n' "$failed" "$total" "$total" >&2 + return 1 + fi printf 'PASS %s/%s explicit test command(s) from %s\n' "$i" "$total" "$dir" } case "${1:-}" in --list) parse "${2:-$DEFAULT_DIR}" ;; --run) - dir=${2:-$DEFAULT_DIR}; shard=1/1 - case "${3:-}" in --shard) shard=${4:?--shard needs N/M} ;; "") ;; *) printf 'usage: %s --run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;; esac - run "$dir" "$shard" ;; + dir=${2:-$DEFAULT_DIR}; shard=1/1; ff=0; shift 2 2>/dev/null || shift $# + while [ "$#" -gt 0 ]; do + case "$1" in + --shard) shard=${2:?--shard needs N/M}; shift 2 ;; + --fail-fast) ff=1; shift ;; + *) printf 'usage: %s --run [DIR] [--shard N/M] [--fail-fast]\n' "$0" >&2; exit 2 ;; + esac + done + run "$dir" "$shard" "$ff" ;; -h|--help) sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' ;; *) printf 'usage: %s --list|--run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;; esac