diff --git a/scripts/check_explicit_test_commands.sh b/scripts/check_explicit_test_commands.sh index 25afe18e3f..c18864ac75 100755 --- a/scripts/check_explicit_test_commands.sh +++ b/scripts/check_explicit_test_commands.sh @@ -190,10 +190,36 @@ self_test() { row 2 "REFUSE a directory named like a fragment" 'not a regular file' bash "$R" --list "$td/name" notfact " ...and a refused tree prints NO commands on stdout" test -n "$(bash "$R" --list "$td/name" 2>/dev/null)" # ── the runner: execution ──────────────────────────────────────────── + # PMAT-3587 -- fail-fast is a VERDICT-DISCARDING default. Measured on #3581: one + # test failing on a missing interpreter produced "fail-fast: the 15 command(s) + # after it did not run", and one shard hid 8,576 tests. Fifteen unknowns became + # an absence, and an absence reads as "nothing to report", which is + # indistinguishable from "passed" to every consumer downstream. + # + # The runner's exit code is its OWN vocabulary and never a passthrough of a + # command's rc -- propagating one would let a test that exits 2 mean the same + # thing as "commands were not run": + # 0 every command ran, none failed + # 1 every command ran, F >= 1 FAILED (a measured failure) + # 2 UNMEASURED commands exist (Unknown, doctrine 4) + # + # Ticket falsifier: plant a failing command early, require the EXACT count that + # still ran and the EXACT count skipped. frags "$td/ff" "010-a.cmd=touch $td/a\n" '020-b.cmd=exit 7\n' "030-c.cmd=touch $td/c\n" - row 7 "FAIL-FAST: the first non-zero command's status is the step's status" 'exit 7' bash "$R" --run "$td/ff" + rm -f "$td/a" "$td/c" + row 1 "a failing command no longer stops the run (rc 1 = ran, measured failure)" 'exit 7' bash "$R" --run "$td/ff" fact " ...the command before it ran" test -e "$td/a" - notfact " ...the command after it did NOT run" test -e "$td/c" + fact " ...and the command AFTER it ran too -- no fail-fast" test -e "$td/c" + row 1 "the receipt reports ran/skipped/total as NUMBERS, never an absence" 'RESULT ran=3 skipped=0 total=3 failed=1' bash "$R" --run "$td/ff" + notfact " ...a command's own rc is NOT the runner's rc (7 must not leak out)" \ + bash -c 'bash "$1" --run "$2" > /dev/null 2>&1; [ $? -eq 7 ]' _ "$R" "$td/ff" + # Opt-in fail-fast still exists, and it is UNKNOWN (2), not Fail: the commands it + # did not run are UNMEASURED, and unmeasured is neither green nor red. + rm -f "$td/c" + row 2 "--fail-fast stops early and is UNKNOWN (rc 2), not Fail" 'RESULT ran=2 skipped=1 total=3 failed=1' bash "$R" --run "$td/ff" --fail-fast + notfact " ...the command after the failure did NOT run" test -e "$td/c" + frags "$td/allok" "010-a.cmd=echo alpha\n" "020-b.cmd=echo beta\n" + row 0 "all ran, none failed -> rc 0 and skipped=0" 'RESULT ran=2 skipped=0 total=2 failed=0' bash "$R" --run "$td/allok" frags "$td/in" "010-a.cmd=cat > $td/swallowed\n" "020-b.cmd=touch $td/after\n" row 0 "a command reading stdin gets /dev/null" '^PASS 2/2' bash "$R" --run "$td/in" fact " ...and the next command still ran" test -e "$td/after" diff --git a/scripts/ci_run_explicit_test_commands.sh b/scripts/ci_run_explicit_test_commands.sh index 71593875b4..877bee6d83 100755 --- a/scripts/ci_run_explicit_test_commands.sh +++ b/scripts/ci_run_explicit_test_commands.sh @@ -21,15 +21,27 @@ # * a missing or empty directory is refused -- zero commands is broken wiring # Any violation: rc 2, nothing is printed on stdout, nothing runs. # -# Execution (--run), held equal to the old `&&` chain: +# Execution (--run): # * commands in order, each as `bash -c "$cmd"` with stdin from /dev/null (the # old docker run had no -i; the list is fully read before anything runs) # * a ::group:: header per command -# * FAIL-FAST: the first non-zero command ends the run with THAT exit status +# * NO FAIL-FAST (PMAT-3587). Every command runs. A red run still measures +# everything else it was going to measure, which is exactly when the data is +# most wanted. `--fail-fast` restores the old stop-at-first behaviour. +# * a RESULT line always: `RESULT ran=N skipped=S total=T failed=F`. The skipped +# count is a NUMBER, never an absence -- an absence reads as "nothing to +# report", which is indistinguishable from "passed" to every consumer. +# +# EXIT CODE -- the runner's own vocabulary, never a passthrough of a command's rc +# (propagating one would let a test exiting 2 mean "commands were not run"): +# 0 every command ran, none failed +# 1 every command ran, F >= 1 FAILED -- a measured failure +# 2 UNMEASURED commands exist -- Unknown, doctrine 4; also every +# refusal (bad tree, bad shard, vacuity) # # Usage: # scripts/ci_run_explicit_test_commands.sh --list [DIR] -# scripts/ci_run_explicit_test_commands.sh --run [DIR] +# scripts/ci_run_explicit_test_commands.sh --run [DIR] [--shard N/M] [--fail-fast] # DIR defaults to ci/explicit-test-commands.d. The case table for this script # lives in scripts/check_explicit_test_commands.sh --self-test. set -euo pipefail @@ -96,7 +108,7 @@ parse() { # list exactly once between them (PACK-001). N and M are validated; a shard that # selects zero commands is a refusal, not a pass. run() { - local dir=$1 shard=${2:-1/1} out cmd i=0 total rc n m k=0 sel=0 + local dir=$1 shard=${2:-1/1} ff=${3:-0} out cmd i=0 total rc n m k=0 sel=0 failed=0 first_fail="" local -a cmds if ! [[ "$shard" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then printf 'REFUSE --shard must look like N/M, got %s\n' "$shard" >&2; return 2 @@ -130,20 +142,41 @@ run() { bash -c "$cmd" < /dev/null || rc=$? printf '::endgroup::\n' if [ "$rc" -ne 0 ]; then + failed=$((failed + 1)) + [ -n "$first_fail" ] || first_fail="[$i/$total] exit $rc: $cmd" printf 'FAIL [%s/%s] exit %s: %s\n' "$i" "$total" "$rc" "$cmd" >&2 - printf ' fail-fast: the %s command(s) after it did not run\n' "$((total - i))" >&2 - return "$rc" + if [ "$ff" -eq 1 ]; then + printf ' --fail-fast: the %s command(s) after it were NOT RUN -- unmeasured, not passed\n' "$((total - i))" >&2 + break + fi fi done + # Always, on both paths: the counts are numbers a consumer can read. + printf 'RESULT ran=%s skipped=%s total=%s failed=%s\n' "$i" "$((total - i))" "$total" "$failed" + [ -n "$first_fail" ] && printf 'RESULT first-failure %s\n' "$first_fail" + if [ "$i" -lt "$total" ]; then + printf 'UNKNOWN %s of %s command(s) were not run; unmeasured is neither pass nor fail\n' "$((total - i))" "$total" >&2 + return 2 + fi + if [ "$failed" -gt 0 ]; then + printf 'FAIL %s of %s explicit test command(s) failed; all %s ran\n' "$failed" "$total" "$total" >&2 + return 1 + fi printf 'PASS %s/%s explicit test command(s) from %s\n' "$i" "$total" "$dir" } case "${1:-}" in --list) parse "${2:-$DEFAULT_DIR}" ;; --run) - dir=${2:-$DEFAULT_DIR}; shard=1/1 - case "${3:-}" in --shard) shard=${4:?--shard needs N/M} ;; "") ;; *) printf 'usage: %s --run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;; esac - run "$dir" "$shard" ;; + dir=${2:-$DEFAULT_DIR}; shard=1/1; ff=0; shift 2 2>/dev/null || shift $# + while [ "$#" -gt 0 ]; do + case "$1" in + --shard) shard=${2:?--shard needs N/M}; shift 2 ;; + --fail-fast) ff=1; shift ;; + *) printf 'usage: %s --run [DIR] [--shard N/M] [--fail-fast]\n' "$0" >&2; exit 2 ;; + esac + done + run "$dir" "$shard" "$ff" ;; -h|--help) sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' ;; *) printf 'usage: %s --list|--run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;; esac