PMAT-3587: fail-fast discards verdicts — the default flips, and skipped becomes a number

scripts/check_explicit_test_commands.sh

@@ -190,10 +190,36 @@ self_test() {
     row 2 "REFUSE a directory named like a fragment" 'not a regular file' bash "$R" --list "$td/name"
     notfact "  ...and a refused tree prints NO commands on stdout" test -n "$(bash "$R" --list "$td/name" 2>/dev/null)"
     # ── the runner: execution ────────────────────────────────────────────
+    # PMAT-3587 -- fail-fast is a VERDICT-DISCARDING default. Measured on #3581: one
+    # test failing on a missing interpreter produced "fail-fast: the 15 command(s)
+    # after it did not run", and one shard hid 8,576 tests. Fifteen unknowns became
+    # an absence, and an absence reads as "nothing to report", which is
+    # indistinguishable from "passed" to every consumer downstream.
+    #
+    # The runner's exit code is its OWN vocabulary and never a passthrough of a
+    # command's rc -- propagating one would let a test that exits 2 mean the same
+    # thing as "commands were not run":
+    #     0  every command ran, none failed
+    #     1  every command ran, F >= 1 FAILED      (a measured failure)
+    #     2  UNMEASURED commands exist             (Unknown, doctrine 4)
+    #
+    # Ticket falsifier: plant a failing command early, require the EXACT count that
+    # still ran and the EXACT count skipped.
     frags "$td/ff" "010-a.cmd=touch $td/a\n" '020-b.cmd=exit 7\n' "030-c.cmd=touch $td/c\n"
-    row 7 "FAIL-FAST: the first non-zero command's status is the step's status" 'exit 7' bash "$R" --run "$td/ff"
+    rm -f "$td/a" "$td/c"
+    row 1 "a failing command no longer stops the run (rc 1 = ran, measured failure)" 'exit 7' bash "$R" --run "$td/ff"
     fact "  ...the command before it ran" test -e "$td/a"
-    notfact "  ...the command after it did NOT run" test -e "$td/c"
+    fact "  ...and the command AFTER it ran too -- no fail-fast" test -e "$td/c"
+    row 1 "the receipt reports ran/skipped/total as NUMBERS, never an absence" 'RESULT ran=3 skipped=0 total=3 failed=1' bash "$R" --run "$td/ff"
+    notfact "  ...a command's own rc is NOT the runner's rc (7 must not leak out)" \
+        bash -c 'bash "$1" --run "$2" > /dev/null 2>&1; [ $? -eq 7 ]' _ "$R" "$td/ff"
+    # Opt-in fail-fast still exists, and it is UNKNOWN (2), not Fail: the commands it
+    # did not run are UNMEASURED, and unmeasured is neither green nor red.
+    rm -f "$td/c"
+    row 2 "--fail-fast stops early and is UNKNOWN (rc 2), not Fail" 'RESULT ran=2 skipped=1 total=3 failed=1' bash "$R" --run "$td/ff" --fail-fast
+    notfact "  ...the command after the failure did NOT run" test -e "$td/c"
+    frags "$td/allok" "010-a.cmd=echo alpha\n" "020-b.cmd=echo beta\n"
+    row 0 "all ran, none failed -> rc 0 and skipped=0" 'RESULT ran=2 skipped=0 total=2 failed=0' bash "$R" --run "$td/allok"
     frags "$td/in" "010-a.cmd=cat > $td/swallowed\n" "020-b.cmd=touch $td/after\n"
     row 0 "a command reading stdin gets /dev/null" '^PASS  2/2' bash "$R" --run "$td/in"
     fact "  ...and the next command still ran" test -e "$td/after"

scripts/ci_run_explicit_test_commands.sh

@@ -21,15 +21,27 @@
 #   * a missing or empty directory is refused -- zero commands is broken wiring
 #   Any violation: rc 2, nothing is printed on stdout, nothing runs.
 #
-# Execution (--run), held equal to the old `&&` chain:
+# Execution (--run):
 #   * commands in order, each as `bash -c "$cmd"` with stdin from /dev/null (the
 #     old docker run had no -i; the list is fully read before anything runs)
 #   * a ::group:: header per command
-#   * FAIL-FAST: the first non-zero command ends the run with THAT exit status
+#   * NO FAIL-FAST (PMAT-3587). Every command runs. A red run still measures
+#     everything else it was going to measure, which is exactly when the data is
+#     most wanted. `--fail-fast` restores the old stop-at-first behaviour.
+#   * a RESULT line always: `RESULT ran=N skipped=S total=T failed=F`. The skipped
+#     count is a NUMBER, never an absence -- an absence reads as "nothing to
+#     report", which is indistinguishable from "passed" to every consumer.
+#
+# EXIT CODE -- the runner's own vocabulary, never a passthrough of a command's rc
+# (propagating one would let a test exiting 2 mean "commands were not run"):
+#   0  every command ran, none failed
+#   1  every command ran, F >= 1 FAILED      -- a measured failure
+#   2  UNMEASURED commands exist             -- Unknown, doctrine 4; also every
+#      refusal (bad tree, bad shard, vacuity)
 #
 # Usage:
 #   scripts/ci_run_explicit_test_commands.sh --list [DIR]
-#   scripts/ci_run_explicit_test_commands.sh --run  [DIR]
+#   scripts/ci_run_explicit_test_commands.sh --run  [DIR] [--shard N/M] [--fail-fast]
 # DIR defaults to ci/explicit-test-commands.d. The case table for this script
 # lives in scripts/check_explicit_test_commands.sh --self-test.
 set -euo pipefail
@@ -96,7 +108,7 @@ parse() {
 # list exactly once between them (PACK-001). N and M are validated; a shard that
 # selects zero commands is a refusal, not a pass.
 run() {
-    local dir=$1 shard=${2:-1/1} out cmd i=0 total rc n m k=0 sel=0
+    local dir=$1 shard=${2:-1/1} ff=${3:-0} out cmd i=0 total rc n m k=0 sel=0 failed=0 first_fail=""
     local -a cmds
     if ! [[ "$shard" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then
         printf 'REFUSE --shard must look like N/M, got %s\n' "$shard" >&2; return 2
@@ -130,20 +142,41 @@ run() {
         bash -c "$cmd" < /dev/null || rc=$?
         printf '::endgroup::\n'
         if [ "$rc" -ne 0 ]; then
+            failed=$((failed + 1))
+            [ -n "$first_fail" ] || first_fail="[$i/$total] exit $rc: $cmd"
             printf 'FAIL  [%s/%s] exit %s: %s\n' "$i" "$total" "$rc" "$cmd" >&2
-            printf '      fail-fast: the %s command(s) after it did not run\n' "$((total - i))" >&2
-            return "$rc"
+            if [ "$ff" -eq 1 ]; then
+                printf '      --fail-fast: the %s command(s) after it were NOT RUN -- unmeasured, not passed\n' "$((total - i))" >&2
+                break
+            fi
         fi
     done
+    # Always, on both paths: the counts are numbers a consumer can read.
+    printf 'RESULT ran=%s skipped=%s total=%s failed=%s\n' "$i" "$((total - i))" "$total" "$failed"
+    [ -n "$first_fail" ] && printf 'RESULT first-failure %s\n' "$first_fail"
+    if [ "$i" -lt "$total" ]; then
+        printf 'UNKNOWN %s of %s command(s) were not run; unmeasured is neither pass nor fail\n' "$((total - i))" "$total" >&2
+        return 2
+    fi
+    if [ "$failed" -gt 0 ]; then
+        printf 'FAIL  %s of %s explicit test command(s) failed; all %s ran\n' "$failed" "$total" "$total" >&2
+        return 1
+    fi
     printf 'PASS  %s/%s explicit test command(s) from %s\n' "$i" "$total" "$dir"
 }
 
 case "${1:-}" in
     --list) parse "${2:-$DEFAULT_DIR}" ;;
     --run)
-        dir=${2:-$DEFAULT_DIR}; shard=1/1
-        case "${3:-}" in --shard) shard=${4:?--shard needs N/M} ;; "") ;; *) printf 'usage: %s --run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;; esac
-        run "$dir" "$shard" ;;
+        dir=${2:-$DEFAULT_DIR}; shard=1/1; ff=0; shift 2 2>/dev/null || shift $#
+        while [ "$#" -gt 0 ]; do
+            case "$1" in
+                --shard) shard=${2:?--shard needs N/M}; shift 2 ;;
+                --fail-fast) ff=1; shift ;;
+                *) printf 'usage: %s --run [DIR] [--shard N/M] [--fail-fast]\n' "$0" >&2; exit 2 ;;
+            esac
+        done
+        run "$dir" "$shard" "$ff" ;;
     -h|--help) sed -n '2,35p' "$0" | sed 's/^# \{0,1\}//' ;;
     *) printf 'usage: %s --list|--run [DIR] [--shard N/M]\n' "$0" >&2; exit 2 ;;
 esac