noahgift · Sep 28, 2026 · Sep 28, 2026
diff --git a/scripts/check_ladder_apr_release_pin.sh b/scripts/check_ladder_apr_release_pin.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# check_ladder_apr_release_pin.sh — the release cells run the apr UNDER TEST, never a PATH lookup (#3715 A3).
+#
+# WHY. Operator 2026-09-28 17:28Z: "cells and every gate step invoke the binary UNDER TEST by absolute path
+# (the rc.1 artifact) with a sha256 check; each receipt records `apr --version` SHA == rc.1 tag SHA.
+# Falsifier: plant a different-sha apr first on PATH -> cell refuses. PATH lookup of apr on the gate path = 0."
+# Four `apr` binaries once coexisted on one box and a bare `apr` resolved to a 26-day-old one. A cell measured
+# on the wrong binary is a confident verdict about code nobody is shipping.
+#
+# WHAT. model_ladder.sh step 0, release mode (APR_RELEASE_SHA256 / APR_RELEASE_COMMIT set), driven through
+# `--lock-probe` -- the same apr_locked every cell goes through -- in a temp root whose PATH has a PLANTED apr
+# first. Every fake apr logs each invocation, so "the planted apr ran zero times" is observed, not assumed.
+#
+# Exit: 0 every case as expected and every mutant killed · 1 a case or mutant landed wrong · 2 could not check.
+set -uo pipefail
+cd "$(dirname "$0")/.." || exit 2
+LADDER_SH=scripts/model_ladder.sh
+for t in sha256sum flock choom; do command -v "$t" > /dev/null || { echo "check_ladder_apr_release_pin: $t absent" >&2; exit 2; }; done
+[ -f "$LADDER_SH" ] || { echo "check_ladder_apr_release_pin: $LADDER_SH absent" >&2; exit 2; }
+
+W=$(mktemp -d "${TMPDIR:-/tmp}/apr-pin.XXXXXX") || exit 2
+trap 'rm -rf -- "${W:?}"' EXIT
+GOOD=abcdef1234567890abcdef1234567890abcdef12   # the "release commit"
+OTHER=0123456789abcdef0123456789abcdef01234567
+mkdir -p "$W/root" "$W/rc" "$W/path" "$W/imp"
+for d in contracts scripts evidence Cargo.toml; do ln -s "$PWD/$d" "$W/root/$d"; done
+LOG="$W/calls"
+fake() { # <file> <label> <commit-it-claims>
+  printf '#!/bin/sh\necho "%s $*" >> "%s"\n[ "$1" = --version ] && { echo "apr 0.70.0 (%s)"; exit 0; }\nexit 0\n' "$2" "$LOG" "${3:0:9}" > "$1"
+  chmod +x "$1"
+}
+fake "$W/rc/apr" rc "$GOOD"          # the artifact under test
+fake "$W/path/apr" planted "$OTHER"  # a different-sha apr, first on PATH
+fake "$W/imp/apr" impostor "$GOOD"   # different bytes, but CLAIMS the release commit
+# A bare name is checked in one place and executed from another: the ladder's cwd holds the real bytes (so
+# -f and sha256 pass on them), while exec resolves the same name through PATH -- to an impostor claiming the
+# release commit. Only the ABSOLUTE-path rule stops this one: bytes checked != bytes run.
+cp "$W/rc/apr" "$W/root/rc-apr"; chmod +x "$W/root/rc-apr"
+fake "$W/path/rc-apr" impostor "$GOOD"
+sum() { local s; s=$(sha256sum -- "$1") && printf '%s' "${s%% *}"; }
+RC_SUM=$(sum "$W/rc/apr"); PL_SUM=$(sum "$W/path/apr")
+
+# probe <script> <APR> <sha|-> <commit|-> -> sets RC, CALLS (non---version invocations by label)
+probe() {
+  : > "$LOG"
+  local env=(PATH="$W/path:$PATH" MODEL_LADDER_ROOT="$W/root" MODEL_LADDER_GPU_LOCK="$W/lock" MODEL_LADDER_LOCK_WAIT=5 APR="$2")
+  [ "$3" != - ] && env+=(APR_RELEASE_SHA256="$3")
+  [ "$4" != - ] && env+=(APR_RELEASE_COMMIT="$4")
+  env "${env[@]}" timeout 60 bash "$1" --lock-probe qa cell > "$W/out" 2>&1; RC=$?
+  CALLS=$(grep -v -- ' --version' "$LOG" | cut -d' ' -f1 | sort | uniq -c | tr -s ' ' | tr '\n' ';')
+}
+
+# name | APR | sha | commit | want rc | want the ONLY cell invocation (label or "none")
+CASES="green-planted-on-path|$W/rc/apr|$RC_SUM|$GOOD|0|rc
+bare-apr-is-a-path-lookup|apr|$RC_SUM|$GOOD|2|none
+bare-name-checked-in-cwd-run-from-path|rc-apr|$RC_SUM|$GOOD|2|none
+absolute-planted|$W/path/apr|$RC_SUM|$GOOD|2|none
+impostor-claims-the-commit|$W/imp/apr|$RC_SUM|$GOOD|2|none
+right-bytes-wrong-commit|$W/path/apr|$PL_SUM|$GOOD|2|none
+half-pin-sha-only|$W/rc/apr|$RC_SUM|-|2|none
+half-pin-commit-only|$W/rc/apr|-|$GOOD|2|none
+sha-not-hex|$W/rc/apr|nothex|$GOOD|2|none"
+
+run_cases() { # <script> [quiet] -> number of cases that landed wrong
+  local s="$1" q="${2:-}" bad=0 name apr sha commit wrc wcall want
+  while IFS='|' read -r name apr sha commit wrc wcall; do
+    probe "$s" "$apr" "$sha" "$commit"
+    want=$([ "$wcall" = none ] && echo "" || echo " 1 $wcall;")
+    if [ "$RC" = "$wrc" ] && [ "$CALLS" = "$want" ]; then [ -z "$q" ] && echo "ok    $name (rc $RC, cell ran: ${wcall})"
+    else bad=$((bad + 1)); [ -z "$q" ] && { echo "FAIL  $name: rc $RC (want $wrc), cell invocations '${CALLS}' (want '${want}')"; sed 's/^/        /' "$W/out" | tail -n 3; }
+    fi
+  done <<< "$CASES"
+  return "$bad"
+}
+
+echo "== cases (planted apr first on PATH in every case)"
+run_cases "$LADDER_SH"; bad=$?
+rows=$(printf '%s\n' "$CASES" | wc -l)
+
+echo "== mutants (each must turn a case RED)"
+mutant() { # <name> <python-literal old> <new>
+  local m="$W/mut-$1.sh"
+  python3 - "$LADDER_SH" "$m" "$2" "$3" <<'PY' || { echo "FAIL  mutant $1: anchor not found once -- the check no longer tracks $LADDER_SH"; bad=$((bad + 1)); return; }
+import sys
+src, dst, old, new = sys.argv[1:]
+s = open(src).read()
+sys.exit(1) if s.count(old) != 1 else open(dst, "w").write(s.replace(old, new, 1))
+PY
+  if run_cases "$m" quiet; then echo "FAIL  mutant $1 SURVIVED"; bad=$((bad + 1)); else echo "ok    mutant $1 killed"; fi
+}
+mutant no-release-pin '  release_pin || exit 2' '  true || exit 2'
+mutant no-absolute-check '    /*) ;;
+    *) echo "decline: release mode needs' '    *) ;;
+    /*) echo "decline: release mode needs'
+mutant no-sha256-check '[ "$got" = "$want" ] ||' 'true ||'
+mutant no-version-check '*"(${commit:0:9})"*) ;;' '*) ;;'
+mutant path-lookup-in-cell 'choom -n 1000 -- "$APR" "$@"; }' 'choom -n 1000 -- apr "$@"; }'
+
+[ "$rows" -ge 9 ] || { echo "VACUOUS $rows case(s), fewer than the 9 declared" >&2; exit 1; }
+[ "$bad" = 0 ] && { echo "PASS $rows cases + 5 mutants: release cells run only the pinned artifact; PATH lookups of apr = 0"; exit 0; }
+echo "RED $bad"; exit 1
diff --git a/scripts/check_ladder_cell_timeout_takt.sh b/scripts/check_ladder_cell_timeout_takt.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# check_ladder_cell_timeout_takt.sh — H1 (hard per-cell timeout = FAIL) and H5 (takt tripwires) in the cells producer.
+#
+# WHY. Operator 2026-09-28 17:50Z: "H1 hard per-cell timeout. Timeout = FAIL, blocks FINAL, never a skip. Default
+# 10 min; after sampling, per class = 4 x measured p95 (floor 5 min). The timeout value is recorded in every
+# receipt." and "H5 Takt tripwires after rc.1: at +2 h, +4 h, +6 h, completed cells >= 90% of plan, or ANDON
+# immediately." A hung 4B cell held lambda's GPU for hours, and the old subprocess timeout killed only `flock`:
+# the apr grandchild kept the card after the cell was written down. So the case that matters is not "the timer
+# fired" but "the process GROUP is gone and the row says FAIL with the timeout it ran under".
+#
+# Runs the SHIPPED scripts/lib/model_ladder_cells_produce.py (imported), then each mutant must turn a case RED.
+# Exit: 0 all cases + mutants as expected · 1 a case or mutant landed wrong · 2 could not check.
+set -uo pipefail
+cd "$(dirname "$0")/.." || exit 2
+LIB=scripts/lib
+[ -f "$LIB/model_ladder_cells_produce.py" ] || { echo "check_ladder_cell_timeout_takt: producer absent" >&2; exit 2; }
+W=$(mktemp -d "${TMPDIR:-/tmp}/cell-tmo.XXXXXX") || exit 2
+trap 'rm -rf -- "${W:?}"' EXIT
+
+cases() { # <libdir> [quiet] -> 0 when every case lands as expected
+  python3 - "$1" "$W" "${2:-}" <<'PY'
+import importlib, json, os, sys, time
+lib, w, quiet = sys.argv[1], sys.argv[2], sys.argv[3]
+sys.path.insert(0, lib)
+P = importlib.import_module("model_ladder_cells_produce")
+bad = 0
+def case(name, ok, why):
+    global bad
+    if not ok:
+        bad += 1
+    if not quiet or not ok:
+        print(("ok    " if ok else "FAIL  ") + name + ("" if ok else ": " + why))
+
+# H1-a: a hung apr whose grandchild holds on -- the whole group must be gone, rc 124, timed_out set.
+apr = os.path.join(w, "hang-apr")
+pidf = os.path.join(w, "grandchild.pid")
+open(apr, "w").write(f"#!/bin/sh\nsleep 20 &\necho $! > {pidf}\nwait\n")
+os.chmod(apr, 0o755)
+R = P.Runner(apr, "", 5, 2)
+t0 = time.monotonic(); rc, out, err = R.call(["run"]); el = time.monotonic() - t0
+time.sleep(0.3)
+gpid = int(open(pidf).read())
+alive = os.path.exists(f"/proc/{gpid}") and "Z" not in open(f"/proc/{gpid}/stat").read().split()[2]
+case("timeout-kills-the-group", rc == 124 and R.timed_out and not alive and el < 10,
+     f"rc={rc} timed_out={R.timed_out} grandchild_alive={alive} elapsed={el:.1f}s")
+if alive:
+    os.kill(gpid, 9)
+
+# H1-a2: the timeout bounds the CELL, not each apr call: two 1.5 s calls under one 2 s cell deadline -> the 2nd times out.
+slow = os.path.join(w, "slow-apr")
+open(slow, "w").write("#!/bin/sh\nsleep 1.5\n")
+os.chmod(slow, 0o755)
+R = P.Runner(slow, "", 5, 2)
+R.deadline = time.monotonic() + R.timeout
+rc1 = R.call(["run"])[0]; rc2 = R.call(["run"])[0]
+case("timeout-bounds-the-cell-not-the-call", rc1 == 0 and rc2 == 124 and R.timed_out, f"rc1={rc1} rc2={rc2}")
+
+# H1-b: the row a timeout produces is a FAIL naming the timeout, and carries timeout_s.
+row = P.stamp({"verdict": "pass", "reason": "answered"}, time.monotonic(), 600, True)
+case("timeout-row-is-fail", row["verdict"] == "fail" and row["reason"].startswith("TIMEOUT") and row["timeout_s"] == 600
+     and row["timed_out"] is True, json.dumps(row))
+row = P.stamp({"verdict": "pass", "reason": "answered"}, time.monotonic(), 600, False)
+case("no-timeout-row-keeps-verdict", row["verdict"] == "pass" and row["timeout_s"] == 600 and "wall_s" in row, json.dumps(row))
+
+# H1-c: class timeouts floor at 5 min; no class -> the 10-min default.
+class A: pass
+a = A(); a.timeout = P.DEFAULT_CELL_TIMEOUT_S; a.class_timeouts = {"m.gguf|run": 100, "m.gguf|chat": 1200}
+case("class-timeout-floor", P.cell_timeout(a, {"file": "m.gguf"}, "run") == 300, str(P.cell_timeout(a, {"file": "m.gguf"}, "run")))
+case("class-timeout-used", P.cell_timeout(a, {"file": "m.gguf"}, "chat") == 1200, "")
+case("default-timeout-10min", P.cell_timeout(a, {"file": "m.gguf"}, "serve") == 600 and P.DEFAULT_CELL_TIMEOUT_S == 600, "")
+
+# H5: at +2 h 9/10 is met (90%), at +4 h 17/20 is an ANDON, +6 h not yet reached -> not judged.
+plan = os.path.join(w, "plan.json"); log = os.path.join(w, "takt.jsonl")
+json.dump({"start_epoch": 1000, "checkpoints": [{"at_s": 7200, "planned_cells": 10}, {"at_s": 14400, "planned_cells": 20},
+                                                {"at_s": 21600, "planned_cells": 30}]}, open(plan, "w"))
+now = [1000 + 7200]
+T = P.Takt(plan, log, clock=lambda: now[0])
+P.COMPLETED[0] = 9
+r1 = T.tick()
+case("takt-90pct-met", len(r1) == 1 and not r1[0]["andon"] and not os.path.exists(log + ".andon"), json.dumps(r1))
+now[0] = 1000 + 14400 + 5; P.COMPLETED[0] = 17
+r2 = T.tick()
+andon = open(log + ".andon").read() if os.path.exists(log + ".andon") else ""
+case("takt-below-90pct-andon", len(r2) == 1 and r2[0]["andon"] and "ANDON H5" in andon, json.dumps(r2) + " andon=" + andon)
+case("takt-future-not-judged", T.fired == 2 and len(open(log).read().splitlines()) == 2, f"fired={T.fired}")
+try:
+    json.dump({"start_epoch": 1, "checkpoints": []}, open(plan, "w")); P.Takt(plan, log); empty_ok = False
+except ValueError:
+    empty_ok = True
+case("takt-empty-plan-refused", empty_ok, "an empty plan was accepted")
+sys.exit(1 if bad else 0)
+PY
+}
+
+echo "== cases"
+timeout 120 bash -c "$(declare -f cases); W=$W; cases $LIB"; bad=$?
+echo "== mutants (each must turn a case RED)"
+mutant() { # <name> <old> <new>
+  local md="$W/mut-$1"; mkdir -p "$md"; cp "$LIB"/*.py "$md/"
+  python3 - "$md/model_ladder_cells_produce.py" "$2" "$3" <<'PY' || { echo "FAIL  mutant $1 did not apply -- it proves nothing"; bad=1; return; }
+import sys
+p, old, new = sys.argv[1:]
+s = open(p).read()
+sys.exit(1) if s.count(old) != 1 else open(p, "w").write(s.replace(old, new, 1))
+PY
+  if timeout 120 bash -c "$(declare -f cases); W=$W; cases $md quiet" > /dev/null 2>&1; then echo "FAIL  mutant $1 SURVIVED"; bad=1; else echo "ok    mutant $1 killed"; fi
+}
+mutant kill-only-leader 'os.killpg(p.pid, signal.SIGKILL)' 'p.kill()'
+mutant no-new-session 'text=True, start_new_session=True)' 'text=True)'
+mutant timeout-not-fail '        row["verdict"] = "fail"
+' '        pass
+'
+mutant per-call-not-per-cell 'left = self.timeout if self.deadline is None else' 'left = self.timeout if True else'
+mutant no-floor 'return max(int(t), TIMEOUT_FLOOR_S) if' 'return int(t) if'
+mutant takt-always-ok 'ok = done >= self.RATIO * plan' 'ok = True'
+mutant takt-no-andon-file '                with open(self.log + ".andon", "a") as f:
+                    f.write(line + "\n")' '                pass'
+[ "$bad" = 0 ] && { echo "PASS H1 timeout (group kill, FAIL, timeout_s) + H5 takt: all cases and 7 mutants"; exit 0; }
+echo "RED"; exit 1
diff --git a/scripts/check_release_models_t1.sh b/scripts/check_release_models_t1.sh
@@ -82,6 +82,9 @@ A_CHOOM_R='choom -n 1000 -- bash scripts/model_ladder.sh --host $REMOTE_HOST'
 A_CHOOM_L='    choom -n 1000 -- bash scripts/model_ladder.sh --host "$LOCAL_HOST"'
 A_CELLS_R='bash scripts/model_ladder.sh --host $REMOTE_HOST --cells --out'
 A_CELLS_L='bash scripts/model_ladder.sh --host "$LOCAL_HOST" --cells --out'
+# A3 (operator 2026-09-28 17:28Z): both legs hand the ladder the artifact under test by ABSOLUTE path + sha256 + commit.
+A_APR_L='    APR="$apr" APR_RELEASE_SHA256="${sum%% *}" APR_RELEASE_COMMIT="$sha" \'
+A_APR_R='APR="\$apr" APR_RELEASE_SHA256="\${sum%% *}" APR_RELEASE_COMMIT="$sha" \\'
 R_VER='    out="$(cd "$root" && bash "$judge" --version "$version" 2>&1)"; rc=$?'
 R_RED='        *) printf '"'"'FAIL  R7 model matrix NOT green for %s (rc %s):\n%s\n'"'"' "$version" "$rc" \'
 # The FAIL-lines line also appears on the emergency-scope path (#4046 merge-back), so the
@@ -94,7 +97,7 @@ R_NOJUDGE='    if [ ! -f "$judge" ]; then'
 env_die() { printf 'ENV   %s -- the table judged nothing, not a pass\n' "$*" >&2; exit 2; }
 for f in "$AUTOPILOT" "$MODELS" "$PARAMS" "$PREFLIGHT"; do [ -r "$f" ] || env_die "no $f"; done
 for t in git python3; do command -v "$t" > /dev/null 2>&1 || env_die "no $t"; done
-for a in "$A_SHARED" "$A_255" "$A_BUILD" "$A_NORCPT" "$A_RED" "$A_DECLINE" "$A_PROOF" "$A_DISK" "$A_CHOOM_R" "$A_CHOOM_L" "$A_CELLS_R" "$A_CELLS_L"; do
+for a in "$A_SHARED" "$A_255" "$A_BUILD" "$A_NORCPT" "$A_RED" "$A_DECLINE" "$A_PROOF" "$A_DISK" "$A_CHOOM_R" "$A_CHOOM_L" "$A_CELLS_R" "$A_CELLS_L" "$A_APR_L" "$A_APR_R"; do
     grep -qF -- "$a" "$MODELS" || env_die "models_t1.sh has no '$a' line -- the subject moved"
 done
 for a in "$A_DIE" "$A_RDIE" "$A_RNODR"; do
@@ -207,9 +210,14 @@ fixture() {
 #!/usr/bin/env bash
 while [ $# -gt 0 ]; do case "$1" in --host) h=$2; shift 2 ;; --out) o=$2; shift 2 ;; --cells) c=1; shift ;; *) shift ;; esac; done
 [ "${c:-0}" = 1 ] || { echo "fixture: no --cells on $h, so the receipt would carry no cells[] (#3715 B1)"; exit 2; }
+# A3: the real ladder's step 0 refuses these; the stub refuses the same way so the wiring is what is tested.
+case "${APR:-}" in /*) ;; *) echo "fixture: \$APR is not an absolute path on $h ('${APR:-}') -- A3"; exit 2 ;; esac
+s=$(sha256sum -- "$APR") && [ "${s%% *}" = "${APR_RELEASE_SHA256:-}" ] \
+    || { echo "fixture: \$APR sha256 is not APR_RELEASE_SHA256 on $h -- A3"; exit 2; }
+c=${APR_RELEASE_COMMIT:-unset}; case "$("$APR" --version)" in *"(${c:0:9})"*) ;; *) echo "fixture: \$APR is not built from APR_RELEASE_COMMIT on $h -- A3"; exit 2 ;; esac
 printf 'ladder host=%s oom=%s flock=%s\n' "$h" "${FX_OOM:-none}" "${FX_FLOCK:-0}" >> "$FX_LOG"
 [ "${FX_NO_RECEIPT_HOST:-}" = "$h" ] && { echo "fixture: measured nothing on $h"; exit 2; }
-apr="${CARGO_TARGET_DIR:-$PWD/target}/release/apr"
+apr="$APR"
 v=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)
 red=0; [ "${FX_RED_HOST:-}" = "$h" ] && red=1
 mkdir -p "$o"
@@ -458,13 +466,15 @@ mutant no-choom         "$MODELS" "$A_CHOOM_R" 'bash scripts/model_ladder.sh --h
 mutant wrapper-flock    "$MODELS" "$A_CHOOM_L" '    flock /tmp/apr-gpu.lock choom -n 1000 -- bash scripts/model_ladder.sh --host "$LOCAL_HOST"' oom_victim
 mutant no-cells-remote   "$MODELS" "$A_CELLS_R" 'bash scripts/model_ladder.sh --host $REMOTE_HOST --out' green_pair
 mutant no-cells-local    "$MODELS" "$A_CELLS_L" 'bash scripts/model_ladder.sh --host "$LOCAL_HOST" --out' green_pair
+mutant apr-unpinned-local "$MODELS" "$A_APR_L" '    APR="$apr" \' green_pair
+mutant apr-bare-remote  "$MODELS" "$A_APR_R" 'APR=apr APR_RELEASE_SHA256="\${sum%% *}" APR_RELEASE_COMMIT="$sha" \\' green_pair
 mutant r7-no-version    "$PREFLIGHT" "$R_VER" '    out="$(cd "$root" && bash "$judge" 2>&1)"; rc=$?' r7_green preflight
 mutant r7-red-is-go     "$PREFLIGHT" "$R_RED" '        *) return 0; printf '"'"'FAIL  R7 model matrix NOT green for %s (rc %s):\n%s\n'"'"' "$version" "$rc" \' r7_red preflight
 mutant r7-no-fail-lines "$PREFLIGHT" "$R_LINES" "$R_LINES_HEAD"$'\n''               "" ;;' r7_missing preflight
 mutant r7-decline-is-go "$PREFLIGHT" "$R_DECLINE" '        2) return 0; echo "FAIL  R7 the model-matrix judge DECLINED' r7_decline preflight
 mutant r7-no-judge-ok   "$PREFLIGHT" "$R_NOJUDGE" '    if false; then' r7_nojudge preflight
 
 # VACUITY FLOOR: a table that ran fewer rows than it declares is not a pass.
-[ "$rows" -ge 41 ] || { printf 'VACUOUS %s row(s) ran, fewer than the 41 declared\n' "$rows" >&2; exit 1; }
+[ "$rows" -ge 43 ] || { printf 'VACUOUS %s row(s) ran, fewer than the 43 declared\n' "$rows" >&2; exit 1; }
 [ "$fails" -eq 0 ] || { printf 'RED   %s of %s row(s) failed\n' "$fails" "$rows" >&2; exit 1; }
 printf 'PASS  %s row(s): the model matrix runs at T-1 on both hosts, every failure to prove the release STOPs before the tag, and R7 refuses the same failures at T-4 (#3717)\n' "$rows"