diff --git a/docs/roadmaps/entries/PMAT-3928.yaml b/docs/roadmaps/entries/PMAT-3928.yaml new file mode 100644 index 0000000000..2155ffc1f6 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3928.yaml @@ -0,0 +1,17 @@ +- id: PMAT-3928 + github_issue: 3928 + item_type: task + title: 'ladder: judge the run verb''s output too (#3928) — re-review fixes' + status: planned + priority: high + assigned_to: null + created: 2026-09-23T19:21:03Z + updated: 2026-09-23T19:21:03Z + spec: null + acceptance_criteria: + - 'Cop ruling (B) 2026-09-23 on #3932. (1) why-builder names run; (2) exit codes 3/4 restored on chat/code truncation paths; (3) real gx10 CPU run --verbose capture for layer 3 (both backends, per #3928 acceptance).' + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 3e99df56e1..5a0b5a10b7 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -21208,3 +21208,20 @@ roadmap: labels: - kind:docs notes: null +- id: PMAT-3928 + github_issue: 3928 + item_type: task + title: 'ladder: judge the run verb''s output too (#3928) — re-review fixes' + status: planned + priority: high + assigned_to: null + created: 2026-09-23T19:21:03Z + updated: 2026-09-23T19:21:03Z + spec: null + acceptance_criteria: + - 'Cop ruling (B) 2026-09-23 on #3932. (1) why-builder names run; (2) exit codes 3/4 restored on chat/code truncation paths; (3) real gx10 CPU run --verbose capture for layer 3 (both backends, per #3928 acceptance).' + phases: [] + subtasks: [] + estimated_effort: null + labels: [] + notes: null diff --git a/scripts/check_ladder_output_judged.sh b/scripts/check_ladder_output_judged.sh index 61f3909b97..148ecebaac 100755 --- a/scripts/check_ladder_output_judged.sh +++ b/scripts/check_ladder_output_judged.sh @@ -118,6 +118,20 @@ cat <<'JSON' JSON } +# #3928: the same row with `run` as the offender. Planted separately from the chat +# fixture because "the detector stopped flagging" and "the verdict stopped reading it" +# are different failures, and #3901 was the fix that only closed one of them. +row_run_garbage() { +cat <<'JSON' +{"cuda":{"ran":true,"fallback":false,"escaped_special":false,"rc":0, + "verbs":{"run":{"ran":true,"rc":0,"output_bad":"gibberish (fragment ' zombie' repeats 3+ times)"}, + "chat":{"ran":true,"rc":0}, + "code":{"ran":true,"rc":0}, + "serve":{"probed":true,"teardown":"clean","routes":{ + "/v1/completions|stream=false":{"http":200,"output_bad":null}}}}}} +JSON +} + QA_OK='{"capability_match":{"passed":true,"skipped":false,"message":"ok"}, "golden_output":{"passed":true,"skipped":false,"message":"ok"}, "gates":{},"gates_failed":[],"gates_reported":2}' @@ -131,14 +145,43 @@ green_of() { # green_of -> true|false check_verdict() { # -> 0 ok local src="$1" g - g=$(green_of "$src" "$(row_chat_garbage)") || return 2 - if [ "$g" = "false" ]; then - printf ' ok %-30s green=false\n' "chat-rc0-bad-output" - return 0 - fi - printf ' FAIL %-30s green=%s — a verb that ran and produced garbage is recorded as working\n' \ - "chat-rc0-bad-output" "$g" - return 1 + local rc=0 + for case in chat run; do + if [ "$case" = chat ]; then g=$(green_of "$src" "$(row_chat_garbage)") || return 2 + else g=$(green_of "$src" "$(row_run_garbage)") || return 2; fi + if [ "$g" = "false" ]; then + printf ' ok %-30s green=false\n' "$case-rc0-bad-output" + else + printf ' FAIL %-30s green=%s -- a verb that ran and produced garbage is recorded as working\n' \ + "$case-rc0-bad-output" "$g"; rc=1 + fi + done + return $rc +} + +# A red row must SAY why (#3932 re-review). The verdict reads `run` since #3928; the +# explanation builder did not, so a row red only on run output printed `unknown` -- +# #3901/#3902's defect, a third time. Same extraction as check_ladder_serve_verdict.sh. +extract_why() { + local src="$1" body + body=$(awk '/why=.*python3 -c/{f=1; next} f && /^print\(/{print; exit} f' "$src") + grep -q 'w.append' <<< "$body" || { echo " the extracted why builder does not append reasons" >&2; return 2; } + printf '%s' "$body" | sed "s/')\$//" +} + +RED_ONLY_ON_RUN_OUTPUT='{"capability_match":{"passed":true,"skipped":false,"message":"ok"}, + "golden_output":{"passed":true,"skipped":false,"message":"ok"}, + "backends":'"$(row_run_garbage)"'}' + +check_run_reason() { # -> 0 the run-only red names run, 1 it does not + local src="$1" why got + why=$(extract_why "$src") || return 2 + got=$(printf '%s' "$RED_ONLY_ON_RUN_OUTPUT" | python3 -c "$why" 2>/dev/null) + case "$got" in + *'verb `run` RAN (rc=0) and produced bad output'*) + printf ' ok %-30s %s\n' "reason:run-output" "${got:0:90}" ;; + *) printf ' FAIL %-30s a row red only on run output says: %s\n' "reason:run-output" "${got:-empty}"; return 1 ;; + esac } # ── layer 3: WHAT TEXT IS JUDGED (#3925) ───────────────────────────────────── @@ -223,16 +266,87 @@ Loading model... T } +# #3928: `apr run --verbose`. VERBATIM from gx10 -- the chatter is the point: kernel +# counts, VRAM figures and a list of hex pointers, none of which is the model speaking. +cap_run_verbose() { cat <<'T' +[GH-480] Patched 2 backward branch(es) for sm_121 JIT workaround +[trueno#243] Manual graph: 591 kernels. first_args=Some(["0xe326a87db200", "0xe326a87e2e00", "0xe326b87a4000"]), last_args=Some(["0xe326e0320000", "0xe32427c00000"]) +Backend: GPU (NVIDIA GB10, 122502 MB VRAM) +[DEBUG] generated token ids: [151668, 271, 785, 6722, 315, 9625, 374, 12095, 13] + +Output: +The capital of France is Paris. + +Completed in 5.08s (cached) +T +} +# #3928 acceptance asks for BOTH backends. The CPU capture, VERBATIM: gx10, 2026-09-23, +# apr 0.69.1 (7168e4b68) sha256 01e821432fe8c463…, Qwen3-1.7B-Q4_K_M.gguf sha256 b139949c5bd74937…, +# the ladder's own invocation (`run --prompt … --max-tokens 16 --verbose --no-gpu`), rc=0. Its chrome +# differs from the GPU one: `verbose:` preamble, the formatted prompt, the raw decode. +cap_run_verbose_cpu() { cat <<'T' +verbose: apr 0.69.1 +verbose: offline = off +verbose: contract gate = enforced over 1 path(s) +verbose: model = /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf (1107409472 bytes) +=== APR Run === + +Source: /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf +Using mmap for 1056MB model +[DEBUG] has_chat_template=true, filename_instruct=false +[DEBUG] formatted_prompt="<|im_start|>user\nWhat is the capital of France? Answer briefly.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" +[DEBUG] add_bos=false, encoded 22 tokens: [151644, 872, 198, 3838, 374, 279, 6722, 315, 9625, 30, 21806, 26753, 13, 151645, 198, 151644, 77091, 198, 151667, 271, 151668, 271] +Loading model: /home/noah/models/Qwen3-1.7B-Q4_K_M.gguf +[BOS-FALLBACK] No tokenizer.ggml.bos_token_id in GGUF — using architecture default for 'qwen3' +Architecture: Transformer [GGUF: qwen3] (28 layers, vocab_size=151936) +Config: hidden_size=2048, context_length=40960, quant=mixed(Q4_K×168,Q6_K×28) lm_head=Q6_K, threads=20 +Model loaded in 799.4ms +Backend: CPU (SIMD-accelerated) +[DEBUG] input_count=22, total_tokens=29, generated_count=7 +[DEBUG] generated token ids: [785, 6722, 315, 9625, 374, 12095, 13] +[DEBUG] raw decoded: "The capital of France is Paris." + +Output: +The capital of France is Paris. + +Completed in 7.04s (cached) +T +} +cap_run_degenerate() { cat <<'T' +Backend: GPU (NVIDIA GB10, 122502 MB VRAM) + +Output: +zombie zombie zombie zombie + +Completed in 5.08s (cached) +T +} +# `Output:` opened and never closed -- the process was cut off mid-reply, so whatever +# is there is a fragment. A refusal to judge, never a clean reply. +cap_run_unterminated() { cat <<'T' +Backend: GPU (NVIDIA GB10, 122502 MB VRAM) + +Output: +The capital of France is Paris. +T +} + # name|capture-fn|expect (clean = no reason · bad = a verdict about the reply · -# red = a refusal to judge, which must never be silent) +# red: = a refusal to judge, which must never be silent, NAMING what is missing: +# envelope · reply · unclosed. The first cut of #3928 swapped exit codes 3 and 4, so +# a capture WITH an envelope was reported as missing one; a bare `red` could not see it.) extraction_cases() { cat <<'CASES' chrome-wrapped-correct-answer|cap_correct_answer|clean code-json-envelope-not-uuid|cap_code_json|clean chrome-does-not-mask-gibberish|cap_chrome_gibberish|bad reply-containing-You-judged-whole|cap_reply_with_you|bad -truncated-capture-no-envelope|cap_no_envelope|red -envelope-but-no-reply|cap_no_reply|red +truncated-capture-no-envelope|cap_no_envelope|red:envelope +envelope-but-no-reply|cap_no_reply|red:reply +run-verbose-chatter-not-judged|cap_run_verbose|clean +run-verbose-cpu-chatter-not-judged|cap_run_verbose_cpu|clean +run-degenerate-reply-in-block|cap_run_degenerate|bad +run-output-never-closed|cap_run_unterminated|red:unclosed CASES } @@ -242,8 +356,15 @@ run_extraction_table() { # -> 0 all as expected [ -n "$name" ] || continue got=$(judge_capture "$src" "$($fn)") || return 2 if [ -z "$got" ]; then cls=clean - elif case "$got" in "could not"*) true ;; *) false ;; esac; then cls=red - else cls=bad; fi + else + case "$got" in + "could not find the backend envelope"*) cls=red:envelope ;; + "could not locate the"*) cls=red:reply ;; + "could not find the closing"*) cls=red:unclosed ;; + "could not"*) cls=red:unnamed ;; + *) cls=bad ;; + esac + fi if [ "$cls" = "$want" ]; then printf ' ok %-34s %s\n' "$name" "$cls" else @@ -255,12 +376,12 @@ run_extraction_table() { # -> 0 all as expected if [ "$SELF_TEST" = 1 ]; then [ -f "$SCRIPT" ] || { echo "cannot read $SCRIPT" >&2; exit 2; } - m1=$(mktemp); m2=$(mktemp); m3=$(mktemp); m4=$(mktemp) - trap 'rm -f "$m1" "$m2" "$m3" "$m4"' EXIT + m1=$(mktemp); m2=$(mktemp); m3=$(mktemp); m4=$(mktemp); m5=$(mktemp); m6=$(mktemp); m7=$(mktemp); m8=$(mktemp) + trap 'rm -f "$m1" "$m2" "$m3" "$m4" "$m5" "$m6" "$m7" "$m8"' EXIT echo "self-test: the shipped script" run_detector_table "$SCRIPT" > /dev/null && check_verdict "$SCRIPT" > /dev/null \ - && run_extraction_table "$SCRIPT" > /dev/null \ + && check_run_reason "$SCRIPT" > /dev/null && run_extraction_table "$SCRIPT" > /dev/null \ || { echo "SELF-TEST FAILED: the shipped script is already red" >&2; exit 1; } echo " GREEN (expected)" @@ -310,7 +431,59 @@ if [ "$SELF_TEST" = 1 ]; then fi echo " RED (expected)" - echo "self-test: PASS — red when the detector stops flagging, when the verdict stops reading it, when an unlocatable reply passes silently, and when the reply is cut short" + # Mutant 5: the extractor is bypassed and the raw capture is judged again -- the + # exact state #3925 fixed, and so the one regression this file exists to catch. + # aprender-3e ran it by hand while folding #3926 and it was caught; naming it here + # means the next person does not have to re-derive that, and a refactor that + # reintroduces raw judging fails rather than being noticed in review. + sed 's#| assistant_reply); rc=$?#| cat); rc=$?#' "$SCRIPT" > "$m5" + cmp -s "$SCRIPT" "$m5" && { echo "SELF-TEST INCONCLUSIVE: mutant 5 changed nothing" >&2; exit 1; } + echo "self-test: mutant 5 (extractor bypassed, the transcript judged raw again)" + if run_extraction_table "$m5" > /dev/null 2>&1; then + echo "SELF-TEST FAILED: mutant 5 passed -- the chrome is being judged and nothing noticed" >&2 + exit 1 + fi + echo " RED (expected)" + + # Mutant 6 (#3928): when `Completed in` never arrives, take the rest of the capture + # instead of refusing. A cut-off run then reads as a clean reply -- the silent-pass + # hazard again, on the path added for `run`. + sed -e 's|^elif oi >= 0 and ci >= 0:|elif oi >= 0:|' \ + -e 's|^ body = lines\[oi + 1:ci\]| body = lines[oi + 1:]|' "$SCRIPT" > "$m6" + cmp -s "$SCRIPT" "$m6" && { echo "SELF-TEST INCONCLUSIVE: mutant 6 changed nothing" >&2; exit 1; } + echo "self-test: mutant 6 (an unterminated run reply is taken as whole)" + if run_extraction_table "$m6" > /dev/null 2>&1; then + echo "SELF-TEST FAILED: mutant 6 passed -- a cut-off run reply reads as clean" >&2 + exit 1 + fi + echo " RED (expected)" + + # Mutant 7 (#3928): `run` drops out of the green expression while chat and code stay. + # Mutant 2 removes the output_bad check for EVERY verb at once, so it cannot tell + # whether `run` was ever wired in -- which is exactly how #3886 folded serve and left + # chat and code unread in the same structure. One verb, one mutant. + sed 's|and verb_ok(v, "run") and verb_ok(v, "chat")|and verb_ok(v, "chat")|' "$SCRIPT" > "$m7" + cmp -s "$SCRIPT" "$m7" && { echo "SELF-TEST INCONCLUSIVE: mutant 7 changed nothing" >&2; exit 1; } + echo "self-test: mutant 7 (green stops reading the run verb)" + if check_verdict "$m7" > /dev/null 2>&1; then + echo "SELF-TEST FAILED: mutant 7 passed -- run garbage no longer reddens the row" >&2 + exit 1 + fi + echo " RED (expected)" + + # Mutant 8 (#3932 re-review): the explanation builder stops naming `run`. The row + # stays red (mutant 7's check still passes) and says `unknown` -- the verdict moved + # and the explanation did not. + sed 's|for _vn in ("run", "chat", "code"):|for _vn in ("chat", "code"):|' "$SCRIPT" > "$m8" + cmp -s "$SCRIPT" "$m8" && { echo "SELF-TEST INCONCLUSIVE: mutant 8 changed nothing" >&2; exit 1; } + echo "self-test: mutant 8 (the red reason stops naming the run verb)" + if check_run_reason "$m8" > /dev/null 2>&1; then + echo "SELF-TEST FAILED: mutant 8 passed -- a row red only on run output explains nothing" >&2 + exit 1 + fi + echo " RED (expected)" + + echo "self-test: PASS — red when the detector stops flagging, when the verdict stops reading it, when an unlocatable reply passes silently, when the reply is cut short, when the extractor is bypassed entirely, when an unterminated run reply is taken as whole, when green stops reading the run verb, and when the red reason stops naming it" exit 0 fi @@ -319,6 +492,7 @@ rc=0 run_detector_table "$SCRIPT" || rc=$? [ "$rc" = 2 ] && exit 2 check_verdict "$SCRIPT" || rc=1 +check_run_reason "$SCRIPT" || rc=1 run_extraction_table "$SCRIPT" || rc=$? [ "$rc" = 2 ] && exit 2 if [ "$rc" = 0 ]; then diff --git a/scripts/model_ladder.sh b/scripts/model_ladder.sh index 66b052ce65..898ed4182d 100755 --- a/scripts/model_ladder.sh +++ b/scripts/model_ladder.sh @@ -238,18 +238,42 @@ for i in range(len(lines) - 1, -1, -1): if d is not None and "backend" in d: env = i break -if env < 0: - sys.exit(3) start = -1 -for i in range(env - 1, -1, -1): - if lines[i].lstrip().startswith("Assistant:"): - start = i - break +if env >= 0: + for i in range(env - 1, -1, -1): + if lines[i].lstrip().startswith("Assistant:"): + start = i + break + +# 3. `apr run --verbose` (#3928). Its capture carries kernel counts, VRAM figures and +# a list of hex pointers; judged raw that is #3925 again with different chrome. The +# reply sits between two lines WE print: a bare `Output:` and `Completed in `. +oi = ci = -1 if start < 0: - sys.exit(4) + for i, l in enumerate(lines): + if l.strip() == "Output:": + oi = i + if oi >= 0: + for i in range(oi + 1, len(lines)): + if lines[i].lstrip().startswith("Completed in "): + ci = i + break -body = [lines[start].lstrip()[len("Assistant:"):].lstrip()] + lines[start + 1:env] +# Exit codes keep the meanings they had before `run` was added (#3932 re-review: the +# first cut swapped 3 and 4 for chat/code, so the message named the wrong missing piece): +# 3 = no backend envelope at all (truncated) 4 = envelope, but no `Assistant:` reply +# 5 = `Output:` opened and `Completed in` never came (a truncated `run` capture) +if start >= 0: + body = [lines[start].lstrip()[len("Assistant:"):].lstrip()] + lines[start + 1:env] +elif oi >= 0 and ci >= 0: + body = lines[oi + 1:ci] +elif oi >= 0: + sys.exit(5) +elif env < 0: + sys.exit(3) +else: + sys.exit(4) # The session prints one fixed line on `/exit`. Strip THAT EXACT STRING and nothing # else: a general "starts with You:" rule is the cut this function exists to avoid. @@ -275,6 +299,7 @@ judge_reply() { case $rc in 0) printf '%s' "$text" | gibberish_reason || true ;; 3) printf 'could not find the backend envelope in %s bytes of captured %s output -- the capture is truncated, so nothing about the reply was measured' "$n" "$verb" ;; + 5) printf 'could not find the closing `Completed in` line after `Output:` in %s bytes of captured %s output -- the capture is truncated, so nothing about the reply was measured' "$n" "$verb" ;; *) printf 'could not locate the %s reply in %s bytes of captured output' "$verb" "$n" ;; esac } @@ -662,6 +687,14 @@ PY if grep -qF -e '\u{200b}' -e $'\u200b' <<< "$fp"; then esc=true; fi if [ "$b" != cpu ] && [ -z "$GPU_NAME" ]; then ran=false; fi [ $run_rc -eq 0 ] || ran=false + # #3928: `run` was the last of the operator's four verbs whose TEXT nothing read. + # Its capture was inspected for two markers (fallback, escaped_special) and never + # for content, and `apr qa`'s golden leg does not cover it: qa runs ONCE PER RUNG, + # before this loop, while `run` is measured per backend -- so a model correct on + # one backend and degenerate on the other passed. That is the exact shape #3921 + # was created for. + run_bad="" + if [ $run_rc -eq 0 ]; then run_bad=$(judge_reply "$run_out" run); fi # ── #3828: the OTHER THREE VERBS ──────────────────────────────────────────── # This loop ran `apr run` and nothing else, so `serve`, `chat` and `code` cells in @@ -697,7 +730,8 @@ PY [ $first = 1 ] || be_json="$be_json,"; first=0 be_json="$be_json\"$b\":{\"ran\":$ran,\"fallback\":$fb,\"escaped_special\":$esc,\"rc\":$run_rc" - be_json="$be_json,\"verbs\":{\"run\":{\"ran\":$ran,\"rc\":$run_rc}" + run_bad_json=$(printf '%s' "${run_bad:-}" | json_str_or_null) + be_json="$be_json,\"verbs\":{\"run\":{\"ran\":$ran,\"rc\":$run_rc,\"output_bad\":$run_bad_json}" # #3921: `output_bad` carries the REASON, or null. A string here is a verdict # about what the verb produced; the rc beside it is only about whether it ran. chat_bad_json=$(printf '%s' "${chat_bad:-}" | json_str_or_null) @@ -790,7 +824,7 @@ def verb_ok(v, name): green = cap_ok and qa.get("golden_output", {}).get("passed", False) \ and all(v["ran"] and not v["fallback"] and not v.get("escaped_special") and serve_ok(v) - and verb_ok(v, "chat") and verb_ok(v, "code") + and verb_ok(v, "run") and verb_ok(v, "chat") and verb_ok(v, "code") for v in be.values()) # ANTI-VACUITY (#3863 item 16). `apr qa` exiting non-zero means at least one gate # failed. If the row records NO failed gate, the receipt cannot account for its own @@ -882,7 +916,9 @@ for b,v in r["backends"].items(): # #3902: the same omission #3901 found for serve, caught in the SAME pass this # time rather than after. A verdict that starts consulting a field while the # explanation builder does not is precisely how a row becomes red and silent. - for _vn in ("chat", "code"): + # #3932 re-review: `run` joined `green` (#3928) and, again, not this builder — a row + # red ONLY on run output printed `unknown`. Same list as the verdict, or it is #3902. + for _vn in ("run", "chat", "code"): _x=(v.get("verbs") or {}).get(_vn) if not isinstance(_x, dict): w.append(b+": verb `%s` absent from the receipt"%_vn) elif not _x.get("ran") or (_x.get("rc") or 0)!=0: w.append(b+": verb `%s` did not run (rc=%s)"%(_vn,_x.get("rc")))