From fa8623a5d8b62f2bc349d83287a9a132c8ea2e32 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Tue, 22 Sep 2026 01:22:57 +0200 Subject: [PATCH] =?UTF-8?q?PMAT-3797:=20CRUX=20serve=20verb=20=E2=80=94=20?= =?UTF-8?q?apr=20serve,=20llama-server=20and=20ollama=20through=20ONE=20Op?= =?UTF-8?q?enAI=20client,=20non-streaming=20and=20streaming;=20the=20key's?= =?UTF-8?q?=20additive=20mode;=20modes=20declared=20on=20the=20serve=20run?= =?UTF-8?q?=20correspondence=20entry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #3797 Co-Authored-By: Claude Opus 5 (1M context) --- docs/roadmaps/entries/PMAT-3797.yaml | 22 ++++ docs/roadmaps/roadmap.yaml | 22 ++++ evidence/crux/verb-correspondence.yaml | 16 ++- scripts/check_crux_inference_judge.sh | 35 ++++++- scripts/crux_inference_dogfood.sh | 134 +++++++++++++++++++++---- scripts/lib/crux_inference_judge.py | 36 +++++-- scripts/lib/crux_openai_client.py | 90 +++++++++++++++++ 7 files changed, 325 insertions(+), 30 deletions(-) create mode 100644 docs/roadmaps/entries/PMAT-3797.yaml create mode 100755 scripts/lib/crux_openai_client.py diff --git a/docs/roadmaps/entries/PMAT-3797.yaml b/docs/roadmaps/entries/PMAT-3797.yaml new file mode 100644 index 0000000000..35f6908b22 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-3797.yaml @@ -0,0 +1,22 @@ +- id: PMAT-3797 + github_issue: 3797 + item_type: task + title: 'CRUX slice 4 (#3739): the serve verb — apr serve, llama-server and ollama through ONE OpenAI client, streaming + non-streaming, plus hf/llamafile servers' + status: in_progress + priority: critical + assigned_to: aprender-76 + created: 2026-09-21T23:55:00Z + updated: 2026-09-22T00:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3797 done_when 1, verbatim: "`scripts/crux_inference_dogfood.sh --verbs run,serve` runs ONE serve cell per model under one hold of the GPU lock: `apr serve run` and `llama-server` each load the model once, next to the running ollama daemon; every golden prompt goes through `scripts/lib/crux_openai_client.py`, non-streaming and streaming; then the servers stop, and ollama unloads (keep_alive 0, plus the unload wait)." Teardown kills every server before waiting, and waits on those pids only. The first real run hung here: a bare `wait` after the first kill blocked on the other server.' + - 'done_when 2, verbatim: "One client for every server. `crux_openai_client.py` sends the same body (model, messages, max_tokens, temperature, seed, stream) and writes the row-contract JSON `{\"text\", \"reported\": {device, stream, finish_reason, usage, chunks}}`. It reads no clock and computes no rate." MEASURED against the three servers on lambda (cpu): non-stream and stream both answer, and the usage is transcribed (apr serve prompt_tokens 18; llama-server and ollama 36).' + - 'done_when 3/4, verbatim: "Plugin engines (hf, llamafile) run `gen --verb ''serve run''` ... The judge reads every serve row through the client JSON and applies the same RED rule. The key gains an additive `mode` (nonstream | stream) for serve cells only. apr serve''s cells carry `backend_verified: false`." Per aprender-97 (#3715), the correspondence ENTRY for `serve run` declares `modes: [nonstream, stream]`, with per-engine modes (llama.cpp and ollama both, hf and llamafile nonstream), so pv never names a verb.' + - 'done_when 5, MEASURED: `bash scripts/check_crux_inference_judge.sh` gives 44 ok, 0 broke. The serve rows: stream and non-stream are distinct GREEN cells with apr serve''s backend unverified; an apr serve that answered nothing (connection refused) while llama-server answered is RED. Mutants: dropping the mode from the key breaks 1; dropping the serve backend flag breaks 1.' + - 'done_when 6, MEASURED on lambda (cpu lane, qwen2.5-coder-1.5b Q4_K_M, `--verbs run,serve`), rerun after the teardown fix and finishing unaided: PASS, 9 cells (3 run + 6 serve), 9 GREEN. apr serve, llama-server@d1d3c3396 and ollama 0.33.2 each answered every golden prompt in both modes. No server process and no ollama import was left behind. apr serve answers 2+2 with "4" from an 18-token prompt, where llama-server and ollama send 36 and answer "2 + 2 equals 4.": apr serve does not inject the GGUF template''s default system prompt. It is recorded; the byte-equal tmpl rows against HF are what will judge it.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Refs #3739 (slice 4; not a closing keyword for the parent). Closes #3797 when merged. STACKED on crux-3739-chat (#3796), so the diff to judge is `origin/crux-3739-chat...HEAD`. SCOPE: scripts/crux_inference_dogfood.sh (the serve cell, the `serve` verb token mapped to key `serve run`, emit_gen/cell_result/rows_for carrying verb and mode), scripts/lib/crux_openai_client.py (new), scripts/lib/crux_inference_judge.py (serve parsing, the `mode` key part), scripts/check_crux_inference_judge.sh (serve rows), evidence/crux/verb-correspondence.yaml (`modes` on the serve run entry, regenerated from the same v1.1 surface, sha 207c3297), and this fragment. NOT in this slice and named in `not_covered`: TTFT and decode rate through `apr test llm bench` (the canonical client, PERF-009, the next increment), and max accepted context. The client computes no rate: check_no_competing_harnesses stays count=0.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 65bc185996..d1bbf0c562 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -20043,3 +20043,25 @@ roadmap: labels: - kind:code notes: 'Refs #3739 (slice 3; not a closing keyword for the parent). Closes #3795 when this merges. STACKED on crux-3774-verb-correspondence (#3789), so the diff to judge is `origin/crux-3774-verb-correspondence...HEAD`. SCOPE: scripts/crux_inference_dogfood.sh (the chat verb), scripts/lib/crux_pty_chat.py (new), scripts/lib/crux_inference_judge.py (chat parsing), scripts/check_crux_inference_judge.sh (chat rows), scripts/crux_inference_prompts.json (the chat prompt; the golden drift check covers only the golden rung), and this fragment. FOUND while building it: #3794, where `apr chat --no-gpu` still uploads the GGUF to CUDA (VRAM held outside the GPU lock; generation honours force_cpu, the load path does not) and apr chat reports no machine-readable backend. The pty helper starts only the argv it is given and reads no clock for a metric (its per-turn timeout is a bound), so the competing-harness count stays 0.' +- id: PMAT-3797 + github_issue: 3797 + item_type: task + title: 'CRUX slice 4 (#3739): the serve verb — apr serve, llama-server and ollama through ONE OpenAI client, streaming + non-streaming, plus hf/llamafile servers' + status: in_progress + priority: critical + assigned_to: aprender-76 + created: 2026-09-21T23:55:00Z + updated: 2026-09-22T00:40:00Z + spec: null + acceptance_criteria: + - 'Issue #3797 done_when 1, verbatim: "`scripts/crux_inference_dogfood.sh --verbs run,serve` runs ONE serve cell per model under one hold of the GPU lock: `apr serve run` and `llama-server` each load the model once, next to the running ollama daemon; every golden prompt goes through `scripts/lib/crux_openai_client.py`, non-streaming and streaming; then the servers stop, and ollama unloads (keep_alive 0, plus the unload wait)." Teardown kills every server before waiting, and waits on those pids only. The first real run hung here: a bare `wait` after the first kill blocked on the other server.' + - 'done_when 2, verbatim: "One client for every server. `crux_openai_client.py` sends the same body (model, messages, max_tokens, temperature, seed, stream) and writes the row-contract JSON `{\"text\", \"reported\": {device, stream, finish_reason, usage, chunks}}`. It reads no clock and computes no rate." MEASURED against the three servers on lambda (cpu): non-stream and stream both answer, and the usage is transcribed (apr serve prompt_tokens 18; llama-server and ollama 36).' + - 'done_when 3/4, verbatim: "Plugin engines (hf, llamafile) run `gen --verb ''serve run''` ... The judge reads every serve row through the client JSON and applies the same RED rule. The key gains an additive `mode` (nonstream | stream) for serve cells only. apr serve''s cells carry `backend_verified: false`." Per aprender-97 (#3715), the correspondence ENTRY for `serve run` declares `modes: [nonstream, stream]`, with per-engine modes (llama.cpp and ollama both, hf and llamafile nonstream), so pv never names a verb.' + - 'done_when 5, MEASURED: `bash scripts/check_crux_inference_judge.sh` gives 44 ok, 0 broke. The serve rows: stream and non-stream are distinct GREEN cells with apr serve''s backend unverified; an apr serve that answered nothing (connection refused) while llama-server answered is RED. Mutants: dropping the mode from the key breaks 1; dropping the serve backend flag breaks 1.' + - 'done_when 6, MEASURED on lambda (cpu lane, qwen2.5-coder-1.5b Q4_K_M, `--verbs run,serve`), rerun after the teardown fix and finishing unaided: PASS, 9 cells (3 run + 6 serve), 9 GREEN. apr serve, llama-server@d1d3c3396 and ollama 0.33.2 each answered every golden prompt in both modes. No server process and no ollama import was left behind. apr serve answers 2+2 with "4" from an 18-token prompt, where llama-server and ollama send 36 and answer "2 + 2 equals 4.": apr serve does not inject the GGUF template''s default system prompt. It is recorded; the byte-equal tmpl rows against HF are what will judge it.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Refs #3739 (slice 4; not a closing keyword for the parent). Closes #3797 when merged. STACKED on crux-3739-chat (#3796), so the diff to judge is `origin/crux-3739-chat...HEAD`. SCOPE: scripts/crux_inference_dogfood.sh (the serve cell, the `serve` verb token mapped to key `serve run`, emit_gen/cell_result/rows_for carrying verb and mode), scripts/lib/crux_openai_client.py (new), scripts/lib/crux_inference_judge.py (serve parsing, the `mode` key part), scripts/check_crux_inference_judge.sh (serve rows), evidence/crux/verb-correspondence.yaml (`modes` on the serve run entry, regenerated from the same v1.1 surface, sha 207c3297), and this fragment. NOT in this slice and named in `not_covered`: TTFT and decode rate through `apr test llm bench` (the canonical client, PERF-009, the next increment), and max accepted context. The client computes no rate: check_no_competing_harnesses stays count=0.' diff --git a/evidence/crux/verb-correspondence.yaml b/evidence/crux/verb-correspondence.yaml index 8b51181a71..31ed70e719 100644 --- a/evidence/crux/verb-correspondence.yaml +++ b/evidence/crux/verb-correspondence.yaml @@ -12,7 +12,8 @@ engines: - ollama - hf - llamafile -rule: 'every leaf of the surface has exactly one entry; every entry states every engine, as a mapping or {none: }. A mapped engine is a required :CruxCell (#3715/#3745 S2 reads this file).' +rule: 'every leaf of the surface has exactly one entry; every entry states every engine, as a mapping or {none: }. A mapped engine is a required :CruxCell (#3715/#3745 S2 reads this file). An entry + with `modes` owes at least one :CruxCell per declared mode; an entry without owes rows with no mode.' verbs: - key: attn-parity-lint comparators: @@ -2251,25 +2252,38 @@ verbs: none: a pre-flight capacity plan (VRAM budget, roofline); it runs no model - key: serve run judged_by: golden + modes: + - nonstream + - stream comparators: llama.cpp: binary: llama-server args: --port: --port --context-length: -c + modes: + - nonstream + - stream ollama: subcommand: serve args: {} + modes: + - nonstream + - stream hf: api: transformers serve args: --port: --port + modes: + - nonstream llamafile: binary: llamafile args: : --server --port: --port --context-length: -c + modes: + - nonstream - key: shard comparators: llama.cpp: diff --git a/scripts/check_crux_inference_judge.sh b/scripts/check_crux_inference_judge.sh index 4655fadc5d..0091421dcf 100755 --- a/scripts/check_crux_inference_judge.sh +++ b/scripts/check_crux_inference_judge.sh @@ -387,7 +387,7 @@ case "$got" in "False degenerate output"*) ok "hf's degenerate '!!!!' cannot vou # C1-C3. the chat verb (#3739 slice 3): judged on the FINAL turn; apr's backend is # recorded as unverified until apr chat reports one (#3794). apr_chat_out() { # apr_chat_out — apr chat's transcript shape - printf 'Detected ChatML chat template\nYou: [3.9s, ~2 tok/s]\nAssistant: %s\nYou: [5.9s, ~1 tok/s]\nAssistant: %s\nYou: \nGoodbye!\n' "$3" "$4" > "$1/apr-$2.out" + printf 'Detected ChatML chat template\nYou: \nAssistant: %s\nYou: \nAssistant: %s\nYou: \nGoodbye!\n' "$3" "$4" > "$1/apr-$2.out" : > "$1/apr-$2.err" } chat_json() { # chat_json — the pty helper's JSON @@ -416,6 +416,39 @@ ROW_VERB=chat row "$d/manifest.jsonl" llama.cpp $CP 0 "$d/llama-$CP.json" "" run_judge "$d"; GOT_RC=$? expect "an apr chat transcript with no Assistant turn is no answer (RED)" "$d" 1 $CP RED +# S1-S3. the serve verb (#3739 slice 4): every server through the ONE OpenAI client, +# nonstream and stream as distinct cells (the key's additive `mode`). +serve_json() { # serve_json [error] — crux_openai_client.py's output shape + python3 -c 'import json,sys +t = sys.argv[5] or None +d = {"text": t, "reported": {"device": "fixture", "stream": sys.argv[4] == "stream", "finish_reason": "stop" if t else None, "usage": None, "chunks": 0}} +if len(sys.argv) > 6: d["error"] = sys.argv[6] +json.dump(d, open(sys.argv[1], "w"))' "$1/$2-$3-$4.json" "$2" "$3" "$4" "$5" "${@:6}" +} +serve_row() { # serve_row + python3 - "$@" <<'PY2' +import json, sys +m, eng, pid, mode, rc, o = sys.argv[1:7] +open(m, "a").write(json.dumps({"kind": "gen", "engine": eng, "prompt_id": pid, "rc": int(rc), "stdout": o, "stderr": None, + "refused": None, "model_sha256": "%s", "host": "fixture", "verb": "serve run", "thinking": "off", "backend": "gpu", + "mode": mode}) + "\n") +PY2 +} +d=$(newcase serve_green); control_green "$d" +for mode in nonstream stream; do + serve_json "$d" apr $P $mode "4"; serve_json "$d" llama $P $mode "2 + 2 equals 4." + serve_row "$d/manifest.jsonl" apr $P $mode 0 "$d/apr-$P-$mode.json"; serve_row "$d/manifest.jsonl" llama.cpp $P $mode 0 "$d/llama-$P-$mode.json" +done +run_judge "$d"; GOT_RC=$? +got=$(python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); print(sorted((c["key"].get("mode"), c["verdict"], c["engines"]["apr"].get("backend_verified")) for c in r["cells"] if c["key"]["verb"]=="serve run"))' "$d/receipt.json" 2>/dev/null) +[ "$GOT_RC" = 0 ] && [ "$got" = "[('nonstream', 'GREEN', False), ('stream', 'GREEN', False)]" ] && ok "serve nonstream and stream are distinct GREEN cells; apr serve's backend unverified" || broke "serve green: rc $GOT_RC '$got'" +d=$(newcase serve_apr_refused); control_green "$d" +serve_json "$d" apr $P nonstream "" "URLError: [Errno 111] Connection refused"; serve_json "$d" llama $P nonstream "2 + 2 equals 4." +serve_row "$d/manifest.jsonl" apr $P nonstream 3 "$d/apr-$P-nonstream.json"; serve_row "$d/manifest.jsonl" llama.cpp $P nonstream 0 "$d/llama-$P-nonstream.json" +run_judge "$d"; GOT_RC=$? +got=$(python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); print([c["verdict"] for c in r["cells"] if c["key"]["verb"]=="serve run"])' "$d/receipt.json" 2>/dev/null) +[ "$GOT_RC" = 1 ] && [ "$got" = "['RED']" ] && ok "an apr serve that answered nothing while llama-server answered is RED" || broke "serve apr refused: rc $GOT_RC '$got'" + # D1-D4. tok: raw-text ids, BYTE-EQUAL or RED. d=$(newcase tok_equal); control_green "$d" printf '{"tokens": [9707, 11, 1879]}\n' > "$d/apr-tok.json"; printf '{"tokens": [9707, 11, 1879]}\n' > "$d/hf-tok.json" diff --git a/scripts/crux_inference_dogfood.sh b/scripts/crux_inference_dogfood.sh index f2f48b840e..94292e9249 100755 --- a/scripts/crux_inference_dogfood.sh +++ b/scripts/crux_inference_dogfood.sh @@ -99,11 +99,12 @@ done [ -n "$VERSION" ] || decline "usage: $0 --model ..." [ "${#MODELS[@]}" -gt 0 ] || decline "no --model given; this slice takes models by path" case "$BACKEND" in gpu|cpu) ;; *) decline "--backend is gpu or cpu, got '$BACKEND'" ;; esac -case ",$VERBS," in - ,run,|,run,chat,|,chat,run,) ;; - *,chat,*) decline "verbs '$VERBS': chat needs run beside it (--verbs run,chat), because the positive control lives in the run verb" ;; - *) decline "verbs '$VERBS': this driver runs 'run' and 'chat'; serve and code are later slices of #3739 and are listed as not covered in every receipt" ;; -esac +# Verbs: run (always: the positive control lives there), chat, serve (the +# correspondence key `serve run`). code is a later slice of #3739. +case ",$VERBS," in *,run,*) ;; *) decline "verbs '$VERBS': run must be included, because the positive control lives in the run verb" ;; esac +for v in ${VERBS//,/ }; do + case "$v" in run|chat|serve) ;; *) decline "verb '$v': this driver runs run, chat and serve; code is a later slice of #3739" ;; esac +done want() { case ",$ENGINES," in *",$1,"*) return 0 ;; *) return 1 ;; esac; } want apr || decline "apr is the subject; --engines must include it" [ -f "$PROMPTS" ] || decline "prompt set $PROMPTS not found" @@ -302,13 +303,13 @@ run_cell() { # run_cell ; sets CELL_WHY when the lock was not had fi return "$rc" } -cell_result() { # cell_result [stdout file] — the engine's row from its cell files +cell_result() { # cell_result [stdout file] [mode] — the engine's row from its cell files local rc unl="" - if [ -n "$CELL_WHY" ]; then emit_gen "$1" "$2" "" "" "" "$CELL_WHY"; return; fi - if [ ! -f "$3.rc" ]; then emit_gen "$1" "$2" "" "" "" "the cell ran but this engine's line never finished"; return; fi + if [ -n "$CELL_WHY" ]; then emit_gen "$1" "$2" "" "" "" "$CELL_WHY" "" "${5:-}"; return; fi + if [ ! -f "$3.rc" ]; then emit_gen "$1" "$2" "" "" "" "the cell ran but this engine's line never finished" "" "${5:-}"; return; fi rc=$(cat "$3.rc") [ -f "$3.unloaded" ] && unl=$(cat "$3.unloaded") - emit_gen "$1" "$2" "$rc" "${4:-$3.out}" "$3.err" "" "$unl" + emit_gen "$1" "$2" "$rc" "${4:-$3.out}" "$3.err" "" "$unl" "${5:-}" } cell_add_stdin() { # cell_add_stdin cmd... — like cell_add, fed a file local cell="$1" prefix="$2" input="$3"; shift 3 @@ -316,29 +317,32 @@ cell_add_stdin() { # cell_add_stdin cmd... — like printf '> %q 2> %q < %q; echo $? > %q\n' "$prefix.out" "$prefix.err" "$input" "$prefix.rc"; } >> "$cell" } -emit_gen() { # emit_gen [ollama_unloaded] - python3 - "$MANIFEST" "$1" "$2" "$3" "$4" "$5" "$6" "${7:-}" "$SHA" "$HOST" "$VERB" "$THINK" "$BACKEND" <<'PY' +emit_gen() { # emit_gen [ollama_unloaded] [mode] + python3 - "$MANIFEST" "$1" "$2" "$3" "$4" "$5" "$6" "${7:-}" "$SHA" "$HOST" "${VERB_KEY:-$VERB}" "$THINK" "$BACKEND" "${8:-}" <<'PY' import json, sys -m, eng, pid, rc, o, e, ref, unl, sha, host, verb, think, be = sys.argv[1:14] +m, eng, pid, rc, o, e, ref, unl, sha, host, verb, think, be, mode = sys.argv[1:15] row = {"kind": "gen", "engine": eng, "prompt_id": pid, "rc": int(rc) if rc.lstrip("-").isdigit() else None, "stdout": o or None, "stderr": e or None, "refused": ref or None, "model_sha256": sha, "host": host, "verb": verb, "thinking": think, "backend": be} if unl: row["ollama_unloaded"] = unl == "true" +if mode: + row["mode"] = mode open(m, "a").write(json.dumps(row) + "\n") PY } -rows_for() { # rows_for : how many gen rows that engine has for the prompt - python3 - "$MANIFEST" "$1" "$2" "$SHA" <<'PY' +rows_for() { # rows_for : how many gen rows that engine has for the prompt in this verb + python3 - "$MANIFEST" "$1" "$2" "$SHA" "${VERB_KEY:-$VERB}" <<'PY' import json, sys -m, eng, pid, sha = sys.argv[1:5] +m, eng, pid, sha, verb = sys.argv[1:6] n = 0 for line in open(m): if line.strip(): r = json.loads(line) - n += r.get("kind") == "gen" and r.get("engine") == eng and r.get("prompt_id") == pid and r.get("model_sha256") == sha + n += (r.get("kind") == "gen" and r.get("engine") == eng and r.get("prompt_id") == pid + and r.get("model_sha256") == sha and r.get("verb") == verb) print(n) PY } @@ -386,6 +390,92 @@ PY [ "$ok" = 1 ] } +# ---- the serve verb: ONE cell per model ------------------------------------------- +# The issue, verbatim: "serve: `apr serve` · `llama-server` · `ollama serve`, all through +# the SAME OpenAI `/v1/chat/completions` client, streaming + non-streaming". Each server +# loads the model ONCE per cell, every run prompt goes through scripts/lib/ +# crux_openai_client.py twice (nonstream, stream), then the servers stop and ollama +# unloads, all inside one hold of the GPU lock. The plugin engines (hf, llamafile) are +# `gen --verb 'serve run'`: they run their own server per call, non-streaming. +serve_wait_line() { # serve_wait_line + printf 'for i in $(seq 1 %q); do curl -sf http://127.0.0.1:%q%s > /dev/null 2>&1 && break; kill -0 "$(cat %q)" 2>/dev/null || break; sleep 1; done\n' \ + "$TMO" "$2" "$3" "$4" >> "$1" +} +serve_cell() { + local d="$WORK/$SHA12/serve" cell pa pl pid mode st ext_extra + mkdir -p "$d" + cell="$d/cell-serve.sh" + pa=$(free_port); pl=$(free_port) + printf '#!/usr/bin/env bash\n# one CRUX serve cell: every server loads the model once; every prompt, nonstream + stream\n' > "$cell" + { printf '%q ' "$APR" serve run "$M" --port "$pa" "$APR_BE"; printf '> %q 2>&1 < /dev/null &\necho $! > %q\n' "$d/apr-serve.log" "$d/apr-serve.pid"; } >> "$cell" + serve_wait_line "$cell" "$pa" /health "$d/apr-serve.pid" + if [ "$LLAMA_OK" = 1 ]; then + { printf '%q ' "$LLAMA_SERVER" -m "$M" --port "$pl" --host 127.0.0.1 -c "$CTX" -ngl "$NGL" "${LLAMA_DEV[@]}" + printf '> %q 2>&1 < /dev/null &\necho $! > %q\n' "$d/llama-serve.log" "$d/llama-serve.pid"; } >> "$cell" + serve_wait_line "$cell" "$pl" /health "$d/llama-serve.pid" + fi + for pid in $PIDS; do + for mode in nonstream stream; do + st=(); [ "$mode" = stream ] && st=(--stream) + cell_add "$cell" "$d/apr-$pid-$mode" python3 scripts/lib/crux_openai_client.py --url "http://127.0.0.1:$pa" \ + --model default --messages "$WORK/messages-$pid.json" --max-tokens "$MAXTOK" --temperature "$TEMP" \ + --seed "$SEED" "${st[@]}" --device "apr serve $APR_BE" --out "$d/apr-$pid-$mode.json" + [ "$LLAMA_OK" = 1 ] && cell_add "$cell" "$d/llama-$pid-$mode" python3 scripts/lib/crux_openai_client.py \ + --url "http://127.0.0.1:$pl" --model gguf --messages "$WORK/messages-$pid.json" --max-tokens "$MAXTOK" \ + --temperature "$TEMP" --seed "$SEED" "${st[@]}" --device "$LLAMA_DEVICE" --out "$d/llama-$pid-$mode.json" + [ "$OLLAMA_OK" = 1 ] && [ -z "$OL_REFUSED" ] && cell_add "$cell" "$d/ollama-$pid-$mode" python3 scripts/lib/crux_openai_client.py \ + --url "$OLLAMA_HOST_URL" --model "$OL_NAME" --messages "$WORK/messages-$pid.json" --max-tokens "$MAXTOK" \ + --temperature "$TEMP" --seed "$SEED" "${st[@]}" --device "$OL_DEVICE" --extra '{"keep_alive": 0}' \ + --out "$d/ollama-$pid-$mode.json" + done + done + # Stop EVERY server before waiting, and wait on those pids only: a bare `wait` after + # the first kill blocks on the other server, which is still running (found on the + # first real serve run: the cell hung at teardown with every request answered). + srv_pids=("$d/apr-serve.pid"); [ "$LLAMA_OK" = 1 ] && srv_pids+=("$d/llama-serve.pid") + { printf 'srv=""; for f in'; printf ' %q' "${srv_pids[@]}"; printf '; do srv="$srv $(cat "$f")"; done\n' + printf 'kill $srv 2> /dev/null; wait $srv 2> /dev/null\n'; } >> "$cell" + [ "$OLLAMA_OK" = 1 ] && [ -z "$OL_REFUSED" ] && cell_add_ollama_unload "$cell" "$d/ollama-serve" "$OL_NAME" + for eng in hf llamafile; do + want "$eng" && [ "${EXT_OK[$eng]}" = 1 ] || continue + [ "$eng" = hf ] && [ -n "$HF_MODEL_WHY" ] && continue + case "${EXT_SCRIPT[$eng]}" in *.py) ext_run=(python3) ;; *) ext_run=(bash) ;; esac + ext_extra=() + [ "$eng" = llamafile ] && ext_extra=(--interface server) + [ "$eng" = hf ] && ext_extra=("${HF_SRC[@]}") + for pid in $PIDS; do + cell_add "$cell" "$d/$eng-$pid.driver" "${ext_run[@]}" "${EXT_SCRIPT[$eng]}" gen \ + --model "$M" --model-sha256 "$SHA" --verb "serve run" --prompt-id "$pid" \ + --messages "$WORK/messages-$pid.json" --prompt-file "$WORK/prompt-$pid.txt" \ + --thinking "$THINK" --backend "$BACKEND" --host "$HOST" \ + --max-tokens "$MAXTOK" --seed "$SEED" --temperature "$TEMP" --context "$CTX" "${ext_extra[@]}" + done + done + printf 'exit 0\n' >> "$cell" + + declare -A before=() + for pid in $PIDS; do before[hf-$pid]=$(rows_for hf "$pid"); before[llamafile-$pid]=$(rows_for llamafile "$pid"); done + run_cell "$cell" + for pid in $PIDS; do + for mode in nonstream stream; do + cell_result apr "$pid" "$d/apr-$pid-$mode" "$d/apr-$pid-$mode.json" "$mode" + if [ "$LLAMA_OK" = 1 ]; then cell_result llama.cpp "$pid" "$d/llama-$pid-$mode" "$d/llama-$pid-$mode.json" "$mode" + elif want llama.cpp; then emit_gen llama.cpp "$pid" "" "" "" "$LLAMA_WHY" "" "$mode"; fi + if [ "$OLLAMA_OK" = 1 ] && [ -z "$OL_REFUSED" ]; then cell_result ollama "$pid" "$d/ollama-$pid-$mode" "$d/ollama-$pid-$mode.json" "$mode" + elif want ollama; then emit_gen ollama "$pid" "" "" "" "${OL_REFUSED:-$OLLAMA_WHY}" "" "$mode"; fi + done + for eng in hf llamafile; do + want "$eng" || continue + if [ "${EXT_OK[$eng]}" != 1 ]; then emit_gen "$eng" "$pid" "" "" "" "${EXT_WHY[$eng]}" "" nonstream; continue; fi + if [ "$eng" = hf ] && [ -n "$HF_MODEL_WHY" ]; then emit_gen hf "$pid" "" "" "" "$HF_MODEL_WHY" "" nonstream; continue; fi + if [ -n "$CELL_WHY" ]; then emit_gen "$eng" "$pid" "" "" "" "$CELL_WHY" "" nonstream + elif [ "$(rows_for "$eng" "$pid")" -le "${before[$eng-$pid]}" ]; then + emit_gen "$eng" "$pid" "" "" "" "engine driver ${EXT_SCRIPT[$eng]} gen exited $(cat "$d/$eng-$pid.driver.rc" 2>/dev/null || echo '?') without appending a row: $(tail -c 200 "$d/$eng-$pid.driver.err" 2>/dev/null | tr '\n' ' ')" "" nonstream + fi + done + done +} + # ---- the run --------------------------------------------------------------------- printf -- '--- CRUX inference dogfood %s on %s (%s lane) ---\n' "$VERSION" "$HOST" "$BACKEND" printf ' apr %s\n' "$APR_VERSION_LINE" @@ -498,6 +588,12 @@ PY fi for VERB in ${VERBS//,/ }; do + VERB_KEY=$VERB + if [ "$VERB" = serve ]; then + VERB_KEY="serve run" + serve_cell + continue + fi for pid in $(pids_for "$VERB"); do content=$(cat "$WORK/prompt-$pid.txt") d="$WORK/$SHA12/$VERB" @@ -607,12 +703,12 @@ meta = { **json.load(open(ext)), "models": [json.loads(l) for l in open(models) if l.strip()], "not_covered": [ - ("verbs serve and code (later slices of #3739)" if "chat" in verbs.split(",") - else "verbs chat, serve and code (later slices of #3739)"), + "verbs not run here: " + ", ".join(v for v in ("chat", "serve", "code") if v not in verbs.split(",")), + *(["apr serve's backend is unverified: its responses report none"] if "serve" in verbs.split(",") else []), *(["apr chat's backend is unverified: apr chat reports none (#3794)"] if "chat" in verbs.split(",") else []), "thinking ON (apr has no toggle until #3723)", "consumer-brief context rungs (#3716) and each engine's max accepted context", - "TTFT, which belongs to the serve verb's single OpenAI client", + "TTFT and decode rate: the serve verb's measurement goes through `apr test llm bench` (PERF-009), the next increment", ], } json.dump(meta, open(out, "w"), indent=2) diff --git a/scripts/lib/crux_inference_judge.py b/scripts/lib/crux_inference_judge.py index abef8a2b62..d77ece439f 100644 --- a/scripts/lib/crux_inference_judge.py +++ b/scripts/lib/crux_inference_judge.py @@ -104,7 +104,8 @@ def parse_apr(stdout, stderr): "reported_by": "apr", "prompt_tokens": out["prompt_token_count"], "completion_tokens": doc.get("tokens_generated"), - "decode_rate_tokens_per_second": doc.get("tok_per_sec"), + "decode_rate": doc.get("tok_per_sec"), + "rate_unit": "tokens per second, as the engine reported it", "inference_ms": doc.get("inference_time_ms"), } text = doc.get("text") @@ -137,8 +138,9 @@ def parse_llamacpp_cli(stdout, prompt_text): out["answer"] = rest[:m.start()].strip() out["reported"] = { "reported_by": "llama.cpp", - "prompt_rate_tokens_per_second": float(m.group(1)), - "decode_rate_tokens_per_second": float(m.group(2)), + "prompt_rate": float(m.group(1)), + "decode_rate": float(m.group(2)), + "rate_unit": "tokens per second, as the engine reported it", "prompt_tokens": None, "completion_tokens": None, } @@ -162,8 +164,9 @@ def num(key): "reported_by": "ollama", "prompt_tokens": int(pe) if pe is not None else None, "completion_tokens": int(ev) if ev is not None else None, - "prompt_rate_tokens_per_second": num("prompt eval rate"), - "decode_rate_tokens_per_second": num("eval rate"), + "prompt_rate": num("prompt eval rate"), + "decode_rate": num("eval rate"), + "rate_unit": "tokens per second, as the engine reported it", "load_duration": stats.get("load duration"), "total_duration": stats.get("total duration"), } @@ -245,7 +248,17 @@ def engine_entry(row, prompt): stdout, stderr = read_text(row.get("stdout")), read_text(row.get("stderr")) content = prompt["messages"][-1]["content"] engine = row["engine"] - if row.get("verb") == "chat": + if row.get("verb") == "serve run": + # serve (#3739 slice 4): every server, apr's included, is read through the ONE + # OpenAI client's contract JSON (scripts/lib/crux_openai_client.py). + p = parse_engine_json(stdout) + if engine == "apr": + # apr serve's responses report no backend: recorded, not scored as verified. + e["backend_verified"] = False + elif engine not in COMPARATORS: + e["why"] = "unknown engine %r" % engine + return e + elif row.get("verb") == "chat": # chat (#3739 slice 3): apr's own transcript; every other engine writes the # row-contract JSON (the pty helper for llama.cpp/ollama, the plugin drivers). if engine == "apr": @@ -503,7 +516,10 @@ def collect(args): keys = [] by_key = {} for r in gens: - k = (r["model_sha256"], r["host"], r["verb"], r["thinking"], prompts[r["prompt_id"]]["rung"], r["prompt_id"]) + # serve cells come in two modes (nonstream, stream): an additive key part, + # absent for run and chat. Plugin serve rows are non-streaming. + mode = r.get("mode") or ("nonstream" if r["verb"] == "serve run" else "") + k = (r["model_sha256"], r["host"], r["verb"], r["thinking"], prompts[r["prompt_id"]]["rung"], r["prompt_id"], mode) if k not in by_key: keys.append(k) by_key[k] = {} @@ -525,7 +541,8 @@ def collect(args): entries[eng]["correct"] = ok[eng] said = {e: norm(v["answer"]) for e, v in entries.items() if v.get("answered")} cells.append({ - "key": dict(zip(("model_sha256", "host", "verb", "thinking", "rung", "prompt_id"), k)), + "key": dict(zip(("model_sha256", "host", "verb", "thinking", "rung", "prompt_id"), k[:6]), + **({"mode": k[6]} if k[6] else {})), "verdict": verdict, # additive (aprender-97, #3715): pv reads the control by this flag, never by a prompt name "positive_control": bool(prompt.get("control")), @@ -633,7 +650,8 @@ def render_md(r): else: tps = "unmeasured: " + tp.get("why", "") lines.append("| %s | %s | %s | %s | **%s** | %s | %s |" % ( - names.get(k["model_sha256"], k["model_sha256"][:12]), k["verb"], k["thinking"], k["prompt_id"], + names.get(k["model_sha256"], k["model_sha256"][:12]), k["verb"], k["thinking"], + k["prompt_id"] + ("@" + k["mode"] if k.get("mode") else ""), c["verdict"], " | ".join(cols), tps)) if r.get("deterministic"): lines += ["", "Deterministic rows (byte-equal or RED): %s" % s.get("deterministic"), "", diff --git a/scripts/lib/crux_openai_client.py b/scripts/lib/crux_openai_client.py new file mode 100755 index 0000000000..e6743bc2e6 --- /dev/null +++ b/scripts/lib/crux_openai_client.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""crux_openai_client.py: the ONE OpenAI client every CRUX `serve run` cell uses (#3739, the serve verb). + +The issue, verbatim: "serve: `apr serve` · `llama-server` · `ollama serve`, all through +the SAME OpenAI `/v1/chat/completions` client, streaming + non-streaming". This is that +client: one request, the same body shape for every server, and the answer written as +the row-contract JSON the judge reads for every other engine: + {"text": , "reported": {"device", "stream", "finish_reason", "usage", "chunks"}} + +It measures NOTHING about speed: it reads no clock and computes no rate. TTFT and +decode rate for the serve verb come from `apr test llm bench`, the canonical client +(PERF-009). `usage` is transcribed as the server reported it, and never judged. + +Exit: 0 an answer was read · 3 the server refused, errored or sent no content (the +JSON says why) · 2 usage. +""" +import argparse +import json +import sys +import urllib.error +import urllib.request + + +def main(argv): + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True, help="base URL, e.g. http://127.0.0.1:8080") + ap.add_argument("--model", required=True) + ap.add_argument("--messages", required=True, help='JSON file: {"messages": [...]}') + ap.add_argument("--max-tokens", type=int, required=True) + ap.add_argument("--temperature", type=float, required=True) + ap.add_argument("--seed", type=int, required=True) + ap.add_argument("--stream", action="store_true") + ap.add_argument("--device", default="") + ap.add_argument("--timeout", type=float, default=600) + ap.add_argument("--extra", default="{}", help="JSON merged into the request body (e.g. keep_alive)") + ap.add_argument("--out", required=True) + a = ap.parse_args(argv) + body = {"model": a.model, "messages": json.load(open(a.messages))["messages"], + "max_tokens": a.max_tokens, "temperature": a.temperature, "seed": a.seed, + "stream": bool(a.stream)} + body.update(json.loads(a.extra)) + out = {"text": None, "reported": {"device": a.device or None, "stream": bool(a.stream), + "finish_reason": None, "usage": None, "chunks": 0}} + req = urllib.request.Request(a.url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + rc = 0 + try: + with urllib.request.urlopen(req, timeout=a.timeout) as resp: + if a.stream: + parts = [] + for raw in resp: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + chunk = json.loads(data) + out["reported"]["chunks"] += 1 + if chunk.get("usage"): + out["reported"]["usage"] = chunk["usage"] + for ch in chunk.get("choices") or []: + parts.append((ch.get("delta") or {}).get("content") or "") + if ch.get("finish_reason"): + out["reported"]["finish_reason"] = ch["finish_reason"] + out["text"] = "".join(parts) + else: + doc = json.loads(resp.read().decode("utf-8", "replace")) + ch = (doc.get("choices") or [{}])[0] + out["text"] = (ch.get("message") or {}).get("content") + out["reported"]["finish_reason"] = ch.get("finish_reason") + out["reported"]["usage"] = doc.get("usage") + except urllib.error.HTTPError as exc: + out["error"] = "HTTP %s: %s" % (exc.code, exc.read().decode("utf-8", "replace")[:300]) + rc = 3 + except (urllib.error.URLError, OSError, ValueError) as exc: + out["error"] = "%s: %s" % (type(exc).__name__, exc) + rc = 3 + if rc == 0 and not out["text"]: + out["error"] = "the server returned no content" + rc = 3 + if rc != 0: + out["text"] = None + with open(a.out, "w", encoding="utf-8") as fh: + json.dump(out, fh, ensure_ascii=False) + return rc + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))