feat(serve): logprobs on the chat route, and the V3 serving-shape probe that reads them (PMAT-4971)

README.md

@@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift.
 | Metric | Count | Source of truth |
 |-------:|------:|---|
 | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) |
-| Provable contracts | **<!-- CONTRACT_COUNT_START -->1903<!-- CONTRACT_COUNT_END -->** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) |
+| Provable contracts | **<!-- CONTRACT_COUNT_START -->1904<!-- CONTRACT_COUNT_END -->** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) |
 | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^  - name:'` gives 119 — other same-indent `name:` keys exist in the file) |
 | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` |
 | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) |
@@ -403,7 +403,7 @@ falsification_tests:
   prediction: apr validate bad-model.apr exits non-zero
 ```
 
-The tree carries <!-- CONTRACT_COUNT_START -->1903<!-- CONTRACT_COUNT_END --> contracts across inference, training, quantization, attention, FFN,
+The tree carries <!-- CONTRACT_COUNT_START -->1904<!-- CONTRACT_COUNT_END --> contracts across inference, training, quantization, attention, FFN,
 tokenization, model formats, CLI safety — and this README itself.
 
 ## Migration from old crates

contracts/apr-cli-commands-v1.yaml

@@ -486,8 +486,11 @@ commands:
         replay  the runner itself             (record, playbook, coverage, report)
 
       Routed today: `tensor` (PMAT-481 visual regression), `llm bench`
-      (GH-876 Milestone 2) and `llm ttft-verdict` (#4954). The rest land as they
-      are delegated to the probador library.
+      (GH-876 Milestone 2), `llm ttft-verdict` (#4954), `llm shape-check`
+      (#4971: checks a perf041 witness against the V3 serving-shape rule; reads
+      a file, no network) and `llm parity` (#4971: the perf041 batched-parity
+      probe, ported from the script; drives an endpoint and writes the
+      witness). The rest land as they are delegated to the probador library.
     requires_model: false
     side_effects: [filesystem, network]
     # tensor — export tensor activations for visual regression testing (PMAT-481)

contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json

@@ -1,7 +1,7 @@
 {
-  "census_id_set_sha256": "ced96e20e1de8700f4de7592803f602f279ccecc086cbb4fbd86b349b9028779",
+  "census_id_set_sha256": "04c86e59ff4920150a699521e5b23705cd4c75893e80e95ad4b6a5a68d5077bf",
   "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67",
-  "reasoner_git_sha": "28006ac347c6c8ff337874e39711ea67dbfe8bcf",
+  "reasoner_git_sha": "1f275af9265fe6b591ed884ee3218b01789c97bd",
   "checkable_n": 84,
   "result": {
     "kind": "model",
@@ -10,5 +10,5 @@
     }
   },
   "pc_reasoner": "fired",
-  "cpu_ms": 1
+  "cpu_ms": 2
 }

crates/apr-cli/src/commands/mod.rs

@@ -157,6 +157,12 @@ pub(crate) mod test_llm;
 pub(crate) mod test_llm_band;
 // #4954: `apr test llm ttft-verdict`, the TTFT ratio row as an exit code.
 pub(crate) mod test_llm_ttft;
+// #4971 V3-d: `apr test llm shape-check`, the CLI over perf_gate::v3_shape.
+pub(crate) mod test_llm_shape;
+// #4971 V3-d: `apr test llm parity`, the CLI over the perf041 probe's Rust
+// port (apr_test::llm::parity). It drives a server through tokio.
+#[cfg(feature = "inference")]
+pub(crate) mod test_llm_parity;
 // #2399: gated on the crate it actually needs (aprender-explain, aliased
 // `trueno-explain`) rather than on `full`, so `--features ptx` is enough and a
 // user does not have to pull CUDA + training to analyze a .ptx file.

crates/apr-cli/src/commands/serve/chat.rs

@@ -224,6 +224,12 @@ pub(crate) async fn safetensors_chat_completions_handler(
     use axum::response::{sse::Event, IntoResponse, Sse};
     use futures_util::stream;
 
+    if let Some(refused) =
+        super::handlers::logprobs_refusal_response(&request, "SafeTensors F32 CPU")
+    {
+        return refused;
+    }
+
     // Parse request - try structured first, fallback to raw JSON (GH-160)
     let parsed_request = match parse_chat_completion_request(&request) {
         Ok(req) => req,

crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs

@@ -55,6 +55,24 @@ async fn handle_apr_cpu_completion(
     }
 }
 
+/// #4971 (ASOC-INV-021): `backend` does not compute logprobs, so a request for
+/// them is refused rather than served without them: 501 naming `backend`, or
+/// 400/422 for a bad `logprobs`/`top_logprobs`, as realizar's router answers.
+#[cfg(feature = "inference")]
+#[allow(clippy::disallowed_methods)] // serde_json::json!() macro uses infallible unwrap
+pub(crate) fn logprobs_refusal_response(
+    req: &serde_json::Value,
+    backend: &str,
+) -> Option<axum::response::Response> {
+    use axum::{response::IntoResponse, Json};
+
+    let (status, reason) = realizar::api::json_logprobs_refusal(req, backend)?;
+    let error = serde_json::json!({
+        "error": {"message": reason, "type": "invalid_request_error", "param": "logprobs"}
+    });
+    Some((status, Json(error)).into_response())
+}
+
 /// GH-283: Validate the "model" field in the request matches the loaded model.
 ///
 /// Returns an error response if the model name doesn't match. Accepts "apr" as
@@ -340,6 +358,9 @@ async fn handle_apr_cpu_chat_completion(
     if let Some(err_response) = validate_request_model(req, &s.model_name) {
         return err_response;
     }
+    if let Some(refused) = logprobs_refusal_response(req, "APR CPU (AprTransformer)") {
+        return refused;
+    }
 
     let messages = req.get("messages").and_then(|m| m.as_array());
     let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false);

crates/apr-cli/src/commands/serve/handler_gpu_completion.rs

@@ -262,6 +262,10 @@ async fn handle_gpu_chat_completion(
             return err_response;
         }
     }
+    // #4971: run_gpu_generation computes no logprobs, so a request for them is refused.
+    if let Some(refused) = logprobs_refusal_response(&req, "APR CUDA (AprV2ModelCuda)") {
+        return refused;
+    }
 
     let messages = req.get("messages").and_then(|m| m.as_array());
     let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false);

crates/apr-cli/src/commands/serve/handlers.rs

@@ -357,6 +357,9 @@ async fn wgpu_chat_completion(
     state: Arc<WgpuInferenceState>,
     axum::Json(body): axum::Json<serde_json::Value>,
 ) -> axum::response::Response {
+    if let Some(refused) = logprobs_refusal_response(&body, "WGPU") {
+        return refused;
+    }
     // GH-665: Cap max_tokens to prevent hangs on large values
     let max_tokens = body["max_tokens"].as_u64().unwrap_or(64).min(4096) as usize;
     let stream = body["stream"].as_bool().unwrap_or(false);

crates/apr-cli/src/commands/serve/mod.rs

@@ -28,6 +28,9 @@ mod tests;
 mod tests_contract_enforcement;
 #[cfg(all(test, feature = "inference"))]
 mod tests_route_index_3979;
+// #4971: the chat routes outside realizar's router refuse logprobs (ASOC-INV-021).
+#[cfg(all(test, feature = "inference"))]
+mod tests_logprobs_4971;
 // PP-LLAMA-001 PP-14/PP-15/§9 #8: the offload report the served process
 // publishes. `inference`-gated because the report type comes from realizar.
 #[cfg(test)]
@@ -595,6 +598,38 @@ mod accelerator_guard_tests {
         );
     }
 
+    /// #4971 (ASOC-INV-021): the APR CUDA chat route computes no logprobs, so a
+    /// request for them must be refused before generation, never answered 200
+    /// without them. The handler is cuda-only and has no router test without a
+    /// GPU, so this reads its shipping source: comments dropped, `#[cfg(test)]` cut.
+    #[test]
+    fn the_cuda_chat_route_refuses_logprobs_before_it_generates() {
+        let whole = include_str!("handler_gpu_completion.rs");
+        let shipping = whole.split("#[cfg(test)]").next().unwrap_or(whole);
+        let src: String = shipping
+            .lines()
+            .filter(|l| !l.trim_start().starts_with("//"))
+            .collect::<Vec<_>>()
+            .join("\n");
+        let start = src
+            .find("async fn handle_gpu_chat_completion(")
+            .expect("the APR CUDA chat handler is gone");
+        let body = &src[start..];
+        let body = &body[..body.find("\nfn ").unwrap_or(body.len())];
+        let refuse = body.find(&format!("{}(&req,", "logprobs_refusal_response"));
+        let generate = body.find(&format!("{}(", "run_gpu_generation"));
+        match (refuse, generate) {
+            (Some(r), Some(g)) => assert!(
+                r < g,
+                "handle_gpu_chat_completion generates before it refuses logprobs"
+            ),
+            _ => panic!(
+                "handle_gpu_chat_completion does not refuse logprobs \
+                 (refusal {refuse:?}, generation {generate:?}): it answers 200 without them"
+            ),
+        }
+    }
+
     #[test]
     fn the_resolver_is_actually_called_from_a_decision_path() {
         let src = include_str!("handler_gpu_completion.rs");

crates/apr-cli/src/commands/serve/tests_logprobs_4971.rs

@@ -0,0 +1,93 @@
+//! #4971 (ASOC-INV-021): the `apr serve` chat routes realizar's router does not
+//! mount compute no logprobs, so a request for them is refused (501 naming the
+//! backend, 400/422 for a bad field) and never answered 200 without them.
+//! Driven through the REAL router builders, with no model loaded.
+#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use tower::ServiceExt;
+
+async fn chat(router: &axum::Router, body: serde_json::Value) -> (StatusCode, String) {
+    let req = Request::post("/v1/chat/completions")
+        .header("content-type", "application/json")
+        .body(Body::from(body.to_string()))
+        .unwrap();
+    let resp = router.clone().oneshot(req).await.unwrap();
+    let status = resp.status();
+    let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
+        .await
+        .unwrap();
+    (status, String::from_utf8_lossy(&bytes).into_owned())
+}
+
+/// Every refusal `router` must answer, naming `backend` on the 501; and a
+/// request that does not ask is not refused for logprobs.
+async fn refuses_logprobs(router: &axum::Router, backend: &str) {
+    let messages = serde_json::json!([{"role": "user", "content": "hi"}]);
+    // (logprobs, top_logprobs, status, words the reply must carry)
+    let table: &[(serde_json::Value, serde_json::Value, StatusCode, &str)] = &[
+        (
+            true.into(),
+            serde_json::Value::Null,
+            StatusCode::NOT_IMPLEMENTED,
+            backend,
+        ),
+        (true.into(), 3.into(), StatusCode::NOT_IMPLEMENTED, backend),
+        (
+            serde_json::Value::Null,
+            3.into(),
+            StatusCode::BAD_REQUEST,
+            "requires logprobs",
+        ),
+        (
+            true.into(),
+            21.into(),
+            StatusCode::UNPROCESSABLE_ENTITY,
+            "top_logprobs must be",
+        ),
+        (
+            "yes".into(),
+            serde_json::Value::Null,
+            StatusCode::UNPROCESSABLE_ENTITY,
+            "logprobs must be",
+        ),
+    ];
+    for (logprobs, top, want, words) in table {
+        let body = serde_json::json!({
+            "messages": messages, "max_tokens": 2, "logprobs": logprobs, "top_logprobs": top
+        });
+        let (status, reply) = chat(router, body.clone()).await;
+        assert_eq!(status, *want, "{backend} {body}: {reply}");
+        assert!(reply.contains(words), "{backend} {body}: {reply}");
+    }
+    let plain = serde_json::json!({"messages": messages, "max_tokens": 2, "logprobs": false});
+    let (status, reply) = chat(router, plain).await;
+    assert_ne!(status, StatusCode::NOT_IMPLEMENTED, "{backend}: {reply}");
+    assert!(
+        !reply.contains("`logprobs` is not supported"),
+        "{backend}: {reply}"
+    );
+}
+
+#[tokio::test]
+async fn the_apr_cpu_chat_route_refuses_logprobs() {
+    let r = super::handlers::build_demo_apr_cpu_router_for_test();
+    refuses_logprobs(&r, "APR CPU (AprTransformer)").await;
+}
+
+#[tokio::test]
+async fn the_safetensors_f32_chat_route_refuses_logprobs() {
+    let state = super::safetensors::SafeTensorsState {
+        transformer: None,
+        tokenizer_info: None,
+        model_path: "m.safetensors".into(),
+    };
+    let r = super::safetensors::safetensors_app(
+        serde_json::json!({"count": 0}),
+        true,
+        state,
+        "m".into(),
+    );
+    refuses_logprobs(&r, "SafeTensors F32 CPU").await;
+}

crates/apr-cli/src/commands/test_llm.rs

@@ -467,15 +467,42 @@ mod tests {
 // ===========================================================================
 use crate::LlmSubcommand;
 
-/// Route `apr test llm <SUB>` (GH-876 Milestone 2; PERF-025 band mode).
+/// Route `apr test llm <SUB>` (GH-876 Milestone 2; PERF-025 band mode;
+/// #4954 ttft-verdict and #4971 shape-check, which read files and need no
+/// runtime, and #4971 parity, the perf041 probe).
 ///
 /// # Errors
 /// Propagates whichever mode ran.
-#[cfg(feature = "inference")]
 pub fn dispatch(command: &LlmSubcommand) -> Result<()> {
-    if let LlmSubcommand::TtftVerdict { runs } = command {
-        return super::test_llm_ttft::run(runs);
+    match command {
+        LlmSubcommand::TtftVerdict { runs } => super::test_llm_ttft::run(runs),
+        LlmSubcommand::ShapeCheck { witness } => super::test_llm_shape::run(witness),
+        LlmSubcommand::Parity { .. } => dispatch_parity(command),
+        LlmSubcommand::Bench { .. } => dispatch_bench(command),
     }
+}
+
+/// `apr test llm parity`: the probe drives its server through tokio.
+#[cfg(feature = "inference")]
+fn dispatch_parity(command: &LlmSubcommand) -> Result<()> {
+    super::test_llm_parity::run(command)
+}
+
+/// As [`dispatch_bench`]'s stub: a minimal build names the missing feature.
+///
+/// # Errors
+/// Always: this build cannot run the probe.
+#[cfg(not(feature = "inference"))]
+fn dispatch_parity(_command: &LlmSubcommand) -> Result<()> {
+    Err(CliError::InferenceFailed(
+        "`apr test llm parity` needs the `inference` feature (this apr was built without it)"
+            .to_string(),
+    ))
+}
+
+/// `apr test llm bench`, in either of its two modes.
+#[cfg(feature = "inference")]
+fn dispatch_bench(command: &LlmSubcommand) -> Result<()> {
     let rt = tokio::runtime::Runtime::new()
         .map_err(|e| CliError::InferenceFailed(format!("tokio runtime: {e}")))?;
     match command {
@@ -486,21 +513,25 @@ pub fn dispatch(command: &LlmSubcommand) -> Result<()> {
 
 /// #4041: the benchmark drives its server through tokio, which only the `inference` feature brings. A minimal
 /// build (`--no-default-features`) names the missing feature instead of failing to compile.
-/// `ttft-verdict` reads files only, so it runs in every build (#4954).
+/// `ttft-verdict` and `shape-check` read files only and never reach here, so
+/// they run in every build (#4954, #4971).
 ///
 /// # Errors
 /// `bench`, always: this build cannot run the benchmark.
 #[cfg(not(feature = "inference"))]
-pub fn dispatch(command: &LlmSubcommand) -> Result<()> {
-    if let LlmSubcommand::TtftVerdict { runs } = command {
-        return super::test_llm_ttft::run(runs);
-    }
+fn dispatch_bench(_command: &LlmSubcommand) -> Result<()> {
     Err(CliError::InferenceFailed(
         "`apr test llm bench` needs the `inference` feature (this apr was built without it)"
             .to_string(),
     ))
 }
 
+/// The refusal of a `bench` mode handed another subcommand. `dispatch` routes
+/// only `bench` to them, so this names a routing defect if it ever prints.
+fn not_bench() -> CliError {
+    CliError::InvalidInput("`apr test llm`: only `bench` routes to a benchmark mode".to_string())
+}
+
 /// TWO MODES, ONE ENTRYPOINT — the §4.4-conformant one.
 ///
 /// `--band` selects PERF-024's `run_band`, which until PERF-025 was called by
@@ -544,9 +575,7 @@ async fn dispatch_band(command: &LlmSubcommand) -> Result<()> {
         ..
     } = command
     else {
-        return Err(CliError::InvalidInput(
-            "the band mode runs `bench` only".to_string(),
-        ));
+        return Err(not_bench());
     };
     // Unreachable: clap's `requires = "receipt"` enforces it. Stated rather
     // than unwrapped, because a receipt-less band run would measure for
@@ -622,9 +651,7 @@ async fn dispatch_legacy(command: &LlmSubcommand) -> Result<()> {
         ..
     } = command
     else {
-        return Err(CliError::InvalidInput(
-            "the legacy mode runs `bench` only".to_string(),
-        ));
+        return Err(not_bench());
     };
     run_bench(BenchArgs {
         url,