From 5436bf33e99b8dacf2ae3911f9c1554fb04c322c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 03:37:08 +0200 Subject: [PATCH 01/35] feat(serve): top-k logprobs with the raw logit, for the V3 top-2 margin (PMAT-4971) V3-a step 1. `top_k_logprobs(logits, k)` returns the k best tokens with their logit and logprob, best first, ties broken by the lower id. The top-1 id is the argmax the engine already picks, and the margin V3 reads is top[0].logit - top[1].logit. NaN is never ranked; a 1e30 logit stays finite. Only `gguf/logprobs.rs` comes over; #4026's CLI and session hunks do not. Nothing calls it yet: the request fields, the engine hook and the wire shape are the next commits on this branch. (cherry picked from eaab36be48, #4026) Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 63282defd115e683b5e0d8a6d9e1fd526505a98e) (cherry picked from commit 3c7e778c9574a61c9e798bb9034b712d28f152d9) (cherry picked from commit f4e92784b357ee863cc1203f3a262a8d9a3ad822) --- crates/aprender-serve/src/gguf/logprobs.rs | 126 +++++++++++++++++++++ docs/roadmaps/entries/PMAT-4971.yaml | 18 +++ docs/roadmaps/roadmap.yaml | 18 +++ 3 files changed, 162 insertions(+) create mode 100644 docs/roadmaps/entries/PMAT-4971.yaml diff --git a/crates/aprender-serve/src/gguf/logprobs.rs b/crates/aprender-serve/src/gguf/logprobs.rs index 1359b34a2f..6acd47638d 100644 --- a/crates/aprender-serve/src/gguf/logprobs.rs +++ b/crates/aprender-serve/src/gguf/logprobs.rs @@ -34,3 +34,129 @@ pub fn logprob_of(logits: &[f32], token_id: u32) -> f32 { .ln(); logits[token_id as usize] - max_logit - log_sum_exp } + +/// One of the most likely next tokens at a step (#4026). +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct TopLogprob { + /// Token ID + pub token_id: u32, + /// The raw logit the forward produced for it + pub logit: f32, + /// ln(softmax(logits)[token_id]) + pub logprob: f32, +} + +/// The distribution one generated step was chosen from (#4026): the `K` most +/// likely tokens, before any repetition penalty or sampling, and the token the +/// engine then chose. `step` 0 is the token after the prompt. +#[derive(Debug, Clone, PartialEq, serde::Serialize)] +pub struct StepLogprobs { + /// 0-based index of the generated token this step produced + pub step: usize, + /// The token the engine chose at this step + pub chosen: u32, + /// The `K` most likely tokens, most likely first + pub top: Vec, +} + +/// The `k` most likely tokens in `logits`, most likely first (#4026). +/// +/// Ties go to the lower token id, as [`crate::gguf::ops::argmax`] breaks them, +/// so with no penalty the first entry IS the greedy choice. NaN logits are +/// never ranked. `logprob` uses the same log-sum-exp as [`logprob_of`]. +#[must_use] +pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { + if k == 0 { + return Vec::new(); + } + let max_logit = logits + .iter() + .copied() + .filter(|x| !x.is_nan()) + .fold(f32::NEG_INFINITY, f32::max); + let log_sum_exp: f32 = logits + .iter() + .filter(|x| !x.is_nan()) + .map(|&x| (x - max_logit).exp()) + .sum::() + .ln(); + let mut ranked: Vec<(u32, f32)> = logits + .iter() + .enumerate() + .filter(|(_, x)| !x.is_nan()) + .map(|(i, &x)| (i as u32, x)) + .collect(); + let k = k.min(ranked.len()); + let by_rank = |a: &(u32, f32), b: &(u32, f32)| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)); + if k < ranked.len() { + ranked.select_nth_unstable_by(k, by_rank); + ranked.truncate(k); + } + ranked.sort_by(by_rank); + ranked + .into_iter() + .map(|(token_id, logit)| TopLogprob { + token_id, + logit, + logprob: logit - max_logit - log_sum_exp, + }) + .collect() +} + +#[cfg(test)] +mod tests_4026 { + use super::*; + + fn ids(top: &[TopLogprob]) -> Vec { + top.iter().map(|t| t.token_id).collect() + } + + #[test] + fn top_k_logprobs_case_table() { + let logits = [0.5, 2.0, -1.0, 2.0, 1.0]; + // (k, expected ids): ties go to the lower id, as ops::argmax. + let table: &[(usize, &[u32])] = &[ + (0, &[]), + (1, &[1]), + (2, &[1, 3]), + (3, &[1, 3, 4]), + (5, &[1, 3, 4, 0, 2]), + (99, &[1, 3, 4, 0, 2]), + ]; + for &(k, want) in table { + assert_eq!(ids(&top_k_logprobs(&logits, k)), want, "k={k}"); + } + assert_eq!( + top_k_logprobs(&logits, 1)[0].token_id, + crate::gguf::ops::argmax(&logits), + "top-1 is the greedy choice" + ); + } + + #[test] + fn top_k_logprobs_agree_with_logprob_of_and_sum_to_one() { + let logits = [0.5_f32, 2.0, -1.0, 2.0, 1.0]; + let all = top_k_logprobs(&logits, logits.len()); + for t in &all { + assert!((t.logprob - logprob_of(&logits, t.token_id)).abs() < 1e-6); + assert_eq!(t.logit, logits[t.token_id as usize]); + } + let total: f32 = all.iter().map(|t| t.logprob.exp()).sum(); + assert!((total - 1.0).abs() < 1e-5, "sum={total}"); + } + + #[test] + fn top_k_logprobs_never_rank_nan() { + let logits = [f32::NAN, 1.0, f32::NAN, 0.0]; + let top = top_k_logprobs(&logits, 4); + assert_eq!(ids(&top), vec![1, 3]); + assert!(top.iter().all(|t| t.logprob.is_finite())); + } + + #[test] + fn top_k_logprobs_survive_huge_logits() { + let top = top_k_logprobs(&[1.0e30, 0.0], 2); + assert_eq!(top[0].logprob, 0.0); + assert!(top[1].logprob.is_finite() && top[1].logprob <= -1.0e29); + } +} diff --git a/docs/roadmaps/entries/PMAT-4971.yaml b/docs/roadmaps/entries/PMAT-4971.yaml new file mode 100644 index 0000000000..262c2e1a66 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4971.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4971 + github_issue: 4971 + item_type: task + title: 'V3 serving-shape parity: lambda perf041 witness for Qwen3.5-4B at c in {1,4,8,16}' + status: planned + priority: high + assigned_to: null + created: 2026-10-10T01:35:02Z + updated: 2026-10-10T01:35:02Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3597 diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 0bd9fdbcec..cddc83e834 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -24681,6 +24681,24 @@ Not done, pending the ruling: moving the 31 fixture files to an obviously synthe - kind:code notes: null epic: 3998 +- id: PMAT-4971 + github_issue: 4971 + item_type: task + title: 'V3 serving-shape parity: lambda perf041 witness for Qwen3.5-4B at c in {1,4,8,16}' + status: planned + priority: high + assigned_to: null + created: 2026-10-10T01:35:02Z + updated: 2026-10-10T01:35:02Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null + epic: 3597 - id: PMAT-5053 github_issue: 5053 item_type: task From d65af5660e81e4106677b826a26aa94b6763cb13 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 04:01:36 +0200 Subject: [PATCH 02/35] feat(serve): Session records each step's logprobs; a greedy step keeps the device argmax (PMAT-4971) Session::generate_with_logprobs records, per generated token, the chosen token's logprob and the top-N tokens with their raw logits, from the logits the choice read (after the repetition penalty, before temperature). generate delegates to it with no recording, so its behaviour is unchanged. A device-greedy step keeps its device argmax. forward_greedy takes read_logits; when it is set the CUDA path copies the logits that argmax read to the host after it, and the token never comes from the host: - decode: forward_gpu_resident_to_token_id_reading (new gguf/cuda/token_logits.rs) takes the same one of the three decode paths with the same argmax; the replay path copies the workspace logits buffer (download_workspace_logits), the CPU-argmax and eager/capture paths hand over the logits they downloaded. - prefill: FirstToken::TokenAndLogits runs the PMAT-083 extraction steps on one logits buffer (new cuda/executor/layers/first_token_logits.rs) and copies it after the GPU argmax, so token and logits come from the same buffer. A device path that was asked for the logits and returned none is an error, never a silent host recompute. uses.rs and prefill.rs are untouched: both carry older functions over the pre-commit complexity cap, so the new paths live in their own modules. Contract apr-serve-openai-compat-v1 1.21.0, ASOC-INV-021. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 64c76496aa903fdae30a2ab40165fe32f58bee68) (cherry picked from commit b6560e7b0ee933d39f441f0f3aa081e5b8ef0dc8) (cherry picked from commit 3a9bcb7c72e83044afd0d9af8e08f7ce9aad902e) --- contracts/apr-serve-openai-compat-v1.yaml | 46 ++++- .../executor/layers/first_token_logits.rs | 130 ++++++++++++++ .../src/cuda/executor/layers/mod.rs | 2 + .../src/cuda/executor/layers/par-062.rs | 17 ++ .../src/gguf/cuda/generate_2.rs | 68 ++++++-- crates/aprender-serve/src/gguf/cuda/mod.rs | 4 + .../src/gguf/cuda/token_logits.rs | 128 ++++++++++++++ .../gguf/inference/forward/dense_session.rs | 54 ++++-- .../forward/dense_session_borrowed.rs | 12 +- crates/aprender-serve/src/gguf/logprobs.rs | 119 +++++++++++-- crates/aprender-serve/src/session.rs | 165 ++++++++++++++---- crates/aprender-serve/src/session_tests.rs | 153 +++++++++++++++- 12 files changed, 809 insertions(+), 89 deletions(-) create mode 100644 crates/aprender-serve/src/cuda/executor/layers/first_token_logits.rs create mode 100644 crates/aprender-serve/src/gguf/cuda/token_logits.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index ecee5da692..712138921c 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -1,7 +1,7 @@ contract: apr-serve-openai-compat metadata: kind: pattern - version: "1.19.0" + version: "1.21.0" description: > OpenAI-compatible serve layer (/v1/chat/completions, /v1/completions, /v1/embeddings) fidelity invariants. Established by an adversarial audit @@ -390,6 +390,32 @@ proof_obligations: AppState::demo() / AppState::new() again. (AppState::with_registry stays permitted: ERROR-MESSAGES-NAME-A-REAL-REMEDY names it deliberately as the embedder API that enables registry mode.) + - id: ASOC-INV-021 + type: invariant + property: > + LOGPROBS-HONOURED-OR-REFUSED (#4971, V3-a of the 0.71 serving-shape + parity work; ASOC-INV-020 is #3568's). /v1/chat/completions takes the + OpenAI `logprobs` (bool) and `top_logprobs` (0..=20). With `logprobs: + true` the reply carries `choices[0].logprobs.content`: one entry per + generated token, in order, so its length equals + `usage.completion_tokens`. Each entry is {token, logprob, bytes, + top_logprobs}. `top_logprobs` holds the N best tokens of that step, best + first, ties to the lower id. Every logprob is <= 0. On a stream the + entries ride on the chunk that releases their text, and anything still + held is released on the finishing chunk, so the entries of all chunks + concatenate to the same list. REQUESTING THEM NEVER CHANGES THE TOKEN. + The engine reads the logits buffer that its own choice read: on a + device-argmax backend the device still picks the token and the buffer + is copied out after it. So at temperature 0 the ids with and without + `logprobs` are identical, and the chosen token is top_logprobs[0]. The + top-2 margin V3 needs is top_logprobs[0].logprob - + top_logprobs[1].logprob, which equals the logit margin because both + share one log-sum-exp. REFUSED, never dropped: `top_logprobs` above 20, + or `top_logprobs` without `logprobs: true`, is a client error that + names the field. A backend that cannot compute them answers 501 and + names itself; it never answers 200 without them. NO-REGRESSION: a + request without `logprobs` is served as before, and its reply has no + `logprobs` key. falsification_tests: - id: FALSIFY-STOP-TRUNCATE-754 @@ -620,3 +646,21 @@ falsification_tests: test_harness: 'cargo test -p aprender-serve --lib -- chat_stream_route_2375::stream_route_with_no_model_is_503_not_404 chat_stream_route_2375::stream_route_with_an_unknown_model_name_is_still_404' expected_output: "test result: ok" if_fails: 'A client retry policy keyed on status treats "the server has not loaded a model yet" as permanent (404 = this route does not exist) and stops retrying, on the two most-used routes of the OpenAI surface. Mutation-verified in both directions: forcing NOT_FOUND reds the 503 half, forcing SERVICE_UNAVAILABLE reds the 404 half.' + - id: FALSIFY-LOGPROBS-TOPK-4971 + name: top_k_logprobs_rank_like_the_argmax + prediction: 'top_k_logprobs(logits, k) returns min(k, vocab) entries, best first, ties to the lower id, so entry 0 is the token gguf::ops::argmax picks. Case table: logits [0.5, 2.0, -1.0, 2.0, 1.0] give ids [] / [1] / [1,3] / [1,3,4] / [1,3,4,0,2] for k 0/1/2/3/5, and k 99 gives all five. Each logprob equals logprob_of for its id, and the exp of all five sums to 1. NaN is never ranked, and a 1e30 logit stays finite. StepLogprobs::of records the chosen token''s own logprob even when it is not in the top N.' + test_harness: 'cargo test -p aprender-serve --lib gguf::logprobs::' + expected_output: "test result: ok" + if_fails: 'top_logprobs[0] is not the token the greedy engine chose, so the V3 margin (top[0] - top[1]) is measured between the wrong two tokens.' + - id: FALSIFY-LOGPROBS-SAME-TOKEN-4971 + name: requesting_logprobs_never_changes_the_token + prediction: 'A Session over a backend with a device-argmax path generates the same ids with and without logprobs. With them it takes the device path every step (forward_greedy with read_logits), never the host forward, and the id it returns is the device''s, even when a planted host argmax would pick another token. Each step''s StepLogprobs has top[0] == chosen. A planted near-tie (two logits 0.01 apart) reads as a margin below 0.05, and a clear winner reads as a margin above 1. Without logprobs the device path is called with read_logits false, so no logits are copied.' + test_harness: 'cargo test -p aprender-serve --lib session::tests::logprobs_4971' + expected_output: "test result: ok" + if_fails: 'Asking for the margin changes the kernels that choose the token. The witness then measures a different run from the one it claims, which is the defect V3 exists to rule out.' + - id: FALSIFY-LOGPROBS-WIRE-4971 + name: chat_logprobs_are_served_or_refused_by_name + prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry token equal to top_logprobs[0].token. Streamed, the entries of all chunks concatenate to completion_tokens entries. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. A backend with no logprobs path (the registry fallback) answers 501 and names itself.' + test_harness: 'cargo test -p aprender-serve --lib api::tests::logprobs_4971' + expected_output: "test result: ok" + if_fails: 'A client that asks for logprobs gets a 200 without them (serde ignores unknown keys, which was main''s behaviour before #4971). The perf041 witness then records no margin and S5 can never pass.' diff --git a/crates/aprender-serve/src/cuda/executor/layers/first_token_logits.rs b/crates/aprender-serve/src/cuda/executor/layers/first_token_logits.rs new file mode 100644 index 0000000000..473f7ad5b6 --- /dev/null +++ b/crates/aprender-serve/src/cuda/executor/layers/first_token_logits.rs @@ -0,0 +1,130 @@ +//! #4971: the prefill's first token with the logits its argmax read. +//! +//! `prefill_extract_first_token` (PMAT-083, `prefill.rs`) chooses the first +//! token on the device and drops the logits. A request for logprobs needs those +//! logits, so this module runs the same steps (output RMSNorm, LM head GEMV, +//! LM head bias, GPU argmax) on one logits buffer and copies that buffer to the +//! host after the argmax. The token and the logits come from the same buffer, +//! so the reported distribution is the one the choice was made from. + +#![allow(clippy::wildcard_imports)] + +use super::super::*; + +impl CudaExecutor { + /// [`Self::prefill_extract_first_token`], with the logits the argmax read + /// copied into `out` (resized to `vocab_size`) after the argmax (#4971). + /// + /// Same preconditions: call it after `prefill_eager` and before + /// `force_workspace_reinit`, while `hidden_buf2` holds the prefill output. + pub(crate) fn prefill_extract_first_token_and_logits( + &mut self, + last_pos: usize, + hidden_dim: u32, + vocab_size: u32, + epsilon: f32, + out: &mut Vec, + ) -> Result { + let logits_buf = self.prefill_last_row_logits(last_pos, hidden_dim, vocab_size, epsilon)?; + let token = self.gpu_argmax(logits_buf.as_ptr(), vocab_size)?; + out.clear(); + out.resize(vocab_size as usize, 0.0); + self.stream.synchronize()?; + logits_buf.copy_to_host(out)?; + Ok(token) + } + + /// Steps 1-4 of [`Self::prefill_extract_first_token`]: the last prompt + /// row's logits, left on the device. + fn prefill_last_row_logits( + &mut self, + last_pos: usize, + hidden_dim: u32, + vocab_size: u32, + epsilon: f32, + ) -> Result, GpuError> { + let normed_buf = self.prefill_last_row_normed(last_pos, hidden_dim, epsilon)?; + let lm_head_ptr = self.lm_head_ptr; + // #3908: a consistent declaration wins over the size guess (BF16 == F16 in size). + let lm_head_qtype = WeightQuantType::resolve_declared_or_sized( + Some(self.lm_head_qtype), + self.lm_head_len, + vocab_size as usize, + hidden_dim as usize, + ) + .unwrap_or(self.lm_head_qtype); + if lm_head_ptr == 0 { + return Err(GpuError::InvalidLaunchConfig( + "#4971: lm_head not loaded".to_string(), + )); + } + + let logits_buf = GpuBuffer::::new(&self.context, vocab_size as usize)?; + self.q8_activation_valid = false; // LM head input differs from layer GEMVs + self.gemv_dispatch( + lm_head_qtype, + lm_head_ptr, + &normed_buf, + &logits_buf, + vocab_size, + hidden_dim, + )?; + + if self.lm_head_bias_ptr != 0 && self.lm_head_bias_len > 0 { + // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). + let bias_buf = unsafe { + GpuBuffer::::from_raw_parts(self.lm_head_bias_ptr, self.lm_head_bias_len) + }; + self.residual_add_into(&logits_buf, &bias_buf, &logits_buf, vocab_size)?; + std::mem::forget(bias_buf); + } + Ok(logits_buf) + } + + /// The last prompt row of `hidden_buf2`, through the output RMSNorm. + fn prefill_last_row_normed( + &mut self, + last_pos: usize, + hidden_dim: u32, + epsilon: f32, + ) -> Result, GpuError> { + let hidden_buf2_ptr = self + .workspace + .hidden_buf2 + .as_ref() + .ok_or_else(|| { + GpuError::InvalidLaunchConfig( + "#4971: hidden_buf2 missing for first token extraction".to_string(), + ) + })? + .as_ptr(); + let offset_bytes = last_pos as u64 * hidden_dim as u64 * 4; // f32 = 4 bytes + let last_hidden_ptr = hidden_buf2_ptr + offset_bytes; + + let output_norm_ptr = self.output_norm_ptr; + let output_norm_len = self.output_norm_len; + if output_norm_ptr == 0 { + return Err(GpuError::InvalidLaunchConfig( + "#4971: output_norm not loaded".to_string(), + )); + } + + // Non-owning view of the last position's hidden state (1 × hidden_dim). + let last_hidden = + // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). + unsafe { GpuBuffer::::from_raw_parts(last_hidden_ptr, hidden_dim as usize) }; + let normed = GpuBuffer::::new(&self.context, hidden_dim as usize).and_then(|normed| { + self.rmsnorm_ptr_into( + &last_hidden, + output_norm_ptr, + output_norm_len, + &normed, + hidden_dim, + epsilon, + ) + .map(|()| normed) + }); + std::mem::forget(last_hidden); + normed + } +} diff --git a/crates/aprender-serve/src/cuda/executor/layers/mod.rs b/crates/aprender-serve/src/cuda/executor/layers/mod.rs index 9db138d6bf..4a4bb8f458 100644 --- a/crates/aprender-serve/src/cuda/executor/layers/mod.rs +++ b/crates/aprender-serve/src/cuda/executor/layers/mod.rs @@ -6,6 +6,8 @@ mod batched; mod cublas_prefill; mod ffn; +/// #4971: the prefill's first token with the logits its argmax read. +mod first_token_logits; mod forward; mod graph_decode; mod graphed; diff --git a/crates/aprender-serve/src/cuda/executor/layers/par-062.rs b/crates/aprender-serve/src/cuda/executor/layers/par-062.rs index bfd23e3ef9..49fc360609 100644 --- a/crates/aprender-serve/src/cuda/executor/layers/par-062.rs +++ b/crates/aprender-serve/src/cuda/executor/layers/par-062.rs @@ -550,4 +550,21 @@ impl CudaExecutor { Ok(gpu_result) } + + /// #4971: copy the first `out.len()` logits the last forward left in the + /// workspace to the host. Called after the device argmax has read them, + /// so asking for the logits never changes the token or the kernels that + /// chose it. The batched forward leaves `m × vocab` there, row by row. + /// + /// # Errors + /// No logits buffer was allocated, or `out` is longer than it. + pub fn download_workspace_logits(&self, out: &mut [f32]) -> Result<(), GpuError> { + let buf = self + .workspace + .logits_buf + .as_ref() + .ok_or_else(|| GpuError::InvalidParameter("logits_buf not allocated".into()))?; + self.stream.synchronize()?; + buf.copy_to_host(out) + } } diff --git a/crates/aprender-serve/src/gguf/cuda/generate_2.rs b/crates/aprender-serve/src/gguf/cuda/generate_2.rs index 196902b5be..d6346607bf 100644 --- a/crates/aprender-serve/src/gguf/cuda/generate_2.rs +++ b/crates/aprender-serve/src/gguf/cuda/generate_2.rs @@ -247,6 +247,27 @@ impl OwnedQuantizedModelCuda { prefill_count: usize, trace: bool, extract_first_token: bool, + ) -> Result> { + let first = if extract_first_token { + FirstToken::Token + } else { + FirstToken::Skip + }; + self.run_prefill_extracting(prompt, cache, prefill_count, trace, first) + } + + /// [`Self::run_prefill`] with the first-token extraction spelled out: with + /// [`FirstToken::TokenAndLogits`] (#4971) the logits the batched prefill's + /// device argmax read are copied into the vector after it. A serial + /// prefill extracts nothing, so it returns `None` and leaves the vector + /// alone. + pub(crate) fn run_prefill_extracting( + &mut self, + prompt: &[u32], + cache: &mut OwnedQuantizedKVCache, + prefill_count: usize, + trace: bool, + first: FirstToken<'_>, ) -> Result> { if prefill_count == 0 { // No prefill PHASE ran. `None`, not `Some(0.0)`: a zero would be @@ -362,23 +383,26 @@ impl OwnedQuantizedModelCuda { // Runs output RMSNorm + LM head GEMV + GPU argmax on the last position. // This eliminates the separate first decode step (~7ms savings). // Must happen BEFORE force_workspace_reinit (hidden_buf2 still valid). - let first_token = if extract_first_token { - let token = self - .executor - .prefill_extract_first_token( - last_row, // the last position's row in the final chunk - hidden_dim as u32, - vocab_size as u32, - eps, - ) - .map_err(|e| RealizarError::UnsupportedOperation { - operation: "prefill_extract_first_token".to_string(), - reason: format!("PMAT-083 first token extraction failed: {e}"), - })?; - Some(token) - } else { - None - }; + // `last_row` is the last position's row in the final chunk. #4971: with + // `TokenAndLogits` the same steps run on one logits buffer, copied to + // the host after the argmax. + let (hidden, vocab) = (hidden_dim as u32, vocab_size as u32); + let first_token = match first { + FirstToken::Skip => None, + FirstToken::Token => Some( + self.executor + .prefill_extract_first_token(last_row, hidden, vocab, eps), + ), + FirstToken::TokenAndLogits(out) => Some( + self.executor + .prefill_extract_first_token_and_logits(last_row, hidden, vocab, eps, out), + ), + } + .transpose() + .map_err(|e| RealizarError::UnsupportedOperation { + operation: "prefill_extract_first_token".to_string(), + reason: format!("PMAT-083 first token extraction failed: {e}"), + })?; // CORRECTNESS-016: Log KV cache fingerprint after batched prefill. // Non-destructive: just reads the KV cache, no serial comparison. @@ -1119,6 +1143,16 @@ impl OwnedQuantizedModelCuda { } } +/// What a prefill hands back besides the KV cache (#4971). +pub(crate) enum FirstToken<'a> { + /// Nothing: the caller runs the first decode itself. + Skip, + /// The batched prefill's device argmax of the last row. + Token, + /// That argmax and, copied after it, the logits it read. + TokenAndLogits(&'a mut Vec), +} + /// #4590: warn, once per process, when this model on this GPU is a 0.70 known /// issue (Qwen3 on sm_12x: correct but very slow). Unconditional like the /// prefill line, so a user sees why the run is slow before waiting on it. diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index add810fbf5..a39b316119 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -41,6 +41,8 @@ mod forward_qwen35_cuda; mod forward_qwen3_moe_resident; mod generation; mod speculative; +/// #4971: a greedy decode step that also hands back the logits its argmax read. +mod token_logits; mod weights; // Re-export types for public API @@ -54,6 +56,8 @@ pub use forward_qwen35_cuda::{ pub use forward_qwen3_moe_resident::{Qwen3MoeCudaModel, Qwen3MoeCudaState, Qwen3MoeShape}; // PMAT-072: Step-wise batched decode state for lock-releasing scheduler pub use generation::BatchedDecodeState; +// #4971: what a prefill extracts for the session's first token. +pub(crate) use generation::FirstToken; use crate::error::{RealizarError, Result}; diff --git a/crates/aprender-serve/src/gguf/cuda/token_logits.rs b/crates/aprender-serve/src/gguf/cuda/token_logits.rs new file mode 100644 index 0000000000..307cfa6776 --- /dev/null +++ b/crates/aprender-serve/src/gguf/cuda/token_logits.rs @@ -0,0 +1,128 @@ +//! #4971: a greedy decode step that also hands back the logits its argmax read. +//! +//! [`OwnedQuantizedModelCuda::forward_gpu_resident_to_token_id`] chooses on one +//! of three paths. The CPU-argmax path (LM head bias or `CORRECTNESS_MODE=1`) +//! and the eager/graph-capture path both download the logits and take a host +//! argmax; the graph-replay path takes a GPU argmax over the workspace logits +//! buffer. Asking for the logits takes the same path, with the same kernels and +//! the same argmax, and copies the logits the argmax read to the host after it. + +use super::{OwnedQuantizedKVCache, OwnedQuantizedModelCuda, RealizarError, Result}; + +impl OwnedQuantizedModelCuda { + /// [`Self::forward_gpu_resident_to_token_id`], and with `logits_out` + /// (#4971) the logits the argmax read, copied to the host after it. + /// + /// # Errors + /// + /// As [`Self::forward_gpu_resident_to_token_id`], and a failed logits copy. + pub fn forward_gpu_resident_to_token_id_reading( + &mut self, + token_id: u32, + cache: &mut OwnedQuantizedKVCache, + position: usize, + logits_out: Option<&mut Vec>, + ) -> Result { + let Some(out) = logits_out else { + return self.forward_gpu_resident_to_token_id(token_id, cache, position); + }; + if self.model.lm_head_bias.is_some() || correctness_mode() { + *out = self.forward_gpu_resident(token_id, cache, position)?; + } else if self.executor.is_profiling_enabled() || !self.executor.has_decode_graph() { + self.eager_step_logits(token_id, cache, position, out)?; + } else { + let token = self.forward_gpu_resident_to_token_id(token_id, cache, position)?; + out.clear(); + out.resize(self.model.lm_head_weight.out_dim, 0.0); + self.executor.download_workspace_logits(out).map_err(|e| { + step_failed(format!("the logits copy after the device argmax: {e}")) + })?; + return Ok(token); + } + Ok(host_argmax(out)) + } + + /// The eager/graph-capture step of `forward_gpu_resident_to_token_id`, + /// leaving its logits in `out`. + fn eager_step_logits( + &mut self, + token_id: u32, + cache: &mut OwnedQuantizedKVCache, + position: usize, + out: &mut Vec, + ) -> Result<()> { + let hidden_dim = self.model.config.hidden_dim; + let intermediate_dim = self.model.layers[0].ffn_up_weight.out_dim; + let num_layers = self.model.layers.len(); + let vocab_size = self.model.lm_head_weight.out_dim; + let eps = self.model.config.eps; + + self.model.embed_into(token_id, &mut self.embed_buf); + out.clear(); + out.resize(vocab_size, 0.0); + self.executor + .forward_all_layers_gpu_to_logits_graphed( + &self.embed_buf, + out, + position as u32, + num_layers, + hidden_dim as u32, + intermediate_dim as u32, + vocab_size as u32, + eps, + ) + .map_err(|e| { + step_failed(format!( + "forward_all_layers_gpu_to_logits_graphed failed: {e}" + )) + })?; + cache.advance(); + Ok(()) + } +} + +/// `CORRECTNESS_MODE=1`, read as `forward_gpu_resident_to_token_id` reads it. +fn correctness_mode() -> bool { + static CORRECTNESS_MODE: std::sync::OnceLock = std::sync::OnceLock::new(); + *CORRECTNESS_MODE.get_or_init(|| { + std::env::var("CORRECTNESS_MODE") + .map(|v| v == "1") + .unwrap_or(false) + }) +} + +/// The host argmax of `forward_gpu_resident_to_token_id` (ties go to the last +/// maximum, as they do there). +fn host_argmax(logits: &[f32]) -> u32 { + logits + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .map_or(0, |(idx, _)| idx as u32) +} + +fn step_failed(reason: String) -> RealizarError { + RealizarError::UnsupportedOperation { + operation: "forward_gpu_resident_to_token_id_reading".to_string(), + reason, + } +} + +#[cfg(test)] +mod tests_4971 { + use super::host_argmax; + + #[test] + fn host_argmax_breaks_ties_to_the_last_maximum_as_the_decode_step_does() { + // (logits, expected): the decode step's host argmax keeps the LAST max. + let table: &[(&[f32], u32)] = &[ + (&[0.0, 3.0, 1.0], 1), + (&[2.0, 2.0, 1.0], 1), + (&[1.0, 5.0, 5.0, 5.0], 3), + (&[], 0), + ]; + for &(logits, want) in table { + assert_eq!(host_argmax(logits), want, "{logits:?}"); + } + } +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs index 3d8790152d..10a2138ecb 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use crate::error::{RealizarError, Result}; use crate::gguf::{OwnedQuantizedKVCache, OwnedQuantizedModel}; +use crate::session::GreedyStep; /// The prefix of every line that reports a dense CUDA forward giving up for the /// CPU. A log scan keys on it. @@ -301,7 +302,8 @@ impl DenseForward { &mut self, tokens: &[u32], start: usize, - ) -> std::result::Result, Step> { + read_logits: bool, + ) -> std::result::Result, Step> { #[cfg(feature = "cuda")] if matches!(self.backend, Backend::Cuda { .. }) { let start = self.resume_at(start)?; @@ -309,16 +311,22 @@ impl DenseForward { unreachable!("matched above"); }; let cache = cache.as_mut().ok_or_else(|| never_reserved("CUDA"))?; - let Some(next) = - cuda_forward_greedy(model, cache, tokens, start, &mut self.batched_prefills) - .map_err(Step::Gpu)? + let Some(next) = cuda_forward_greedy( + model, + cache, + tokens, + start, + &mut self.batched_prefills, + read_logits, + ) + .map_err(Step::Gpu)? else { return Ok(None); }; self.held = tokens.len(); return Ok(Some(next)); } - let _ = (tokens, start); + let _ = (tokens, start, read_logits); Ok(None) } } @@ -380,10 +388,10 @@ pub(crate) fn cuda_forward( } /// [`cuda_forward`] for a greedy step: the argmax is chosen on the device, so -/// no logits cross to the host. A whole prompt goes through one batched -/// prefill that extracts the first token (PMAT-083). `None`: the prefill ran -/// but chose nothing, and the caller must [`cuda_reset`] and take the logits -/// path. +/// no logits cross to the host unless `read_logits` asks for the ones it read +/// (#4971), copied after it. A whole prompt goes through one batched prefill +/// that extracts the first token (PMAT-083). `None`: the prefill ran but chose +/// nothing, and the caller must [`cuda_reset`] and take the logits path. /// /// # Errors /// As [`cuda_forward`]. @@ -394,14 +402,23 @@ pub(crate) fn cuda_forward_greedy( tokens: &[u32], start: usize, batched_prefills: &mut usize, -) -> std::result::Result, String> { + read_logits: bool, +) -> std::result::Result, String> { + use crate::gguf::cuda::FirstToken; + // #4971: the device argmax chooses either way; the logits it read are + // copied after it only when asked for. + let mut logits = read_logits.then(Vec::new); let last = tokens.len() - 1; if start == 0 && last > 0 { + let extract = match logits.as_mut() { + Some(out) => FirstToken::TokenAndLogits(out), + None => FirstToken::Token, + }; let first = model - .run_prefill(tokens, cache, tokens.len(), false, true) + .run_prefill_extracting(tokens, cache, tokens.len(), false, extract) .map_err(|e| gpu_failed("batched prefill", tokens.len(), e))?; *batched_prefills += 1; - return Ok(first); + return Ok(first.map(|token| GreedyStep { token, logits })); } for (pos, &token) in tokens.iter().enumerate().take(last).skip(start) { model @@ -409,8 +426,8 @@ pub(crate) fn cuda_forward_greedy( .map_err(|e| gpu_failed("forward", pos, e))?; } model - .forward_gpu_resident_to_token_id(tokens[last], cache, last) - .map(Some) + .forward_gpu_resident_to_token_id_reading(tokens[last], cache, last, logits.as_mut()) + .map(|token| Some(GreedyStep { token, logits })) .map_err(|e| gpu_failed("forward", last, e)) } @@ -470,8 +487,13 @@ impl crate::session::ArchForward for DenseForward { } } - fn forward_greedy(&mut self, tokens: &[u32], start: usize) -> Result> { - match self.try_forward_greedy(tokens, start) { + fn forward_greedy( + &mut self, + tokens: &[u32], + start: usize, + read_logits: bool, + ) -> Result> { + match self.try_forward_greedy(tokens, start, read_logits) { Ok(next) => Ok(next), Err(step) => { self.held = 0; diff --git a/crates/aprender-serve/src/gguf/inference/forward/dense_session_borrowed.rs b/crates/aprender-serve/src/gguf/inference/forward/dense_session_borrowed.rs index 28389c8971..6767458faa 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/dense_session_borrowed.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/dense_session_borrowed.rs @@ -16,6 +16,7 @@ use crate::error::{RealizarError, Result}; use crate::gguf::{OwnedQuantizedKVCache, OwnedQuantizedModelCuda}; use crate::gguf::dense_session::{cuda_forward, cuda_forward_greedy, cuda_reset}; +use crate::session::GreedyStep; /// A dense CUDA forward over a borrowed model, for one scheduler turn. pub struct BorrowedCudaForward<'a> { @@ -156,8 +157,15 @@ impl crate::session::ArchForward for BorrowedCudaForward<'_> { Ok(logits) } - fn forward_greedy(&mut self, tokens: &[u32], start: usize) -> Result> { - let next = self.run(tokens, start, cuda_forward_greedy)?; + fn forward_greedy( + &mut self, + tokens: &[u32], + start: usize, + read_logits: bool, + ) -> Result> { + let next = self.run(tokens, start, |model, cache, tokens, start, prefills| { + cuda_forward_greedy(model, cache, tokens, start, prefills, read_logits) + })?; // `None`: the prefill ran and chose nothing; the session replays // through `forward`, from 0. if next.is_some() { diff --git a/crates/aprender-serve/src/gguf/logprobs.rs b/crates/aprender-serve/src/gguf/logprobs.rs index 6acd47638d..90993b5937 100644 --- a/crates/aprender-serve/src/gguf/logprobs.rs +++ b/crates/aprender-serve/src/gguf/logprobs.rs @@ -47,28 +47,53 @@ pub struct TopLogprob { } /// The distribution one generated step was chosen from (#4026): the `K` most -/// likely tokens, before any repetition penalty or sampling, and the token the -/// engine then chose. `step` 0 is the token after the prompt. +/// likely tokens, from the logits the choice read (after any repetition +/// penalty, before temperature; #4971), and the token the engine then chose. +/// `step` 0 is the token after the prompt. #[derive(Debug, Clone, PartialEq, serde::Serialize)] pub struct StepLogprobs { /// 0-based index of the generated token this step produced pub step: usize, /// The token the engine chose at this step pub chosen: u32, + /// ln(softmax(logits)[chosen]), recorded whether or not `chosen` is in `top` + pub chosen_logprob: f32, /// The `K` most likely tokens, most likely first pub top: Vec, } -/// The `k` most likely tokens in `logits`, most likely first (#4026). -/// -/// Ties go to the lower token id, as [`crate::gguf::ops::argmax`] breaks them, -/// so with no penalty the first entry IS the greedy choice. NaN logits are -/// never ranked. `logprob` uses the same log-sum-exp as [`logprob_of`]. -#[must_use] -pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { - if k == 0 { - return Vec::new(); +impl StepLogprobs { + /// The record of one step (#4971): the chosen token's own logprob and the + /// `n` best tokens of `logits`. The chosen token need not be among them (a + /// sampled step, or `n` 0), so its logprob is computed on its own. + #[must_use] + pub fn of(step: usize, chosen: u32, logits: &[f32], n: usize) -> Self { + let (max_logit, log_sum_exp) = max_and_log_sum_exp(logits); + let chosen_logprob = logits + .get(chosen as usize) + .map_or(f32::NEG_INFINITY, |&x| x - max_logit - log_sum_exp); + Self { + step, + chosen, + chosen_logprob, + top: top_k_logprobs(logits, n), + } } + + /// `top[0].logit - top[1].logit`: how close the step came to choosing + /// another token (#4971, the V3 near-tie measure). `None` below two entries. + #[must_use] + pub fn top2_margin(&self) -> Option { + match self.top.as_slice() { + [first, second, ..] => Some(first.logit - second.logit), + _ => None, + } + } +} + +/// The largest non-NaN logit and the log-sum-exp of the rest relative to it, +/// so `logit - max - lse` is a logprob. NaN logits take no part. +fn max_and_log_sum_exp(logits: &[f32]) -> (f32, f32) { let max_logit = logits .iter() .copied() @@ -80,6 +105,20 @@ pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { .map(|&x| (x - max_logit).exp()) .sum::() .ln(); + (max_logit, log_sum_exp) +} + +/// The `k` most likely tokens in `logits`, most likely first (#4026). +/// +/// Ties go to the lower token id, as [`crate::gguf::ops::argmax`] breaks them, +/// so with no penalty the first entry IS the greedy choice. NaN logits are +/// never ranked. `logprob` uses the same log-sum-exp as [`logprob_of`]. +#[must_use] +pub fn top_k_logprobs(logits: &[f32], k: usize) -> Vec { + if k == 0 { + return Vec::new(); + } + let (max_logit, log_sum_exp) = max_and_log_sum_exp(logits); let mut ranked: Vec<(u32, f32)> = logits .iter() .enumerate() @@ -160,3 +199,61 @@ mod tests_4026 { assert!(top[1].logprob.is_finite() && top[1].logprob <= -1.0e29); } } + +#[cfg(test)] +mod tests_4971 { + use super::*; + + #[test] + fn step_records_the_chosen_logprob_even_outside_the_top_n() { + let logits = [0.5_f32, 2.0, -1.0, 2.0, 1.0]; + // Token 2 is the least likely: a sampled step can still choose it. + let s = StepLogprobs::of(7, 2, &logits, 2); + assert_eq!(s.step, 7); + assert_eq!(s.chosen, 2); + assert_eq!(s.top.iter().map(|t| t.token_id).collect::>(), [1, 3]); + assert!((s.chosen_logprob - logprob_of(&logits, 2)).abs() < 1e-6); + // With n 0 there is no top, and the chosen logprob is still there. + let s0 = StepLogprobs::of(0, 1, &logits, 0); + assert!(s0.top.is_empty()); + assert!((s0.chosen_logprob - logprob_of(&logits, 1)).abs() < 1e-6); + } + + #[test] + fn chosen_logprob_matches_its_top_entry_and_ignores_nan() { + let logits = [f32::NAN, 1.0, 3.0, 0.0]; + let s = StepLogprobs::of(0, 2, &logits, 3); + assert_eq!(s.top[0].token_id, 2); + assert_eq!(s.chosen_logprob, s.top[0].logprob); + assert!(s.chosen_logprob.is_finite()); + // An id past the vocabulary has no probability, never a panic. + assert_eq!( + StepLogprobs::of(0, 99, &logits, 1).chosen_logprob, + f32::NEG_INFINITY + ); + } + + #[test] + fn top2_margin_case_table() { + // (logits, n, expected margin): the V3 near-tie measure is top[0] - top[1]. + let table: &[(&[f32], usize, Option)] = &[ + (&[1.0, 1.01, -3.0], 2, Some(0.01)), + (&[5.0, 1.0, 0.0], 2, Some(4.0)), + (&[2.0, 2.0], 2, Some(0.0)), + (&[5.0, 1.0, 0.0], 1, None), + (&[5.0, 1.0, 0.0], 0, None), + (&[5.0], 2, None), + ]; + for &(logits, n, want) in table { + let got = StepLogprobs::of(0, 0, logits, n).top2_margin(); + match (got, want) { + (Some(g), Some(w)) => assert!((g - w).abs() < 1e-5, "{logits:?} n={n}: {g}"), + (g, w) => assert_eq!(g, w, "{logits:?} n={n}"), + } + } + let near_tie = StepLogprobs::of(0, 1, &[1.0, 1.01, -3.0], 2); + assert!(near_tie.top2_margin().is_some_and(|m| m < 0.05)); + let clear = StepLogprobs::of(0, 0, &[5.0, 1.0, 0.0], 2); + assert!(clear.top2_margin().is_some_and(|m| m > 1.0)); + } +} diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index e52e9ff035..17dcb1d78e 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -39,7 +39,7 @@ use std::sync::Mutex; use std::time::Instant; use crate::error::{RealizarError, Result}; -use crate::gguf::{OwnedQuantizedModel, QuantizedGenerateConfig}; +use crate::gguf::{OwnedQuantizedModel, QuantizedGenerateConfig, StepLogprobs}; /// One architecture's forward on one backend: the only per-arch code a verb /// reaches, and only through a [`Session`]. @@ -122,9 +122,20 @@ pub trait ArchForward { /// the state is as it was, and the session calls /// [`ArchForward::forward`] instead. The default has no such path. /// + /// With `read_logits` (#4971: the caller asked for logprobs) the token is + /// still the device's argmax, and the logits it read are copied to the + /// host after it, in [`GreedyStep::logits`]. Asking for them must never + /// move the step onto another kernel path, because another path can + /// choose another token on a near-tie. + /// /// # Errors /// As [`ArchForward::forward`]. - fn forward_greedy(&mut self, _tokens: &[u32], _start: usize) -> Result> { + fn forward_greedy( + &mut self, + _tokens: &[u32], + _start: usize, + _read_logits: bool, + ) -> Result> { Ok(None) } @@ -164,6 +175,34 @@ pub struct LayerTiming { pub kind: &'static str, } +/// What [`ArchForward::forward_greedy`] chose. +#[derive(Debug, Clone, PartialEq)] +pub struct GreedyStep { + /// The argmax token, chosen on the device. + pub token: u32, + /// The logits that argmax read, copied to the host after it. `Some` + /// exactly when the call asked for them (`read_logits`). + pub logits: Option>, +} + +impl GreedyStep { + /// A step that read back only the token. + #[must_use] + pub fn token(token: u32) -> Self { + Self { + token, + logits: None, + } + } +} + +/// The token [`Session`] chose after a prefix, and the logits the choice was +/// made from when the caller asked for them. +struct Choice { + token: u32, + logits: Option>, +} + /// What one [`Session::generate`] call did. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Turn { @@ -374,47 +413,72 @@ impl Session { } } - /// Make the state hold exactly `tokens` and choose the token after them; - /// return it and how many leading tokens were already held. A greedy - /// choice with no repetition penalty goes through - /// [`ArchForward::forward_greedy`] when the backend has it. + /// Make the state hold exactly `tokens` and choose the token after them. + /// A greedy choice with no repetition penalty goes through + /// [`ArchForward::forward_greedy`] when the backend has it, with or without + /// `want_logits`. Otherwise the choice is made from the logits after the + /// repetition penalty, and those are the logits returned. fn advance_and_choose( &mut self, tokens: &[u32], config: &QuantizedGenerateConfig, rng: &mut rand::rngs::StdRng, - ) -> Result<(u32, usize)> { + want_logits: bool, + ) -> Result { if is_greedy(config) && !penalty_active(config) { - let start = if self.extends(tokens) { - self.processed.len() - } else { - 0 - }; - if self.checkpoint.as_ref().is_some_and(|c| start < c.len()) { - self.checkpoint = None; - } - match self.forward.forward_greedy(tokens, start) { - Ok(Some(next)) => { - self.processed.truncate(start); - self.processed.extend_from_slice(&tokens[start..]); - return Ok((next, start)); - }, - Ok(None) => {}, - Err(e) => { - self.processed.clear(); - self.checkpoint = None; - return Err(e); - }, + if let Some(choice) = self.device_greedy(tokens, want_logits)? { + return Ok(choice); } } - let (mut logits, reused) = self.advance_to(tokens)?; + let (mut logits, _) = self.advance_to(tokens)?; OwnedQuantizedModel::apply_repeat_penalty( &mut logits, tokens, config.repeat_penalty, config.repeat_last_n, ); - Ok((choose_token(&logits, config, rng), reused)) + let token = choose_token(&logits, config, rng); + Ok(Choice { + token, + logits: want_logits.then_some(logits), + }) + } + + /// [`ArchForward::forward_greedy`] over `tokens`, recording what the state + /// then holds. `None` when the backend has no device argmax. + fn device_greedy(&mut self, tokens: &[u32], want_logits: bool) -> Result> { + let start = if self.extends(tokens) { + self.processed.len() + } else { + 0 + }; + if self.checkpoint.as_ref().is_some_and(|c| start < c.len()) { + self.checkpoint = None; + } + match self.forward.forward_greedy(tokens, start, want_logits) { + Ok(Some(step)) => { + self.processed.truncate(start); + self.processed.extend_from_slice(&tokens[start..]); + if want_logits && step.logits.is_none() { + return Err(RealizarError::InvalidShape { + reason: format!( + "{} forward_greedy was asked for the logits and returned none", + self.arch() + ), + }); + } + Ok(Some(Choice { + token: step.token, + logits: step.logits.filter(|_| want_logits), + })) + }, + Ok(None) => Ok(None), + Err(e) => { + self.processed.clear(); + self.checkpoint = None; + Err(e) + }, + } } fn reserve(&mut self, positions: usize) -> Result<()> { @@ -471,7 +535,13 @@ impl Session { let mut tracer = trace.map(|t| t.tracer); // Only a backend that said it would time its layers has timing to undo. let timing_on = layer_timing && self.forward.set_layer_timing(true); - let turn = self.generate_inner(prompt, config, on_token, &mut tracer); + let turn = self.generate_inner( + prompt, + config, + None, + &mut |token, _| on_token(token), + &mut tracer, + ); if timing_on { self.forward.set_layer_timing(false); } @@ -484,11 +554,33 @@ impl Session { self.forward.take_layer_timings() } + /// [`Session::generate`], and with `top_n` (#4971) the record of each step: + /// `on_token` then gets each token with its [`StepLogprobs`] — the chosen + /// token's logprob and the `top_n` most likely tokens. Asking never + /// changes which token is chosen: a greedy step keeps the device argmax and + /// copies the logits it read. A sampled or penalized step is recorded from + /// the logits after the repetition penalty, before temperature. With + /// `None` no logits are read back for a record and `on_token` gets `None`. + /// + /// # Errors + /// As [`Session::generate`], and a backend whose device argmax was asked + /// for the logits and returned none. + pub fn generate_with_logprobs( + &mut self, + prompt: &[u32], + config: &QuantizedGenerateConfig, + top_n: Option, + on_token: &mut dyn FnMut(u32, Option) -> bool, + ) -> Result { + self.generate_inner(prompt, config, top_n, on_token, &mut None) + } + fn generate_inner( &mut self, prompt: &[u32], config: &QuantizedGenerateConfig, - on_token: &mut dyn FnMut(u32) -> bool, + top_n: Option, + on_token: &mut dyn FnMut(u32, Option) -> bool, tracer: &mut Option<&mut crate::inference_trace::InferenceTracer>, ) -> Result { use crate::inference_trace::TraceStep; @@ -505,7 +597,8 @@ impl Session { let prefill_started = Instant::now(); let reused = self.prepare_prompt(prompt)?; let mut rng = rand::rngs::StdRng::seed_from_u64(config.seed); - let (mut next, _) = self.advance_and_choose(prompt, config, &mut rng)?; + let want_logits = top_n.is_some(); + let mut choice = self.advance_and_choose(prompt, config, &mut rng, want_logits)?; let prefill_ended = Instant::now(); record(tracer, TraceStep::TransformerBlock, 0, prefill_started); let mut tokens = prompt.to_vec(); @@ -519,12 +612,16 @@ impl Session { // never computes a token it throws away (#4325). if generated > 1 { let clock = std::time::Instant::now(); - next = self.advance_and_choose(&tokens, config, &mut rng)?.0; + choice = self.advance_and_choose(&tokens, config, &mut rng, want_logits)?; record(tracer, TraceStep::TransformerBlock, generated - 1, clock); } + let next = choice.token; + let logprobs = top_n + .zip(choice.logits.take()) + .map(|(n, logits)| StepLogprobs::of(generated - 1, next, &logits, n)); tokens.push(next); let clock = std::time::Instant::now(); - let keep_going = on_token(next); + let keep_going = on_token(next, logprobs); record(tracer, TraceStep::Decode, generated - 1, clock); if !keep_going || config.stop_tokens.contains(&next) { break; diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index 715c7cf76f..82352d7adb 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -366,11 +366,34 @@ fn digest_separates_order_and_length() { } /// Scripted with a device argmax: counts `forward_greedy` calls and answers -/// `greedy_answer` without logits. +/// `greedy_answer`, with `read_back` as the logits when they are asked for. struct DeviceGreedy { inner: Scripted, greedy_answer: u32, greedy_calls: usize, + /// Each call's `read_logits` (#4971). + reads: Vec, + /// The logits a read hands back; `None` plays a backend that drops them. + read_back: Option>, +} + +impl DeviceGreedy { + fn new(greedy_answer: u32) -> Self { + Self { + inner: Scripted::new(3, 100), + greedy_answer, + greedy_calls: 0, + reads: Vec::new(), + read_back: None, + } + } + + fn reading(greedy_answer: u32, read_back: Vec) -> Self { + Self { + read_back: Some(read_back), + ..Self::new(greedy_answer) + } + } } impl ArchForward for DeviceGreedy { @@ -404,21 +427,30 @@ impl ArchForward for DeviceGreedy { fn restore_checkpoint(&mut self) -> Result { self.inner.restore_checkpoint() } - fn forward_greedy(&mut self, tokens: &[u32], start: usize) -> Result> { + fn forward_greedy( + &mut self, + tokens: &[u32], + start: usize, + read_logits: bool, + ) -> Result> { self.inner.calls.push((tokens.len(), start)); self.inner.held = tokens.len(); self.greedy_calls += 1; - Ok(Some(self.greedy_answer)) + self.reads.push(read_logits); + Ok(Some(GreedyStep { + token: self.greedy_answer, + logits: if read_logits { + self.read_back.clone() + } else { + None + }, + })) } } #[test] fn plain_greedy_takes_the_device_argmax_and_a_penalty_or_sampling_does_not() { - let device = |answer| DeviceGreedy { - inner: Scripted::new(3, 100), - greedy_answer: answer, - greedy_calls: 0, - }; + let device = DeviceGreedy::new; let mut s = Session::new(device(5)); let turn = s .generate(&[7701, 7702], &greedy(2), &mut |_| true) @@ -939,3 +971,108 @@ fn prepare_turn_probes_what_the_first_forward_covers() { "the first forward covers the probe" ); } + +// #4971: logprobs come from the logits the choice read, and asking for them +// never changes the token or the path that chose it. +mod logprobs_4971 { + use super::*; + + fn collect( + s: &mut Session, + prompt: &[u32], + config: &QuantizedGenerateConfig, + top_n: Option, + ) -> Result<(Turn, Vec>)> { + let mut records = Vec::new(); + let turn = s.generate_with_logprobs(prompt, config, top_n, &mut |_, r| { + records.push(r); + true + })?; + Ok((turn, records)) + } + + #[test] + fn the_host_path_records_each_step_and_chooses_the_same_tokens() { + let mut plain = Session::new(Scripted::new(3, 100)); + let want = plain + .generate(&[7901, 7902], &greedy(3), &mut |_| true) + .expect("plain"); + let mut s = Session::new(Scripted::new(3, 100)); + let (turn, records) = collect(&mut s, &[7901, 7902], &greedy(3), Some(2)).expect("turn"); + assert_eq!(turn.tokens, want.tokens); + assert_eq!(records.len(), 3); + for (i, r) in records.iter().enumerate() { + let r = r.as_ref().expect("a record per step"); + assert_eq!(r.step, i); + assert_eq!(r.chosen, 3); + assert_eq!(r.top[0].token_id, 3, "greedy: top-1 is the choice"); + assert_eq!(r.chosen_logprob, r.top[0].logprob); + assert_eq!(r.top.len(), 2); + assert!(r.top2_margin().is_some_and(|m| (m - 1.0).abs() < 1e-6)); + } + // Without top_n nothing is recorded. + let mut s = Session::new(Scripted::new(3, 100)); + let (_, none) = collect(&mut s, &[7901, 7902], &greedy(3), None).expect("turn"); + assert!(none.iter().all(Option::is_none)); + } + + #[test] + fn the_device_argmax_is_kept_and_its_logits_are_reported_as_read() { + // The read-back logits rank token 2 first; the device chose 5. The + // record must keep the device's token and report the distribution. + let read_back = vec![0.0, 0.0, 3.0, 0.0, 0.0, 1.0, 0.0, 0.0]; + let mut s = Session::new(DeviceGreedy::reading(5, read_back.clone())); + let (turn, records) = collect(&mut s, &[7911, 7912], &greedy(2), Some(2)).expect("turn"); + assert_eq!(turn.tokens, vec![7911, 7912, 5, 5]); + assert_eq!(s.engine().greedy_calls, 2, "the device path, every step"); + assert_eq!(s.engine().reads, vec![true, true]); + for r in records.iter().map(|r| r.as_ref().expect("record")) { + assert_eq!(r.chosen, 5); + assert_eq!(r.top[0].token_id, 2); + let want = crate::gguf::logprob_of(&read_back, 5); + assert!((r.chosen_logprob - want).abs() < 1e-6); + } + + // The same turn without logprobs: same tokens, same path, no read. + let mut s = Session::new(DeviceGreedy::reading(5, read_back)); + let (plain, _) = collect(&mut s, &[7911, 7912], &greedy(2), None).expect("turn"); + assert_eq!(plain.tokens, turn.tokens); + assert_eq!(s.engine().greedy_calls, 2); + assert_eq!(s.engine().reads, vec![false, false]); + } + + #[test] + fn a_near_tie_and_a_clear_winner_read_as_their_margins() { + let near = vec![0.0, 1.0, 1.01, -3.0, 0.0, 0.0, 0.0, 0.0]; + let mut s = Session::new(DeviceGreedy::reading(2, near)); + let (_, records) = collect(&mut s, &[7921], &greedy(1), Some(2)).expect("turn"); + let margin = records[0].as_ref().and_then(StepLogprobs::top2_margin); + assert!(margin.is_some_and(|m| m < 0.05), "{margin:?}"); + + let clear = vec![0.0, 5.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let mut s = Session::new(DeviceGreedy::reading(1, clear)); + let (_, records) = collect(&mut s, &[7922], &greedy(1), Some(2)).expect("turn"); + let margin = records[0].as_ref().and_then(StepLogprobs::top2_margin); + assert!(margin.is_some_and(|m| m > 1.0), "{margin:?}"); + } + + #[test] + fn a_device_path_that_drops_the_logits_fails_loudly() { + let mut s = Session::new(DeviceGreedy::new(5)); + let err = collect(&mut s, &[7931, 7932], &greedy(2), Some(2)) + .expect_err("asked for logits, got none"); + assert!(err.to_string().contains("returned none"), "{err}"); + } + + #[test] + fn a_penalized_step_reports_the_logits_after_the_penalty() { + let mut config = greedy(2); + config.repeat_penalty = 2.0; + let mut s = Session::new(Scripted::new(3, 100)); + let (_, records) = collect(&mut s, &[7941], &config, Some(1)).expect("turn"); + let logit = |i: usize| records[i].as_ref().expect("record").top[0].logit; + // Step 0: token 3 is not yet in the context; step 1: it is, halved. + assert_eq!(logit(0), 1.0); + assert_eq!(logit(1), 0.5); + } +} From ef4d87139d3795172208f9ca4b98bbc87623750b Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 04:32:42 +0200 Subject: [PATCH 03/35] feat(serve): chat logprobs on the quantized CPU path; every other backend refuses 501 (PMAT-4971) V3-a slice 2 of #4971 (ASOC-INV-021). /v1/chat/completions now reads `logprobs` and `top_logprobs`. Before this, serde dropped both keys, so a client that asked for logprobs got a 200 without them. - `top_logprobs` outside 0..=20 is refused when the request is deserialized, and the error names the field. `top_logprobs` without `logprobs: true` is a 400 from `field_conflict`, next to the thinking toggle. - The quantized CPU non-streaming turn runs `dense_turn_with_logprobs`. It keeps one record per generated token and drops the stop token's record. A count check refuses a reply whose entries do not match its tokens. - Logprobs are inserted into the serialized reply, so a request without them gets the same bytes as before and no `logprobs` key. - Every backend without a logprobs path answers 501 and names itself: GPU, cached, CUDA, SafeTensors CUDA, APR f32, APR Q4K, Qwen3-MoE, Qwen3.5, the dense registry, and the quantized stream and traced loop. Streamed entries are a later slice. - `BPETokenizer::decode_bytes` gives a token's exact bytes. `decode` was at cognitive 41 and is split into helpers with the same behaviour. FALSIFY-LOGPROBS-WIRE-4971 (`api::tests::logprobs_4971`, through the real router) now states the 501 for a streamed request. It compares an entry with top_logprobs[0] by logprob, because the fixture's logits can tie. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 79765518e1ff32c72ff5c8184d31656594e0b5f1) (cherry picked from commit 230031c9f04670e33431015f8974ddb33105b1aa) (cherry picked from commit 490d395c55d6df4b669e9117ceb064f517124ef3) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- .../aprender-serve/src/api/chat_logprobs.rs | 455 ++++++++++++++++++ .../src/api/cuda_chat_backend.rs | 78 ++- crates/aprender-serve/src/api/mod.rs | 5 + .../aprender-serve/src/api/mod_create_demo.rs | 24 +- .../aprender-serve/src/api/openai_handlers.rs | 32 +- .../src/api/qwen35_chat_backend.rs | 3 +- .../src/api/tests/logprobs_4971.rs | 114 +++++ crates/aprender-serve/src/api/tests/mod.rs | 2 + .../src/api/tests/stream_mode_pp27.rs | 1 + .../gguf/inference/forward/dense_session.rs | 66 ++- crates/aprender-serve/src/tokenizer.rs | 106 ++-- 12 files changed, 808 insertions(+), 80 deletions(-) create mode 100644 crates/aprender-serve/src/api/chat_logprobs.rs create mode 100644 crates/aprender-serve/src/api/tests/logprobs_4971.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 712138921c..11608b1b5e 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -660,7 +660,7 @@ falsification_tests: if_fails: 'Asking for the margin changes the kernels that choose the token. The witness then measures a different run from the one it claims, which is the defect V3 exists to rule out.' - id: FALSIFY-LOGPROBS-WIRE-4971 name: chat_logprobs_are_served_or_refused_by_name - prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry token equal to top_logprobs[0].token. Streamed, the entries of all chunks concatenate to completion_tokens entries. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. A backend with no logprobs path (the registry fallback) answers 501 and names itself.' + prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. A path with no logprobs answers 501 and names itself: the streamed quantized reply (`quantized (streaming)`) until streamed entries land, and every other chat backend.' test_harness: 'cargo test -p aprender-serve --lib api::tests::logprobs_4971' expected_output: "test result: ok" if_fails: 'A client that asks for logprobs gets a 200 without them (serde ignores unknown keys, which was main''s behaviour before #4971). The perf041 witness then records no margin and S5 can never pass.' diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs new file mode 100644 index 0000000000..de76c355e5 --- /dev/null +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -0,0 +1,455 @@ +//! #4971 (V3-a): OpenAI `logprobs` and `top_logprobs` on `/v1/chat/completions`. +//! +//! ASOC-INV-021 (`contracts/apr-serve-openai-compat-v1.yaml`): with `logprobs: +//! true` the reply carries `choices[0].logprobs.content`, one entry per +//! generated token. A backend that does not compute them answers 501 and names +//! itself, and a request without `logprobs` gets a reply with no `logprobs` key. +//! +//! The entries are put into the serialized reply rather than carried as a field +//! of [`super::ChatChoice`], so a reply without them is byte-identical to the +//! reply before #4971. + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; + +use super::{ChatCompletionRequest, ChatCompletionResponse, ErrorResponse}; +use crate::gguf::logprobs::StepLogprobs; +use crate::tokenizer::BPETokenizer; + +/// The most `top_logprobs` OpenAI allows, and so the most this server returns. +pub const MAX_TOP_LOGPROBS: u8 = 20; + +/// `top_logprobs`: how many of each step's best tokens to return, 0 to +/// [`MAX_TOP_LOGPROBS`]. +/// +/// Anything else is refused at deserialization with a reason that names the +/// field, so no handler ever sees it (the [`super::ChoiceCount`] pattern). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub struct TopLogprobs(u8); + +impl TopLogprobs { + /// The number of alternatives each entry carries. + #[must_use] + pub fn get(self) -> usize { + usize::from(self.0) + } +} + +impl<'de> Deserialize<'de> for TopLogprobs { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = serde_json::Value::deserialize(deserializer)?; + let in_range = value + .as_u64() + .and_then(|n| u8::try_from(n).ok()) + .filter(|&n| n <= MAX_TOP_LOGPROBS); + in_range.map(Self).ok_or_else(|| { + let got = if value.is_number() { + format!(", not {value}") + } else { + String::new() + }; + serde::de::Error::custom(format!( + "{}top_logprobs must be an integer from 0 to {MAX_TOP_LOGPROBS}{got}", + crate::api::CLIENT_VISIBLE_MARKER + )) + }) + } +} + +impl ChatCompletionRequest { + /// How many alternatives each entry carries when the request asked for + /// logprobs (`top_logprobs`, 0 when absent), else `None` (#4971). + #[must_use] + pub fn logprobs_top_n(&self) -> Option { + (self.logprobs == Some(true)).then(|| self.top_logprobs.map_or(0, TopLogprobs::get)) + } + + /// `Some(reason)` for `top_logprobs` sent without `logprobs: true` (#4971). + /// OpenAI refuses it; serving it would return none of what was asked for. + #[must_use] + pub fn logprobs_conflict(&self) -> Option { + (self.top_logprobs.is_some() && self.logprobs != Some(true)).then(|| { + "top_logprobs requires logprobs: true; send both or neither (#4971)".to_string() + }) + } + + /// Every cross-field conflict the chat handler refuses with 400 before it + /// picks a backend: the thinking toggle (#3723), then logprobs (#4971). + #[must_use] + pub fn field_conflict(&self) -> Option { + self.thinking_conflict() + .or_else(|| self.logprobs_conflict()) + } +} + +/// `choices[0].logprobs` of a chat reply, in OpenAI's shape. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ChatLogprobs { + /// One entry per generated token, in order. + pub content: Vec, +} + +/// One generated token: what was chosen, and the best tokens of that step. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ChatTokenLogprob { + /// The chosen token's text. + pub token: String, + /// ln P(token) under the logits the choice read. + pub logprob: f32, + /// The token's own bytes, which can be part of one character. + pub bytes: Vec, + /// The `top_logprobs` best tokens of this step, best first. + pub top_logprobs: Vec, +} + +/// One of a step's best tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ChatTopLogprob { + /// The token's text. + pub token: String, + /// ln P(token) at this step. + pub logprob: f32, + /// The token's own bytes. + pub bytes: Vec, +} + +impl ChatLogprobs { + /// The wire entries of the engine's step records, one per step. + #[must_use] + pub fn from_steps(tokenizer: &BPETokenizer, steps: &[StepLogprobs]) -> Self { + let content = steps + .iter() + .map(|step| { + let (token, bytes) = token_text(tokenizer, step.chosen); + let top_logprobs = step + .top + .iter() + .map(|top| { + let (token, bytes) = token_text(tokenizer, top.token_id); + ChatTopLogprob { + token, + logprob: top.logprob, + bytes, + } + }) + .collect(); + ChatTokenLogprob { + token, + logprob: step.chosen_logprob, + bytes, + top_logprobs, + } + }) + .collect(); + Self { content } + } +} + +/// A token's text and bytes as a logprobs entry reports them: the bytes it +/// decodes to and their (lossy) text. A token that decodes to no bytes, such +/// as a special token, is reported by its own spelling. An id the vocabulary +/// lacks (a padded row of the LM head) is reported as an empty token. +fn token_text(tokenizer: &BPETokenizer, id: u32) -> (String, Vec) { + let bytes = tokenizer.decode_bytes(&[id]).unwrap_or_default(); + if bytes.is_empty() { + let spelling = tokenizer.get_token(id).unwrap_or_default(); + return (spelling.to_string(), spelling.as_bytes().to_vec()); + } + (String::from_utf8_lossy(&bytes).into_owned(), bytes) +} + +/// The chat reply as JSON, with `choices[0].logprobs` when the request asked +/// for them (#4971). Without them the reply serializes exactly as before, so +/// it has no `logprobs` key. +pub(crate) fn chat_reply( + response: ChatCompletionResponse, + logprobs: Option, +) -> Response { + let Some(logprobs) = logprobs else { + return Json(response).into_response(); + }; + match with_logprobs(&response, &logprobs) { + Some(body) => Json(body).into_response(), + // Never a 200 without what the request asked for. + None => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(ErrorResponse { + error: "the reply could not carry its logprobs (#4971)".to_string(), + }), + ) + .into_response(), + } +} + +fn with_logprobs( + response: &ChatCompletionResponse, + logprobs: &ChatLogprobs, +) -> Option { + let mut body = serde_json::to_value(response).ok()?; + let choice = body.pointer_mut("/choices/0")?.as_object_mut()?; + choice.insert("logprobs".to_string(), serde_json::to_value(logprobs).ok()?); + Some(body) +} + +/// Why `backend` refuses a request that asked for logprobs, or `None` when it +/// did not ask (#4971). A backend that does not compute them answers 501 with +/// this, never a 200 without them. +#[must_use] +pub(crate) fn logprobs_refusal(request: &ChatCompletionRequest, backend: &str) -> Option { + request.logprobs_top_n().map(|_| { + format!( + "`logprobs` is not supported by the {backend} chat backend; the reply would \ + carry none. Refused rather than served (#4971, ASOC-INV-021)." + ) + }) +} + +/// [`logprobs_refusal`] for the quantized CPU backend, which computes them on +/// its non-streaming dense turn only: its stream and its traced loop refuse. +#[must_use] +pub(crate) fn quantized_logprobs_refusal( + request: &ChatCompletionRequest, + traced: bool, +) -> Option { + let path = match (request.stream, traced) { + (true, _) => "quantized (streaming)", + (false, true) => "quantized (traced)", + (false, false) => return None, + }; + logprobs_refusal(request, path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::{ChatChoice, ChatMessage, Usage}; + use crate::gguf::logprobs::TopLogprob; + + fn request(json: &str) -> Result { + serde_json::from_str(&format!(r#"{{"model":"m","messages":[]{json}}}"#)) + .map_err(|e| e.to_string()) + } + + #[test] + fn top_logprobs_is_refused_outside_0_to_20_naming_the_field() { + // (field json, Some(n) accepted / None refused) + let table: &[(&str, Option)] = &[ + (r#","logprobs":true,"top_logprobs":0"#, Some(0)), + (r#","logprobs":true,"top_logprobs":20"#, Some(20)), + (r#","logprobs":true,"top_logprobs":21"#, None), + (r#","logprobs":true,"top_logprobs":256"#, None), + (r#","logprobs":true,"top_logprobs":-1"#, None), + (r#","logprobs":true,"top_logprobs":2.5"#, None), + (r#","logprobs":true,"top_logprobs":"3""#, None), + ]; + for &(json, want) in table { + match (request(json), want) { + (Ok(r), Some(n)) => assert_eq!(r.logprobs_top_n(), Some(n), "{json}"), + (Err(e), None) => assert!( + e.contains("[request] top_logprobs must be an integer from 0 to 20"), + "{json}: the refusal must name the field: {e}" + ), + (got, _) => panic!("{json}: want {want:?}, got {got:?}"), + } + } + } + + #[test] + fn logprobs_top_n_and_the_conflict_follow_the_two_fields() { + // (field json, logprobs_top_n, conflict?) + let table: &[(&str, Option, bool)] = &[ + ("", None, false), + (r#","logprobs":false"#, None, false), + (r#","logprobs":true"#, Some(0), false), + (r#","logprobs":true,"top_logprobs":3"#, Some(3), false), + (r#","top_logprobs":2"#, None, true), + (r#","logprobs":false,"top_logprobs":0"#, None, true), + (r#","logprobs":true,"top_logprobs":null"#, Some(0), false), + ]; + for &(json, top_n, conflict) in table { + let r = request(json).expect("valid request"); + assert_eq!(r.logprobs_top_n(), top_n, "{json}"); + assert_eq!(r.logprobs_conflict().is_some(), conflict, "{json}"); + assert_eq!(r.field_conflict().is_some(), conflict, "{json}"); + if let Some(reason) = r.field_conflict() { + assert!(reason.contains("top_logprobs"), "{reason}"); + } + } + } + + #[test] + fn a_request_without_logprobs_serializes_without_the_fields() { + let r = request("").expect("valid request"); + let json = serde_json::to_string(&r).expect("serialize"); + assert!(!json.contains("logprobs"), "{json}"); + } + + fn tokenizer() -> BPETokenizer { + let vocab = ["", "Ġhi", "<0xE6>", "<|im_end|>"]; + BPETokenizer::new( + vocab.iter().map(|s| (*s).to_string()).collect(), + vec![], + "", + ) + .expect("test tokenizer") + } + + fn step(chosen: u32, logits: &[f32], n: usize) -> StepLogprobs { + StepLogprobs::of(0, chosen, logits, n) + } + + #[test] + fn entries_carry_each_tokens_text_bytes_and_best_alternatives() { + let tok = tokenizer(); + let steps = [ + step(1, &[0.0, 3.0, 1.0, 2.0], 2), + step(2, &[0.0, 0.0, 5.0, 0.0], 1), + ]; + let lp = ChatLogprobs::from_steps(&tok, &steps); + assert_eq!(lp.content.len(), 2); + + let first = &lp.content[0]; + assert_eq!(first.token, " hi"); + assert_eq!(first.bytes, b" hi".to_vec()); + assert!(first.logprob <= 0.0); + let tops: Vec<&str> = first + .top_logprobs + .iter() + .map(|t| t.token.as_str()) + .collect(); + assert_eq!( + tops, + [" hi", "<|im_end|>"], + "best first; a special token by its spelling" + ); + assert_eq!(first.top_logprobs[1].bytes, b"<|im_end|>".to_vec()); + assert_eq!(first.top_logprobs[0].logprob, first.logprob); + + // A byte token is one byte of a character: its bytes are exact, its text lossy. + let second = &lp.content[1]; + assert_eq!(second.bytes, vec![0xE6]); + assert_eq!(second.top_logprobs.len(), 1); + } + + #[test] + fn an_id_the_vocabulary_lacks_is_an_empty_token() { + assert_eq!(token_text(&tokenizer(), 99), (String::new(), Vec::new())); + } + + fn response() -> ChatCompletionResponse { + ChatCompletionResponse { + id: "chatcmpl-1".to_string(), + object: "chat.completion".to_string(), + created: 0, + model: "m".to_string(), + choices: vec![ChatChoice { + index: 0, + message: ChatMessage { + role: "assistant".to_string(), + content: " hi".to_string(), + ..Default::default() + }, + finish_reason: "length".to_string(), + }], + usage: Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + }, + brick_trace: None, + step_trace: None, + layer_trace: None, + timings: None, + used_gpu: None, + } + } + + async fn body(response: Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read body"); + serde_json::from_slice(&bytes).expect("json body") + } + + #[tokio::test] + async fn the_reply_carries_logprobs_only_when_asked() { + let plain = body(chat_reply(response(), None)).await; + assert_eq!(plain, serde_json::to_value(response()).expect("serialize")); + assert!(plain["choices"][0].get("logprobs").is_none()); + + let lp = ChatLogprobs::from_steps(&tokenizer(), &[step(1, &[0.0, 3.0], 1)]); + let with = body(chat_reply(response(), Some(lp.clone()))).await; + let content: ChatLogprobs = + serde_json::from_value(with["choices"][0]["logprobs"].clone()).expect("logprobs"); + assert_eq!(content, lp); + let mut without = with.clone(); + without["choices"][0] + .as_object_mut() + .expect("choice") + .remove("logprobs"); + assert_eq!(without, plain, "logprobs is the only key the request adds"); + } + + #[test] + fn backends_without_logprobs_refuse_only_requests_that_ask() { + let asks = request(r#","logprobs":true,"top_logprobs":2"#).expect("valid"); + let plain = request(r#","logprobs":false"#).expect("valid"); + let reason = logprobs_refusal(&asks, "CUDA").expect("refused"); + assert!( + reason.contains("CUDA") && reason.contains("logprobs"), + "{reason}" + ); + assert_eq!(logprobs_refusal(&plain, "CUDA"), None); + } + + #[test] + fn the_quantized_backend_refuses_its_stream_and_its_traced_loop() { + let mut asks = request(r#","logprobs":true"#).expect("valid"); + // (stream, traced, refused as) + let table: &[(bool, bool, Option<&str>)] = &[ + (false, false, None), + (true, false, Some("quantized (streaming)")), + (false, true, Some("quantized (traced)")), + (true, true, Some("quantized (streaming)")), + ]; + for &(stream, traced, want) in table { + asks.stream = stream; + let got = quantized_logprobs_refusal(&asks, traced); + match want { + None => assert_eq!(got, None, "stream={stream} traced={traced}"), + Some(path) => assert!( + got.as_deref().is_some_and(|r| r.contains(path)), + "stream={stream} traced={traced}: {got:?}" + ), + } + } + let mut plain = request("").expect("valid"); + plain.stream = true; + assert_eq!(quantized_logprobs_refusal(&plain, true), None); + } + + #[test] + fn a_top_entry_is_the_engines_record() { + let s = StepLogprobs { + step: 0, + chosen: 1, + chosen_logprob: -0.25, + top: vec![TopLogprob { + token_id: 1, + logit: 4.0, + logprob: -0.25, + }], + }; + let lp = ChatLogprobs::from_steps(&tokenizer(), &[s]); + assert_eq!(lp.content[0].logprob, -0.25); + assert_eq!(lp.content[0].top_logprobs[0].logprob, -0.25); + } +} diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 62757c1c64..cd83638885 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -20,7 +20,7 @@ fn try_safetensors_cuda_backend( ) { return Some(r); } - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "SafeTensors CUDA") { Ok(t) => t, Err(r) => return Some(r), }; @@ -113,7 +113,7 @@ async fn try_cuda_backend( let t0 = if ttft_trace { Some(std::time::Instant::now()) } else { None }; let cuda_model_lock = state.cuda_model()?; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "CUDA") { Ok(t) => t, Err(r) => return Some(r), }; @@ -207,6 +207,7 @@ async fn try_cuda_backend( request_tool_choice(request), timings, None, + None, )) } @@ -394,6 +395,33 @@ fn dense_cpu_session( ) } +/// The prompt and reply of a non-streaming quantized turn, and the record of +/// each reply token's logprobs when they were asked for (#4971). +type QuantizedTurn = (Vec, Option>); + +/// #4268: the non-streaming dense CPU turn runs on the one engine, and with +/// `top_n` (#4971) records each step; `--trace` keeps the instrumented loop, +/// which records none (a traced request for logprobs is refused before this). +fn quantized_turn( + model: &Arc, + prompt: &[u32], + config: &crate::gguf::QuantizedGenerateConfig, + top_n: Option, +) -> crate::error::Result { + use crate::gguf::dense_session::{dense_turn, dense_turn_with_logprobs}; + if config.trace { + return model + .generate_with_cache(prompt, config) + .map(|tokens| (tokens, None)); + } + let mut session = dense_cpu_session(model); + match top_n { + None => dense_turn(&mut session, prompt, config).map(|(tokens, _)| (tokens, None)), + Some(n) => dense_turn_with_logprobs(&mut session, prompt, config, n) + .map(|(tokens, steps)| (tokens, Some(steps))), + } +} + /// Quantized model (GGUF serve mode) backend with true streaming. fn try_quantized_backend( state: &AppState, @@ -427,6 +455,11 @@ fn try_quantized_backend( state.should_trace(trace_level), cancel, ); + // #4971: logprobs come from the non-streaming dense turn only. + if let Some(reason) = super::chat_logprobs::quantized_logprobs_refusal(request, q_config.trace) + { + return Some(fail_response(state, StatusCode::NOT_IMPLEMENTED, reason)); + } // #3718: the dense CPU loop spends `effective_max_tokens` (the request clamped // to the context room), so the finish is judged against THAT budget. Judged // against the request's number, every context-clamped cut read as "stop". A @@ -486,22 +519,17 @@ fn try_quantized_backend( } // Non-streaming quantized - // #4268: the dense CPU turn runs on the one engine; `--trace` keeps the - // instrumented loop. - let generated = if q_config.trace { - quantized_model.generate_with_cache(&prompt_ids, &q_config) - } else { - crate::gguf::dense_session::dense_turn( - &mut dense_cpu_session(quantized_model), - &prompt_ids, - &q_config, - ) - .map(|(tokens, _)| tokens) - }; - let generated = match generated { - Ok(g) => g, + let turn = quantized_turn( + quantized_model, + &prompt_ids, + &q_config, + request.logprobs_top_n(), + ); + let (generated, steps) = match turn { + Ok(turn) => turn, Err(e) => return Some(fail_response(state, crate::api::generation_error_status(&e), e)), }; + let logprobs = steps.map(|steps| super::ChatLogprobs::from_steps(&tokenizer, &steps)); let token_ids: Vec = generated.iter().skip(prompt_tokens).copied().collect(); let completion_tokens = token_ids.len(); @@ -526,6 +554,7 @@ fn try_quantized_backend( request_tool_choice(request), None, None, + logprobs, )) } @@ -565,7 +594,7 @@ fn try_apr_transformer_backend( ) { return Some(r); } - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "APR transformer (f32)") { Ok(t) => t, Err(r) => return Some(r), }; @@ -636,6 +665,7 @@ fn try_apr_transformer_backend( request_tool_choice(request), None, None, + None, )) } @@ -716,6 +746,10 @@ fn registry_fallback( ) { return r; } + // #4971: no logprobs here either, so refuse rather than drop them. + if let Some(reason) = super::chat_logprobs::logprobs_refusal(request, "dense registry") { + return fail_response(state, StatusCode::NOT_IMPLEMENTED, reason); + } let model_id = if request.model == "default" || request.model.is_empty() { None } else { @@ -791,6 +825,7 @@ fn registry_fallback( request_tool_choice(request), None, None, + None, ) } @@ -872,7 +907,7 @@ async fn try_apr_q4k_chat_backend( use crate::api::apr_q4k_scheduler::AprQ4kRequest; let q4k_tx = state.apr_q4k_tx()?; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "APR Q4K CUDA") { Ok(t) => t, Err(r) => return Some(r), }; @@ -966,6 +1001,7 @@ async fn try_apr_q4k_chat_backend( request_tool_choice(request), None, None, + None, )) } @@ -1005,7 +1041,8 @@ pub async fn openai_chat_completions_handler( ); // #3723: two spellings of the thinking toggle that disagree are refused, never picked between. - if let Some(reason) = request.thinking_conflict() { + // #4971: so is `top_logprobs` without `logprobs: true`. + if let Some(reason) = request.field_conflict() { return fail_response(&state, StatusCode::BAD_REQUEST, reason); } @@ -1287,7 +1324,7 @@ fn try_qwen3_moe_backend( Ok(m) => m, Err(r) => return Some(r), }; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "Qwen3-MoE") { Ok(t) => t, Err(r) => return Some(r), }; @@ -1406,6 +1443,7 @@ fn try_qwen3_moe_backend( request_tool_choice(request), None, Some(used_gpu), + None, )) } diff --git a/crates/aprender-serve/src/api/mod.rs b/crates/aprender-serve/src/api/mod.rs index 79086d40d0..c1346ed68b 100644 --- a/crates/aprender-serve/src/api/mod.rs +++ b/crates/aprender-serve/src/api/mod.rs @@ -74,6 +74,11 @@ pub mod batch_admission; pub mod cuda_batch_scheduler; #[cfg(feature = "cuda")] pub mod iteration_scheduler; +// #4971: logprobs and top_logprobs on the chat route (ASOC-INV-021). +mod chat_logprobs; +pub use chat_logprobs::{ + ChatLogprobs, ChatTokenLogprob, ChatTopLogprob, TopLogprobs, MAX_TOP_LOGPROBS, +}; mod openai_handlers; mod serve_trace; // #4918: tool calls on the streaming chat path (both SSE builders). diff --git a/crates/aprender-serve/src/api/mod_create_demo.rs b/crates/aprender-serve/src/api/mod_create_demo.rs index 9687866775..a98a73bb69 100644 --- a/crates/aprender-serve/src/api/mod_create_demo.rs +++ b/crates/aprender-serve/src/api/mod_create_demo.rs @@ -1,4 +1,3 @@ - /// Create a demo APR v2 model for testing pub(crate) fn create_demo_apr_model(_input_dim: usize) -> Result { use crate::apr::TensorEntry; @@ -75,7 +74,10 @@ pub struct ChatCompletionRequest { /// (aprender#2375) — see `types::deserialize_temperature_f32`. `/api/chat` /// and `/api/generate` build this struct in Rust rather than deserializing /// it, so their own `options.temperature` carries the same guard. - #[serde(default, deserialize_with = "crate::api::types::deserialize_temperature_f32")] + #[serde( + default, + deserialize_with = "crate::api::types::deserialize_temperature_f32" + )] pub temperature: Option, /// Nucleus sampling #[serde(default)] @@ -159,6 +161,15 @@ pub struct ChatCompletionRequest { /// Absent on both = thinking OFF, production's default since #3801. #[serde(default, skip_serializing_if = "Option::is_none")] pub think: Option, + /// #4971: OpenAI's `logprobs`. When true the reply carries + /// `choices[0].logprobs`, one entry per generated token, or the backend + /// answers 501 (ASOC-INV-021). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + /// #4971: OpenAI's `top_logprobs`, 0 to 20 alternatives per entry. Only + /// with `logprobs: true`; alone it is refused with 400. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, } /// The `chat_template_kwargs` apr honours (#3723). @@ -176,13 +187,18 @@ impl ChatCompletionRequest { /// [`Self::thinking_conflict`] at the handler entry, so this never has to choose. #[must_use] pub fn thinking(&self) -> Option { - self.chat_template_kwargs.and_then(|k| k.enable_thinking).or(self.think) + self.chat_template_kwargs + .and_then(|k| k.enable_thinking) + .or(self.think) } /// `Some(reason)` when the two spellings of the toggle disagree (#3723). #[must_use] pub fn thinking_conflict(&self) -> Option { - match (self.chat_template_kwargs.and_then(|k| k.enable_thinking), self.think) { + match ( + self.chat_template_kwargs.and_then(|k| k.enable_thinking), + self.think, + ) { (Some(a), Some(b)) if a != b => Some(format!( "chat_template_kwargs.enable_thinking={a} contradicts think={b}; send one (#3723)" )), diff --git a/crates/aprender-serve/src/api/openai_handlers.rs b/crates/aprender-serve/src/api/openai_handlers.rs index 470e6f1ce3..a74e32efdf 100644 --- a/crates/aprender-serve/src/api/openai_handlers.rs +++ b/crates/aprender-serve/src/api/openai_handlers.rs @@ -65,6 +65,21 @@ fn require_tokenizer(state: &AppState) -> Result, Response> { }) } +/// [`require_tokenizer`] for a chat backend that does not compute logprobs: +/// a request that asked for them is refused with 501 naming `backend`, never +/// served without them (#4971, ASOC-INV-021). +#[allow(clippy::result_large_err)] +fn require_tokenizer_refusing_logprobs( + state: &AppState, + request: &ChatCompletionRequest, + backend: &str, +) -> Result, Response> { + match super::chat_logprobs::logprobs_refusal(request, backend) { + Some(reason) => Err(fail_response(state, StatusCode::NOT_IMPLEMENTED, reason)), + None => require_tokenizer(state), + } +} + /// D5 (ruling-3715-2010): pass a tokenized turn through, or refuse it with a 400 /// that names the limit when it cannot fit the serving context /// ([`AppState::serving_context`]; `None` means no cap, so nothing is refused). @@ -478,6 +493,8 @@ mod pmat821_chat_handler_threading_tests { chat_template_kwargs: None, think: None, stream_options: None, + logprobs: None, + top_logprobs: None, } } @@ -710,6 +727,7 @@ pub(crate) fn build_chat_response( tool_choice: Option, timings: Option, used_gpu: Option, + logprobs: Option, ) -> Response { let traces = build_trace_data( trace_level, @@ -731,6 +749,7 @@ pub(crate) fn build_chat_response( tool_choice, timings, used_gpu, + logprobs, ) } @@ -751,6 +770,7 @@ pub(crate) fn build_chat_response_traced( tool_choice: Option, timings: Option, used_gpu: Option, + logprobs: Option, ) -> Response { let (brick_trace, step_trace, layer_trace) = traces; let (text, finish_reason) = finalize_chat_text(text, stops, completion_tokens, max_tokens); @@ -770,7 +790,7 @@ pub(crate) fn build_chat_response_traced( ), }; - Json(ChatCompletionResponse { + let response = ChatCompletionResponse { used_gpu, id: request_id, object: "chat.completion".to_string(), @@ -790,8 +810,8 @@ pub(crate) fn build_chat_response_traced( step_trace, layer_trace, timings, - }) - .into_response() + }; + super::chat_logprobs::chat_reply(response, logprobs) } /// Serialize a value to an SSE event, returning `None` if serialization fails. @@ -1180,7 +1200,7 @@ fn try_gpu_backend( use crate::gpu::GpuGenerateConfig; let gpu_model_lock = state.gpu_model()?; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "GPU") { Ok(t) => t, Err(r) => return Some(r), }; @@ -1283,6 +1303,7 @@ fn try_gpu_backend( // absent rather than zero. None, None, + None, )) } @@ -1299,7 +1320,7 @@ fn try_cached_backend( use crate::gguf::QuantizedGenerateConfig; let cached_model = state.cached_model()?; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "cached quantized") { Ok(t) => t, Err(r) => return Some(r), }; @@ -1381,6 +1402,7 @@ fn try_cached_backend( // absent rather than zero. None, None, + None, )) } diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend.rs b/crates/aprender-serve/src/api/qwen35_chat_backend.rs index 9d9685639f..70f69e9260 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend.rs @@ -111,7 +111,7 @@ async fn try_qwen35_backend( "the Qwen3.5 session has no retained GGUF to tokenize with (#3571)", )); }; - let tokenizer = match require_tokenizer(state) { + let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "Qwen3.5") { Ok(t) => t, Err(r) => return Some(r), }; @@ -285,6 +285,7 @@ async fn try_qwen35_backend( // turn that fell back to the CPU still answers 200; this is how the client // (`apr code`'s document) can tell. Some(turn.used_gpu), + None, )) } diff --git a/crates/aprender-serve/src/api/tests/logprobs_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_4971.rs new file mode 100644 index 0000000000..0e2e92a460 --- /dev/null +++ b/crates/aprender-serve/src/api/tests/logprobs_4971.rs @@ -0,0 +1,114 @@ +//! FALSIFY-LOGPROBS-WIRE-4971: `logprobs` / `top_logprobs` on `/v1/chat/completions` +//! (ASOC-INV-021), through the real router on a quantized CPU server. +//! +//! Before #4971 the request type had neither field: serde dropped both, the reply +//! carried no `logprobs`, and a client that asked for them was answered as if it had +//! not. Every test here asserts what that client observes. + +use axum::http::StatusCode; +use serde_json::Value; + +use super::native_routes_2376::{post, quantized_state}; + +const CHAT: &str = "/v1/chat/completions"; + +fn chat_body(extra: &str) -> String { + format!( + r#"{{"model":"default","messages":[{{"role":"user","content":"Hi"}}],"max_tokens":4,"temperature":0{extra}}}"# + ) +} + +async fn chat_json(extra: &str) -> Value { + let (status, body) = post(quantized_state(), CHAT, &chat_body(extra)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + serde_json::from_str(&body).expect("chat reply is JSON") +} + +fn logprob(entry: &Value) -> f64 { + entry["logprob"].as_f64().expect("logprob is a number") +} + +#[tokio::test] +async fn logprobs_has_one_entry_per_completion_token_with_its_best_alternatives() { + let reply = chat_json(r#","logprobs":true,"top_logprobs":3"#).await; + let completion_tokens = reply["usage"]["completion_tokens"] + .as_u64() + .expect("usage.completion_tokens"); + let content = reply["choices"][0]["logprobs"]["content"] + .as_array() + .expect("choices[0].logprobs.content is an array"); + assert!( + completion_tokens > 0, + "the fixture generated nothing: {reply}" + ); + assert_eq!(content.len() as u64, completion_tokens, "{reply}"); + for entry in content { + assert!(entry["token"].is_string(), "{entry}"); + assert!(entry["bytes"].is_array(), "{entry}"); + assert!(logprob(entry) <= 0.0, "{entry}"); + let tops = entry["top_logprobs"].as_array().expect("top_logprobs"); + assert_eq!(tops.len(), 3, "{entry}"); + assert!( + tops.windows(2).all(|w| logprob(&w[0]) >= logprob(&w[1])), + "{entry}" + ); + assert!(tops.iter().all(|t| logprob(t) <= 0.0), "{entry}"); + // Greedy: the chosen token is a best one. Compared by logprob, not by id, + // because the fixture's logits may tie. + assert!((logprob(entry) - logprob(&tops[0])).abs() < 1e-4, "{entry}"); + } +} + +#[tokio::test] +async fn asking_for_logprobs_changes_no_token_and_not_asking_adds_no_key() { + let with = chat_json(r#","logprobs":true,"top_logprobs":2"#).await; + let without = chat_json("").await; + assert_eq!( + with["choices"][0]["message"]["content"], + without["choices"][0]["message"]["content"] + ); + assert_eq!( + with["usage"]["completion_tokens"], + without["usage"]["completion_tokens"] + ); + let choice = without["choices"][0].as_object().expect("choice object"); + assert!(!choice.contains_key("logprobs"), "{without}"); +} + +#[tokio::test] +async fn logprobs_without_top_logprobs_carries_empty_alternatives() { + let reply = chat_json(r#","logprobs":true"#).await; + let content = reply["choices"][0]["logprobs"]["content"] + .as_array() + .expect("content"); + assert!(!content.is_empty(), "{reply}"); + assert!(content + .iter() + .all(|e| e["top_logprobs"].as_array().is_some_and(Vec::is_empty))); +} + +#[tokio::test] +async fn top_logprobs_out_of_range_or_without_logprobs_is_refused_naming_the_field() { + for extra in [ + r#","logprobs":true,"top_logprobs":21"#, + r#","logprobs":true,"top_logprobs":-1"#, + r#","top_logprobs":2"#, + r#","logprobs":false,"top_logprobs":2"#, + ] { + let (status, body) = post(quantized_state(), CHAT, &chat_body(extra)).await; + assert!(status.is_client_error(), "{extra}: {status} {body}"); + assert!(body.contains("top_logprobs"), "{extra}: {body}"); + } +} + +#[tokio::test] +async fn a_streamed_request_for_logprobs_is_refused_not_served_without_them() { + let (status, body) = post( + quantized_state(), + CHAT, + &chat_body(r#","stream":true,"logprobs":true"#), + ) + .await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{body}"); + assert!(body.contains("quantized (streaming)"), "{body}"); +} diff --git a/crates/aprender-serve/src/api/tests/mod.rs b/crates/aprender-serve/src/api/tests/mod.rs index 50a9d274b1..416abd7ad9 100644 --- a/crates/aprender-serve/src/api/tests/mod.rs +++ b/crates/aprender-serve/src/api/tests/mod.rs @@ -78,3 +78,5 @@ mod completions_eog_stop_4339; // aprender#4339: raw /v1/completions stops on EO #[cfg(feature = "gpu")] mod completions_eog_stop_4345; // aprender#4345: the wgpu GpuModel /v1/completions path stops on EOS + EOG too mod stream_tool_calls_4918; // aprender#4918: streamed tool calls arrive as delta.tool_calls, finish tool_calls (T1-T12) +#[cfg(feature = "gpu")] // create_test_quantized_model is gpu-gated +mod logprobs_4971; // aprender#4971: chat logprobs/top_logprobs honoured on the quantized CPU path, refused 4xx/501 elsewhere (ASOC-INV-021) diff --git a/crates/aprender-serve/src/api/tests/stream_mode_pp27.rs b/crates/aprender-serve/src/api/tests/stream_mode_pp27.rs index fd9460737c..3034df11b0 100644 --- a/crates/aprender-serve/src/api/tests/stream_mode_pp27.rs +++ b/crates/aprender-serve/src/api/tests/stream_mode_pp27.rs @@ -316,6 +316,7 @@ async fn nonstream_response_carries_timings_when_measured() { None, Some(timings), None, + None, ); let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) .await diff --git a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs index 10a2138ecb..8ab694de16 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs @@ -537,6 +537,55 @@ pub fn dense_stream( config: &crate::gguf::QuantizedGenerateConfig, on_token: &mut dyn FnMut(u32) -> bool, ) -> Result<(Vec, bool)> { + admit_dense_prompt(session, prompt)?; + let turn = session.generate(prompt, config, &mut |t| { + config.stop_tokens.contains(&t) || on_token(t) + })?; + Ok(without_stop_token(turn, prompt.len(), config)) +} + +/// [`dense_turn`] that also records each generated token's logprobs, with +/// the `top_n` most likely tokens of its step (#4971). The record of the stop +/// token that ends the turn is dropped with the token, so there is exactly +/// one record per token of the reply. +/// +/// # Errors +/// As [`dense_turn`], and a turn whose records do not cover every token of +/// the reply: a reply is never handed back with some of its logprobs missing. +pub fn dense_turn_with_logprobs( + session: &mut crate::session::Session, + prompt: &[u32], + config: &crate::gguf::QuantizedGenerateConfig, + top_n: usize, +) -> Result<(Vec, Vec)> { + admit_dense_prompt(session, prompt)?; + let mut steps = Vec::new(); + let turn = session.generate_with_logprobs(prompt, config, Some(top_n), &mut |t, record| { + if !config.stop_tokens.contains(&t) { + steps.extend(record); + } + true + })?; + let (tokens, _) = without_stop_token(turn, prompt.len(), config); + let generated = tokens.len() - prompt.len(); + if steps.len() != generated { + return Err(RealizarError::UnsupportedOperation { + operation: "dense_turn_with_logprobs".to_string(), + reason: format!( + "{generated} tokens were generated but {} have logprobs (#4971)", + steps.len() + ), + }); + } + Ok((tokens, steps)) +} + +/// A prompt longer than the model's context keeps the error the old loops +/// raised, [`RealizarError::ContextLimitExceeded`]. +fn admit_dense_prompt( + session: &crate::session::Session, + prompt: &[u32], +) -> Result<()> { let maximum = session.context_length(); if prompt.len() > maximum { return Err(RealizarError::ContextLimitExceeded { @@ -544,18 +593,25 @@ pub fn dense_stream( maximum, }); } - let turn = session.generate(prompt, config, &mut |t| { - config.stop_tokens.contains(&t) || on_token(t) - })?; + Ok(()) +} + +/// The turn's prompt and reply without the stop token that ended it (the old +/// loops never kept it; the engine does), and whether the GPU served it. +fn without_stop_token( + turn: crate::session::Turn, + prompt_len: usize, + config: &crate::gguf::QuantizedGenerateConfig, +) -> (Vec, bool) { let mut tokens = turn.tokens; - if tokens.len() > prompt.len() + if tokens.len() > prompt_len && tokens .last() .is_some_and(|t| config.stop_tokens.contains(t)) { tokens.pop(); } - Ok((tokens, turn.used_gpu)) + (tokens, turn.used_gpu) } /// Grow `cache` in place to `target` (what it holds stays held), or build a new one diff --git a/crates/aprender-serve/src/tokenizer.rs b/crates/aprender-serve/src/tokenizer.rs index 5ef9de3bcb..ad55f2a361 100644 --- a/crates/aprender-serve/src/tokenizer.rs +++ b/crates/aprender-serve/src/tokenizer.rs @@ -384,6 +384,23 @@ impl BPETokenizer { /// Returns error if any token ID is invalid pub fn decode(&self, token_ids: &[u32]) -> Result { contract_pre_decode!(); + let bytes = self.decode_bytes(token_ids)?; + + // Decode as UTF-8, replacing invalid sequences + let result = String::from_utf8_lossy(&bytes).into_owned(); + contract_post_decode!(&result); + Ok(result) + } + + /// The raw bytes `token_ids` decode to, before [`Self::decode`] turns them + /// into text (#4971: a logprobs entry reports its token's own bytes, which + /// can be part of one character). Special tokens decode to no bytes, as + /// they decode to no text. + /// + /// # Errors + /// + /// Returns error if any token ID is invalid + pub fn decode_bytes(&self, token_ids: &[u32]) -> Result> { let mut bytes: Vec = Vec::new(); for &id in token_ids { @@ -394,55 +411,56 @@ impl BPETokenizer { operation: "decode_bpe_token".to_string(), reason: format!("Invalid token ID: {id}"), })?; + Self::push_token_bytes(token, &mut bytes); + } + Ok(bytes) + } - // Skip special tokens - if token.starts_with("<|") && token.ends_with("|>") { - continue; - } - if token == "" || token == "" || token == "" || token == "" { - continue; - } + /// Append one token's bytes: none for a special token, the byte of a `<0xNN>` + /// token, else each character's byte-level-BPE byte (split out of + /// `decode_bytes` for #4971, unchanged). + fn push_token_bytes(token: &str, bytes: &mut Vec) { + if Self::is_unprinted_special(token) { + return; + } + if let Some(byte) = Self::byte_token_value(token) { + bytes.push(byte); + return; + } + // Decode GPT-2 style byte-level BPE + for c in token.chars() { + Self::push_char_bytes(c, bytes); + } + } - // Handle byte tokens like <0xE6> - if token.starts_with("<0x") && token.ends_with('>') && token.len() == 6 { - if let Ok(byte_val) = u8::from_str_radix( - token - .get(3..5) - .expect("byte token <0xNN> has len 6, indices 3..5 always valid"), - 16, - ) { - bytes.push(byte_val); - continue; - } - } + /// Special tokens decode to nothing. + fn is_unprinted_special(token: &str) -> bool { + (token.starts_with("<|") && token.ends_with("|>")) + || matches!(token, "" | "" | "" | "") + } - // Decode GPT-2 style byte-level BPE - for c in token.chars() { - match c { - 'Ġ' => bytes.push(b' '), // U+0120 -> space - 'Ċ' => bytes.push(b'\n'), // U+010A -> newline - 'ċ' => bytes.push(b'\n'), // lowercase variant - 'Ḃ' => bytes.push(b'\r'), // U+1E02 -> carriage return - '▁' => bytes.push(b' '), // U+2581 SentencePiece -> space - _ => { - // Try GPT-2 unicode-to-byte mapping - if let Some(byte) = Self::gpt2_char_to_byte(c) { - bytes.push(byte); - } else { - // Regular UTF-8 character - let mut buf = [0u8; 4]; - let encoded = c.encode_utf8(&mut buf); - bytes.extend_from_slice(encoded.as_bytes()); - } - }, - } - } + /// The byte a token like `<0xE6>` stands for; `None` for any other token. + fn byte_token_value(token: &str) -> Option { + if !(token.starts_with("<0x") && token.ends_with('>') && token.len() == 6) { + return None; } + u8::from_str_radix(token.get(3..5)?, 16).ok() + } - // Decode as UTF-8, replacing invalid sequences - let result = String::from_utf8_lossy(&bytes).into_owned(); - contract_post_decode!(&result); - Ok(result) + /// One character of a byte-level-BPE token, as the byte(s) it stands for. + fn push_char_bytes(c: char, bytes: &mut Vec) { + match c { + 'Ġ' => bytes.push(b' '), // U+0120 -> space + 'Ċ' => bytes.push(b'\n'), // U+010A -> newline + 'ċ' => bytes.push(b'\n'), // lowercase variant + 'Ḃ' => bytes.push(b'\r'), // U+1E02 -> carriage return + '▁' => bytes.push(b' '), // U+2581 SentencePiece -> space + // Try GPT-2 unicode-to-byte mapping, else the character's own UTF-8. + _ => match Self::gpt2_char_to_byte(c) { + Some(byte) => bytes.push(byte), + None => bytes.extend_from_slice(c.encode_utf8(&mut [0u8; 4]).as_bytes()), + }, + } } /// Convert a GPT-2 byte-level-BPE unicode character back to its original byte. From a7f578b0d3e81003572d100d0852527898b86843 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 04:41:05 +0200 Subject: [PATCH 04/35] feat(serve): streamed chat logprobs on the quantized CPU path (PMAT-4971) Slice 3 of V3-a. A quantized `stream: true` request for logprobs was refused 501 ("quantized (streaming)"); it is now served. - dense_stream_with_logprobs hands each token on with its step record. The stop token's record is dropped with the token, and a token without a record ends the turn with an error. dense_turn_with_logprobs is now built on it. - The engine sends each record ahead of its token, on an unbounded side channel (StreamLogprobs). The stream holds each entry until a chunk carries text, and that chunk carries all held entries. Whatever is still held at the end goes on the terminal chunk. A token whose record is missing, or belongs to another token, ends the stream with an error event. - Entries are inserted at the JSON level, so a stream without logprobs serializes exactly as before. - spawn_quantized_stream takes the stream out of try_quantized_backend. Its cognitive complexity drops from 25 to 21. - Only the traced loop still refuses ("quantized (traced)"). The tests check four things: - The streamed entries sum to usage.completion_tokens. - They equal the unstreamed reply's entries, token for token. - No chunk has a logprobs key without the request. - A traced request is refused 501 by name. FALSIFY-LOGPROBS-WIRE-4971 is updated to match. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b086a4b8d0e91ef8e37aaf1b68827a81eba378bf) (cherry picked from commit bebdd5b5d58e36b6559e81d5110b74ffc5752566) (cherry picked from commit 9b12726ce96e1bf0bf29a9b7a7e666350bc2fe38) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- .../aprender-serve/src/api/chat_logprobs.rs | 124 +++++++++++++----- .../src/api/cuda_chat_backend.rs | 94 ++++++++----- .../aprender-serve/src/api/openai_handlers.rs | 75 ++++++++++- .../src/api/tests/logprobs_4971.rs | 90 +++++++++++-- .../gguf/inference/forward/dense_session.rs | 54 ++++++-- 6 files changed, 353 insertions(+), 86 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 11608b1b5e..0a1d3de0f2 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -660,7 +660,7 @@ falsification_tests: if_fails: 'Asking for the margin changes the kernels that choose the token. The witness then measures a different run from the one it claims, which is the defect V3 exists to rule out.' - id: FALSIFY-LOGPROBS-WIRE-4971 name: chat_logprobs_are_served_or_refused_by_name - prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. A path with no logprobs answers 501 and names itself: the streamed quantized reply (`quantized (streaming)`) until streamed entries land, and every other chat backend.' + prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. Streamed, the same request answers 200 with the entries on its chunks: their count over the chunks equals the terminal chunk''s usage.completion_tokens, and their tokens and logprobs equal the unstreamed reply''s. A stream without `logprobs` has no `logprobs` key on any chunk. A path with no logprobs answers 501 and names itself: the traced quantized loop (`quantized (traced)`), and every other chat backend.' test_harness: 'cargo test -p aprender-serve --lib api::tests::logprobs_4971' expected_output: "test result: ok" if_fails: 'A client that asks for logprobs gets a 200 without them (serde ignores unknown keys, which was main''s behaviour before #4971). The perf041 witness then records no margin and S5 can never pass.' diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs index de76c355e5..3f3662aef4 100644 --- a/crates/aprender-serve/src/api/chat_logprobs.rs +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -127,29 +127,91 @@ impl ChatLogprobs { pub fn from_steps(tokenizer: &BPETokenizer, steps: &[StepLogprobs]) -> Self { let content = steps .iter() - .map(|step| { - let (token, bytes) = token_text(tokenizer, step.chosen); - let top_logprobs = step - .top - .iter() - .map(|top| { - let (token, bytes) = token_text(tokenizer, top.token_id); - ChatTopLogprob { - token, - logprob: top.logprob, - bytes, - } - }) - .collect(); - ChatTokenLogprob { + .map(|step| ChatTokenLogprob::from_step(tokenizer, step)) + .collect(); + Self { content } + } +} + +impl ChatTokenLogprob { + /// The wire entry of one step record of the engine. + #[must_use] + pub fn from_step(tokenizer: &BPETokenizer, step: &StepLogprobs) -> Self { + let (token, bytes) = token_text(tokenizer, step.chosen); + let top_logprobs = step + .top + .iter() + .map(|top| { + let (token, bytes) = token_text(tokenizer, top.token_id); + ChatTopLogprob { token, - logprob: step.chosen_logprob, + logprob: top.logprob, bytes, - top_logprobs, } }) .collect(); - Self { content } + Self { + token, + logprob: step.chosen_logprob, + bytes, + top_logprobs, + } + } +} + +/// #4971: the logprobs of a live stream. The engine sends each token's record +/// ahead of the token, so the record is here by the time the stream receives +/// the token. The entries are held until a chunk carries text, and that chunk +/// carries all of them; what is still held when the stream ends goes on its +/// terminal chunk. So every entry is sent once, in the order of the tokens. +pub(crate) struct StreamLogprobs { + records: tokio::sync::mpsc::UnboundedReceiver, + held: Vec, +} + +impl StreamLogprobs { + /// The engine's end and the stream's end of one stream's records. + pub(crate) fn channel() -> (tokio::sync::mpsc::UnboundedSender, Self) { + let (tx, records) = tokio::sync::mpsc::unbounded_channel(); + let held = Vec::new(); + (tx, Self { records, held }) + } + + /// Holds the entry of `token`, which the stream has just received. + /// + /// # Errors + /// The engine sent no record for `token`, or the record of another token. + /// The stream then ends with an error rather than go on with entries that + /// do not match its tokens. + pub(crate) async fn take( + &mut self, + tokenizer: &BPETokenizer, + token: u32, + ) -> Result<(), String> { + match self.records.recv().await { + Some(step) if step.chosen == token => { + self.held + .push(ChatTokenLogprob::from_step(tokenizer, &step)); + Ok(()) + }, + Some(step) => Err(format!( + "token {token} arrived with the logprobs of token {} (#4971)", + step.chosen + )), + None => Err(format!( + "token {token} arrived without its logprobs (#4971)" + )), + } + } + + /// The entries held until now, for the chunk about to be sent, or `None` + /// when none are held. + pub(crate) fn release(&mut self) -> Option { + if self.held.is_empty() { + return None; + } + let content = std::mem::take(&mut self.held); + Some(ChatLogprobs { content }) } } @@ -189,11 +251,13 @@ pub(crate) fn chat_reply( } } -fn with_logprobs( - response: &ChatCompletionResponse, +/// `reply` (a chat reply or a stream chunk) as JSON, with `logprobs` as its +/// `choices[0].logprobs`. +pub(crate) fn with_logprobs( + reply: &impl Serialize, logprobs: &ChatLogprobs, ) -> Option { - let mut body = serde_json::to_value(response).ok()?; + let mut body = serde_json::to_value(reply).ok()?; let choice = body.pointer_mut("/choices/0")?.as_object_mut()?; choice.insert("logprobs".to_string(), serde_json::to_value(logprobs).ok()?); Some(body) @@ -213,18 +277,16 @@ pub(crate) fn logprobs_refusal(request: &ChatCompletionRequest, backend: &str) - } /// [`logprobs_refusal`] for the quantized CPU backend, which computes them on -/// its non-streaming dense turn only: its stream and its traced loop refuse. +/// its dense turn, streamed or not: only its traced loop refuses. #[must_use] pub(crate) fn quantized_logprobs_refusal( request: &ChatCompletionRequest, traced: bool, ) -> Option { - let path = match (request.stream, traced) { - (true, _) => "quantized (streaming)", - (false, true) => "quantized (traced)", - (false, false) => return None, - }; - logprobs_refusal(request, path) + if !traced { + return None; + } + logprobs_refusal(request, "quantized (traced)") } #[cfg(test)] @@ -411,14 +473,14 @@ mod tests { } #[test] - fn the_quantized_backend_refuses_its_stream_and_its_traced_loop() { + fn the_quantized_backend_refuses_only_its_traced_loop() { let mut asks = request(r#","logprobs":true"#).expect("valid"); // (stream, traced, refused as) let table: &[(bool, bool, Option<&str>)] = &[ (false, false, None), - (true, false, Some("quantized (streaming)")), + (true, false, None), (false, true, Some("quantized (traced)")), - (true, true, Some("quantized (streaming)")), + (true, true, Some("quantized (traced)")), ]; for &(stream, traced, want) in table { asks.stream = stream; diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index cd83638885..1eaeae260f 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -422,6 +422,59 @@ fn quantized_turn( } } +/// `try_quantized_backend`'s stream: the turn runs on a blocking thread and +/// hands each token to the SSE response through the returned channel. With +/// `top_n` (#4971) each token's logprobs record is sent ahead of the token, +/// to the [`StreamLogprobs`](super::chat_logprobs::StreamLogprobs) returned +/// with it. +fn spawn_quantized_stream( + model: Arc, + prompt: Vec, + config: crate::gguf::QuantizedGenerateConfig, + metrics: Arc, + top_n: Option, +) -> ( + tokio::sync::mpsc::Receiver>, + Option, +) { + use crate::gguf::dense_session::{dense_stream, dense_stream_with_logprobs}; + let (tx, rx) = tokio::sync::mpsc::channel::>(16); + let (records, logprobs) = match top_n { + Some(n) => { + let (records, logprobs) = super::chat_logprobs::StreamLogprobs::channel(); + (Some((records, n)), Some(logprobs)) + }, + None => (None, None), + }; + tokio::task::spawn_blocking(move || { + // Stops when the client goes away — see `streaming_token_sink`. + let mut sink = crate::api::openai_handlers::streaming_token_sink(tx.clone(), metrics); + // #4268: the dense CPU turn runs on the one engine. + let result = match records { + _ if config.trace => model + .generate_with_cache_streaming(&prompt, &config, sink) + .map(drop), + None => { + dense_stream(&mut dense_cpu_session(&model), &prompt, &config, &mut sink).map(drop) + }, + Some((records, n)) => { + let mut session = dense_cpu_session(&model); + dense_stream_with_logprobs(&mut session, &prompt, &config, n, &mut |t, record| { + // A record that cannot be sent means the response is gone, + // and the sink sees that too and records the abandonment. + let _ = records.send(record); + sink(t) + }) + .map(drop) + }, + }; + if let Err(e) = result { + let _ = tx.blocking_send(Err(e.to_string())); + } + }); + (rx, logprobs) +} + /// Quantized model (GGUF serve mode) backend with true streaming. fn try_quantized_backend( state: &AppState, @@ -455,7 +508,7 @@ fn try_quantized_backend( state.should_trace(trace_level), cancel, ); - // #4971: logprobs come from the non-streaming dense turn only. + // #4971: logprobs come from the dense turn; the traced loop records none. if let Some(reason) = super::chat_logprobs::quantized_logprobs_refusal(request, q_config.trace) { return Some(fail_response(state, StatusCode::NOT_IMPLEMENTED, reason)); @@ -472,36 +525,14 @@ fn try_quantized_backend( }; if request.stream { - let (tx, rx) = tokio::sync::mpsc::channel::>(16); - let quantized_model_clone = quantized_model.clone(); - let prompt_ids_clone = prompt_ids.clone(); - let q_config_clone = q_config.clone(); - let sink_metrics = state.metrics.clone(); - - tokio::task::spawn_blocking(move || { - // Stops when the client goes away — see `streaming_token_sink`. - let mut sink = - crate::api::openai_handlers::streaming_token_sink(tx.clone(), sink_metrics); - let result = if q_config_clone.trace { - quantized_model_clone - .generate_with_cache_streaming(&prompt_ids_clone, &q_config_clone, sink) - .map(drop) - } else { - // #4268: the dense CPU turn runs on the one engine. - crate::gguf::dense_session::dense_stream( - &mut dense_cpu_session(&quantized_model_clone), - &prompt_ids_clone, - &q_config_clone, - &mut sink, - ) - .map(drop) - }; - if let Err(e) = result { - let _ = tx.blocking_send(Err(e.to_string())); - } - }); - - return Some(true_streaming_sse_response( + let (rx, logprobs) = spawn_quantized_stream( + Arc::clone(quantized_model), + prompt_ids, + q_config, + state.metrics.clone(), + request.logprobs_top_n(), + ); + return Some(live_sse_response( rx, tokenizer, request_id.to_string(), @@ -515,6 +546,7 @@ fn try_quantized_backend( None, request.stop.as_deref(), crate::api::stream_tool_calls::StreamTools::from_request(request), + logprobs, )); } diff --git a/crates/aprender-serve/src/api/openai_handlers.rs b/crates/aprender-serve/src/api/openai_handlers.rs index a74e32efdf..dae84653d2 100644 --- a/crates/aprender-serve/src/api/openai_handlers.rs +++ b/crates/aprender-serve/src/api/openai_handlers.rs @@ -19,6 +19,7 @@ use axum::{ }; use futures::stream::Stream; +use super::chat_logprobs::StreamLogprobs; use super::stream_tool_calls::{self, StreamTools, ToolCallDetector}; use super::{ build_trace_data, clean_chat_output, format_chat_messages, @@ -821,6 +822,26 @@ fn sse_event(value: &impl serde::Serialize) -> Option> .map(|data| Ok(Event::default().data(data))) } +/// #4971: the SSE event of `chunk`, carrying the logprobs entries held until +/// now. Without any, the chunk serializes exactly as before. A chunk that +/// cannot carry its entries is not sent without them: an error goes instead. +// serde_json::json!() uses infallible unwrap +#[allow(clippy::disallowed_methods)] +fn chunk_event( + chunk: &ChatCompletionChunk, + logprobs: Option<&mut StreamLogprobs>, +) -> Option> { + let Some(entries) = logprobs.and_then(StreamLogprobs::release) else { + return sse_event(chunk); + }; + match super::chat_logprobs::with_logprobs(chunk, &entries) { + Some(value) => sse_event(&value), + None => sse_event(&serde_json::json!({ + "error": "a chunk could not carry its logprobs (#4971)" + })), + } +} + /// Live-stream deltas: decoded RAW, and never split inside a character. /// /// The decode is deliberately RAW. `clean_chat_output()` must never be applied @@ -1082,6 +1103,41 @@ pub(crate) fn true_streaming_sse_response( timings_rx: Option>, stops: Option<&[String]>, tools: Option, +) -> Response { + live_sse_response( + rx, + tokenizer, + request_id, + model_name, + metrics, + start, + max_tokens, + prompt_tokens, + timings_rx, + stops, + tools, + None, + ) +} + +/// [`true_streaming_sse_response`] whose chunks carry the tokens' logprobs +/// (#4971): `logprobs` holds each token's entry until a chunk carries text, +/// and the terminal chunk carries whatever is still held. +// serde_json::json!() uses infallible unwrap +#[allow(clippy::disallowed_methods)] +pub(crate) fn live_sse_response( + rx: tokio::sync::mpsc::Receiver>, + tokenizer: Arc, + request_id: String, + model_name: String, + metrics: Arc, + start: Instant, + max_tokens: usize, + prompt_tokens: usize, + timings_rx: Option>, + stops: Option<&[String]>, + tools: Option, + mut logprobs: Option, ) -> Response { use tokio_stream::wrappers::ReceiverStream; use tokio_stream::StreamExt; @@ -1111,6 +1167,18 @@ pub(crate) fn true_streaming_sse_response( match result { Ok(token_id) => { completion_tokens += 1; + // #4971: the token's entry is held for the chunk that + // releases its text, and a token without one ends the stream. + let taken = match logprobs.as_mut() { + Some(held) => held.take(&tokenizer, token_id).await, + None => Ok(()), + }; + if let Err(e) = taken { + if let Some(evt) = sse_event(&serde_json::json!({ "error": e })) { + yield evt; + } + break; + } // The token is still counted after a stop: the engine // generated it. Draining (not breaking) keeps a stop from // being recorded as an abandoned stream. @@ -1120,7 +1188,7 @@ pub(crate) fn true_streaming_sse_response( .and_then(|t| stream_tool_calls::detect(&mut detector, t)); if let Some(text) = text { let chunk = ChatCompletionChunk::content(&request_id, &model_name, &text); - if let Some(evt) = sse_event(&chunk) { + if let Some(evt) = chunk_event(&chunk, logprobs.as_mut()) { first_content_at.get_or_insert_with(Instant::now); yield evt; } @@ -1160,13 +1228,14 @@ pub(crate) fn true_streaming_sse_response( completion_tokens, total_tokens: prompt_tokens + completion_tokens, }; - if let Some(evt) = sse_event(&ChatCompletionChunk::done_with_usage( + let done = ChatCompletionChunk::done_with_usage( &request_id, &model_name, finish, usage, timings, - )) { + ); + if let Some(evt) = chunk_event(&done, logprobs.as_mut()) { yield evt; } diff --git a/crates/aprender-serve/src/api/tests/logprobs_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_4971.rs index 0e2e92a460..2dd81b8f2e 100644 --- a/crates/aprender-serve/src/api/tests/logprobs_4971.rs +++ b/crates/aprender-serve/src/api/tests/logprobs_4971.rs @@ -5,10 +5,15 @@ //! carried no `logprobs`, and a client that asked for them was answered as if it had //! not. Every test here asserts what that client observes. -use axum::http::StatusCode; +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; use serde_json::Value; +use tower::util::ServiceExt; -use super::native_routes_2376::{post, quantized_state}; +use super::native_routes_2376::{body_string, post, quantized_state}; +use crate::api::create_router; const CHAT: &str = "/v1/chat/completions"; @@ -101,14 +106,79 @@ async fn top_logprobs_out_of_range_or_without_logprobs_is_refused_naming_the_fie } } +/// The `data:` chunks of an SSE body, without the closing `[DONE]`. +fn stream_chunks(body: &str) -> Vec { + body.lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|data| *data != "[DONE]") + .map(|data| serde_json::from_str(data).expect("an SSE chunk is JSON")) + .collect() +} + +async fn stream_chunks_of(extra: &str) -> Vec { + let body = chat_body(&format!(r#","stream":true{extra}"#)); + let (status, body) = post(quantized_state(), CHAT, &body).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let chunks = stream_chunks(&body); + assert!(!chunks.is_empty(), "no chunks: {body}"); + chunks +} + #[tokio::test] -async fn a_streamed_request_for_logprobs_is_refused_not_served_without_them() { - let (status, body) = post( - quantized_state(), - CHAT, - &chat_body(r#","stream":true,"logprobs":true"#), - ) - .await; +async fn a_stream_carries_on_its_chunks_the_entries_of_the_same_reply_unstreamed() { + let chunks = stream_chunks_of(r#","logprobs":true,"top_logprobs":2"#).await; + let completion_tokens = chunks + .iter() + .find_map(|c| c["usage"]["completion_tokens"].as_u64()) + .expect("the terminal chunk carries usage"); + let streamed: Vec<&Value> = chunks + .iter() + .filter_map(|c| c["choices"][0]["logprobs"]["content"].as_array()) + .flatten() + .collect(); + assert!(completion_tokens > 0, "the fixture streamed nothing"); + assert_eq!(streamed.len() as u64, completion_tokens, "{chunks:?}"); + for entry in &streamed { + let tops = entry["top_logprobs"].as_array().expect("top_logprobs"); + assert_eq!(tops.len(), 2, "{entry}"); + assert!(logprob(entry) <= 0.0, "{entry}"); + assert!((logprob(entry) - logprob(&tops[0])).abs() < 1e-4, "{entry}"); + } + // Streaming changes no entry: the same request unstreamed has the same ones. + let reply = chat_json(r#","logprobs":true,"top_logprobs":2"#).await; + let unstreamed = reply["choices"][0]["logprobs"]["content"] + .as_array() + .expect("content"); + assert_eq!(streamed.len(), unstreamed.len(), "{reply}"); + for (s, u) in streamed.iter().zip(unstreamed) { + assert_eq!(s["token"], u["token"], "{s} vs {u}"); + assert!((logprob(s) - logprob(u)).abs() < 1e-4, "{s} vs {u}"); + } +} + +#[tokio::test] +async fn a_stream_without_logprobs_has_no_logprobs_key_on_any_chunk() { + for chunk in stream_chunks_of("").await { + let choice = chunk["choices"][0].as_object().expect("choice object"); + assert!(!choice.contains_key("logprobs"), "{chunk}"); + } +} + +#[tokio::test] +async fn a_traced_request_for_logprobs_is_refused_not_served_without_them() { + let request = Request::builder() + .method("POST") + .uri(CHAT) + .header("content-type", "application/json") + .header("X-Trace-Level", "brick") + .body(Body::from(chat_body(r#","logprobs":true"#))) + .expect("build request"); + let response = create_router(quantized_state()) + .oneshot(request) + .await + .expect("dispatch"); + let status = response.status(); + let body = body_string(response).await; assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{body}"); - assert!(body.contains("quantized (streaming)"), "{body}"); + assert!(body.contains("quantized (traced)"), "{body}"); } diff --git a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs index 8ab694de16..ea29480822 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/dense_session.rs @@ -550,23 +550,21 @@ pub fn dense_stream( /// one record per token of the reply. /// /// # Errors -/// As [`dense_turn`], and a turn whose records do not cover every token of -/// the reply: a reply is never handed back with some of its logprobs missing. +/// As [`dense_stream_with_logprobs`], and a turn whose records do not cover +/// every token of the reply: a reply is never handed back with some of its +/// logprobs missing. pub fn dense_turn_with_logprobs( session: &mut crate::session::Session, prompt: &[u32], config: &crate::gguf::QuantizedGenerateConfig, top_n: usize, ) -> Result<(Vec, Vec)> { - admit_dense_prompt(session, prompt)?; let mut steps = Vec::new(); - let turn = session.generate_with_logprobs(prompt, config, Some(top_n), &mut |t, record| { - if !config.stop_tokens.contains(&t) { - steps.extend(record); - } - true - })?; - let (tokens, _) = without_stop_token(turn, prompt.len(), config); + let (tokens, _) = + dense_stream_with_logprobs(session, prompt, config, top_n, &mut |_, step| { + steps.push(step); + true + })?; let generated = tokens.len() - prompt.len(); if steps.len() != generated { return Err(RealizarError::UnsupportedOperation { @@ -580,6 +578,42 @@ pub fn dense_turn_with_logprobs( Ok((tokens, steps)) } +/// [`dense_stream`] that hands each generated token on together with the +/// record of its logprobs and the `top_n` most likely tokens of its step +/// (#4971). As in [`dense_stream`], the stop token that ends the turn is never +/// handed on, so neither is its record. +/// +/// # Errors +/// As [`dense_turn`], and a step the engine recorded nothing for: a token is +/// never handed on without its logprobs. +pub fn dense_stream_with_logprobs( + session: &mut crate::session::Session, + prompt: &[u32], + config: &crate::gguf::QuantizedGenerateConfig, + top_n: usize, + on_token: &mut dyn FnMut(u32, crate::gguf::logprobs::StepLogprobs) -> bool, +) -> Result<(Vec, bool)> { + admit_dense_prompt(session, prompt)?; + let mut unrecorded = None; + let turn = session.generate_with_logprobs(prompt, config, Some(top_n), &mut |t, record| { + if config.stop_tokens.contains(&t) { + return true; + } + let Some(record) = record else { + unrecorded = Some(t); + return false; + }; + on_token(t, record) + })?; + if let Some(token) = unrecorded { + return Err(RealizarError::UnsupportedOperation { + operation: "dense_stream_with_logprobs".to_string(), + reason: format!("token {token} was generated without its logprobs (#4971)"), + }); + } + Ok(without_stop_token(turn, prompt.len(), config)) +} + /// A prompt longer than the model's context keeps the error the old loops /// raised, [`RealizarError::ContextLimitExceeded`]. fn admit_dense_prompt( From b8d2fa28c074f1b91a747b8dd217abbc5700d705 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 04:47:59 +0200 Subject: [PATCH 05/35] feat(serve): chat logprobs on the Qwen3.5 hybrid, streamed and not (PMAT-4971) The hybrid refused `logprobs` with 501. Its session is the same `Session` the dense CPU path runs, and `generate_with_logprobs` already records each step there, so the backend now serves them instead: - one `qwen35_generate` takes the lock, keeps the stop token (and its record) from the callback and stores `on_gpu`, for both wires; - the unstreamed reply drops the stop token in `qwen35_turn` and refuses a reply whose records do not number its tokens; - the stream sends each record ahead of its token to `live_sse_response`. The hybrid has no device argmax, so a step's record is read from the logits its token was chosen from, on CPU and GPU alike. Without `logprobs` the turn is `generate` as before (`generate` is `generate_with_logprobs` with no `top_n`). New falsifier FALSIFY-LOGPROBS-QWEN35-4971 (CPU, stream, and GPU under --features cuda) on the Qwen3.5-0.8B fixture; an absent model is a SKIP line, which is not_measured. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1bdda289f5989a5fcad5565ff10c03312db92d70) (cherry picked from commit 4ebaa9ed40ca01c90a49c955504d917d1cf659ff) (cherry picked from commit 746b32e8a398d42688d9099a3419910aa66c80a0) --- contracts/apr-serve-openai-compat-v1.yaml | 8 +- .../src/api/qwen35_chat_backend.rs | 265 ++++++++++++------ .../src/api/qwen35_chat_backend_tests.rs | 132 +++++++++ crates/aprender-serve/src/session.rs | 32 ++- 4 files changed, 338 insertions(+), 99 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 0a1d3de0f2..1fe485fbd1 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -660,7 +660,13 @@ falsification_tests: if_fails: 'Asking for the margin changes the kernels that choose the token. The witness then measures a different run from the one it claims, which is the defect V3 exists to rule out.' - id: FALSIFY-LOGPROBS-WIRE-4971 name: chat_logprobs_are_served_or_refused_by_name - prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. Streamed, the same request answers 200 with the entries on its chunks: their count over the chunks equals the terminal chunk''s usage.completion_tokens, and their tokens and logprobs equal the unstreamed reply''s. A stream without `logprobs` has no `logprobs` key on any chunk. A path with no logprobs answers 501 and names itself: the traced quantized loop (`quantized (traced)`), and every other chat backend.' + prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. Streamed, the same request answers 200 with the entries on its chunks: their count over the chunks equals the terminal chunk''s usage.completion_tokens, and their tokens and logprobs equal the unstreamed reply''s. A stream without `logprobs` has no `logprobs` key on any chunk. A path with no logprobs answers 501 and names itself: the traced quantized loop (`quantized (traced)`), and every other chat backend but the Qwen3.5 hybrid (FALSIFY-LOGPROBS-QWEN35-4971).' test_harness: 'cargo test -p aprender-serve --lib api::tests::logprobs_4971' expected_output: "test result: ok" if_fails: 'A client that asks for logprobs gets a 200 without them (serde ignores unknown keys, which was main''s behaviour before #4971). The perf041 witness then records no margin and S5 can never pass.' + - id: FALSIFY-LOGPROBS-QWEN35-4971 + name: the_hybrid_serves_chat_logprobs + prediction: 'On a Qwen3.5-0.8B Q4_K_M server (APR_QWEN35_MODEL, else ~/models), `logprobs: true, top_logprobs: 2` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 2 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob. With --features cuda on a CUDA host the GPU session gives the same (gpu_logprobs_4971_*). The stop token that ends a turn has no entry, as it has no text. An absent model file prints a SKIP line, which is not_measured and never a pass.' + test_harness: 'cargo test -p aprender-serve --lib qwen35_serve_tests::logprobs_4971 -- --nocapture' + expected_output: "test result: ok" + if_fails: 'The V3 witness model is served by the Qwen3.5 backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the model it is specified against.' diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend.rs b/crates/aprender-serve/src/api/qwen35_chat_backend.rs index 70f69e9260..72bd3e2518 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend.rs @@ -1,54 +1,175 @@ // #3571: /v1/chat/completions for the Qwen3.5 hybrid, served from the resident // `Qwen35Session` the server built once at startup. -/// Spawn the blocking generate that feeds the SSE stream. +/// One generate on the resident session: the turn, with the stop token that +/// ends it kept from `on_token` -- it is not content, and with `top_n` (#4971) +/// neither is its logprobs record. A poisoned lock is refused, and `on_gpu` +/// records the route the turn ended on. Returned with the turn: the instant its +/// decode ended (#4954). +fn qwen35_generate( + served: &crate::api::Qwen35Served, + prompt: &[u32], + config: &crate::gguf::QuantizedGenerateConfig, + top_n: Option, + on_token: &mut dyn FnMut(u32, Option) -> bool, +) -> Result<(crate::session::Turn, Instant), String> { + let mut s = served.session.lock().map_err(|_| POISONED.to_string())?; + let turn = s.generate_with_logprobs(prompt, config, top_n, &mut |token, record| { + config.stop_tokens.contains(&token) || on_token(token, record) + }); + let decode_ended = Instant::now(); + served + .on_gpu + .store(s.on_gpu(), std::sync::atomic::Ordering::Relaxed); + turn.map(|turn| (turn, decode_ended)) + .map_err(|e| e.to_string()) +} + +/// The non-streaming Qwen3.5 reply (#4971, APR-OBS-001 OBS-09). +struct Qwen35Reply { + /// The reply's tokens, without the stop token that ended it. + tokens: Vec, + /// With `top_n`, one logprobs record per reply token; else empty. + steps: Vec, + /// What THIS turn ran on, measured by the session at its end (#3719). + used_gpu: bool, + /// What a step|layer trace measured over the turn. + measured: TracedTurn, + /// #4954: the turn's phase split, which the reply's `timings` report. + phases: crate::api::PhaseTimings, +} + +/// The non-streaming Qwen3.5 turn on the resident session: with `top_n` +/// (#4971) one logprobs record per reply token, the stop token's dropped with +/// it; with `tracer` (APR-OBS-001 OBS-09) the turn's events, and with +/// `time_layers` its per-layer time. A poisoned lock is refused, and `on_gpu` +/// records the route the turn ended on. +fn qwen35_turn( + served: &crate::api::Qwen35Served, + prompt: &[u32], + config: &crate::gguf::QuantizedGenerateConfig, + top_n: Option, + mut tracer: Option, + time_layers: bool, +) -> Result { + let mut s = served.session.lock().map_err(|_| POISONED.to_string())?; + let trace = tracer.as_mut().map(|tracer| crate::session::TurnTrace { + tracer, + layers: time_layers, + }); + let mut steps = Vec::new(); + let turn = s.generate_traced_with_logprobs( + prompt, + config, + top_n, + &mut |token, record| { + if !config.stop_tokens.contains(&token) { + steps.extend(record); + } + true + }, + trace, + ); + let decode_ended = Instant::now(); + let layers = if time_layers { + s.take_layer_timings() + } else { + None + }; + let measured = TracedTurn { + tracer, + layers, + num_layers: s.num_layers(), + }; + served + .on_gpu + .store(s.on_gpu(), std::sync::atomic::Ordering::Relaxed); + let turn = turn.map_err(|e| e.to_string())?; + let mut reply = turn.tokens[prompt.len()..].to_vec(); + if reply.last().is_some_and(|t| config.stop_tokens.contains(t)) { + reply.pop(); + } + if top_n.is_some() && steps.len() != reply.len() { + return Err(format!( + "the reply's {} tokens came with {} logprobs records (#4971)", + reply.len(), + steps.len() + )); + } + Ok(Qwen35Reply { + tokens: reply, + steps, + used_gpu: turn.used_gpu, + measured, + phases: crate::api::PhaseTimings::from_marks( + turn.prefill_started, + turn.prefill_ended, + decode_ended, + ), + }) +} + +/// Spawn the blocking generate that feeds the SSE stream, and with `top_n` +/// (#4971) the [`StreamLogprobs`](super::chat_logprobs::StreamLogprobs) its +/// records go to, each ahead of its token. /// /// Extracted from `try_qwen35_backend` (#3844). This was the function's deepest /// nesting -- `if` -> `spawn_blocking` closure -> `match lock()` -> `Ok` arm -> /// `generate` callback closure -> `if let Err` -- and cognitive complexity counts /// NESTING, which is why flattening the nine flat `unwrap_or` arms alone did not move -/// the number. Behaviour is unchanged: same lock, same stop-token-ends-the-turn -/// callback, same `on_gpu` store, same error forwarded down the channel. +/// the number. The lock, the stop-token-ends-the-turn callback and the `on_gpu` +/// store are [`qwen35_generate`]'s; an error is forwarded down the channel. /// -/// #4954: a turn that completes sends its phase split down `timing_tx`, so the -/// stream's terminal chunk carries `timings` (prompt_ms, predicted_ms and the -/// load and first-token edges). A failed turn drops the sender: no timings. +/// #4954: a turn that completes sends its phase split down the returned +/// receiver, so the stream's terminal chunk carries `timings` (prompt_ms, +/// predicted_ms and the load and first-token edges). A failed turn drops the +/// sender: no timings. fn spawn_streaming_generate( session: Arc, input_ids: Vec, gen_config: crate::gguf::QuantizedGenerateConfig, - stop_tokens: Vec, - tx: tokio::sync::mpsc::Sender>, - timing_tx: tokio::sync::oneshot::Sender, sink_metrics: Arc, + top_n: Option, +) -> ( + tokio::sync::mpsc::Receiver>, + Option, + tokio::sync::oneshot::Receiver, ) { + let (tx, rx) = tokio::sync::mpsc::channel::>(64); + let (timing_tx, timing_rx) = tokio::sync::oneshot::channel::(); + let (records, logprobs) = top_n + .map(|_| super::chat_logprobs::StreamLogprobs::channel()) + .unzip(); tokio::task::spawn_blocking(move || { let mut sink = crate::api::openai_handlers::streaming_token_sink(tx.clone(), sink_metrics); - let result = match session.session.lock() { - Ok(mut s) => { - let r = s.generate(&input_ids, &gen_config, &mut |tok| { - // The stop token ends the turn; it is not content. - stop_tokens.contains(&tok) || sink(tok) - }); - let decode_ended = std::time::Instant::now(); - session - .on_gpu - .store(s.on_gpu(), std::sync::atomic::Ordering::Relaxed); - r.map(|turn| { - let _ = timing_tx.send(crate::api::PhaseTimings::from_marks( - turn.prefill_started, - turn.prefill_ended, - decode_ended, - )); - }) - .map_err(|e| e.to_string()) + let result = qwen35_generate( + &session, + &input_ids, + &gen_config, + top_n, + &mut |t, record| { + // A record that cannot be sent means the response is gone, and + // the sink sees that too and records the abandonment. + if let (Some(records), Some(record)) = (&records, record) { + let _ = records.send(record); + } + sink(t) + }, + ); + match result { + Ok((turn, decode_ended)) => { + let _ = timing_tx.send(crate::api::PhaseTimings::from_marks( + turn.prefill_started, + turn.prefill_ended, + decode_ended, + )); + }, + Err(e) => { + let _ = tx.blocking_send(Err(e)); }, - Err(_) => Err(POISONED.to_string()), - }; - if let Err(e) = result { - let _ = tx.blocking_send(Err(e)); } }); + (rx, logprobs, timing_rx) } /// Build the generate config from a chat request, with the CONTEXT-BOUNDED budget. @@ -111,7 +232,7 @@ async fn try_qwen35_backend( "the Qwen3.5 session has no retained GGUF to tokenize with (#3571)", )); }; - let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "Qwen3.5") { + let tokenizer = match require_tokenizer(state) { Ok(t) => t, Err(r) => return Some(r), }; @@ -159,22 +280,14 @@ async fn try_qwen35_backend( let stop_tokens = stop_tokens_unless_ignore_eos(request, state.model_eos_token_id()); // The context-bounded budget, not the request's number: what is decoded and what // `finish_reason` is judged against are the same count. - let gen_config = gen_config_from_request(request, budget, stop_tokens.clone(), cancel.clone()); + let gen_config = gen_config_from_request(request, budget, stop_tokens, cancel.clone()); + // #4971: the session records each step's logprobs when they were asked for. + let top_n = request.logprobs_top_n(); if request.stream { - let (tx, rx) = tokio::sync::mpsc::channel::>(64); - let (timing_tx, timing_rx) = tokio::sync::oneshot::channel::(); - let sink_metrics = state.metrics.clone(); - spawn_streaming_generate( - session, - input_ids, - gen_config, - stop_tokens, - tx, - timing_tx, - sink_metrics, - ); - return Some(crate::api::openai_handlers::true_streaming_sse_response( + let (rx, logprobs, timing_rx) = + spawn_streaming_generate(session, input_ids, gen_config, state.metrics.clone(), top_n); + return Some(crate::api::openai_handlers::live_sse_response( rx, tokenizer, request_id.to_string(), @@ -186,42 +299,20 @@ async fn try_qwen35_backend( Some(timing_rx), request.stop.as_deref(), crate::api::stream_tool_calls::StreamTools::from_request(request), + logprobs, )); } let decode_mapped = mapped.clone(); // APR-OBS-001 OBS-09: a step|layer header runs the tracer over this turn. - let mut tracer = crate::api::serve_trace::tracer_for(trace_level); + let tracer = crate::api::serve_trace::tracer_for(trace_level); let time_layers = trace_level == Some("layer"); - let turn = tokio::task::spawn_blocking(move || match session.session.lock() { - Ok(mut s) => { - let trace = tracer.as_mut().map(|tracer| crate::session::TurnTrace { - tracer, - layers: time_layers, - }); - let r = s.generate_traced(&input_ids, &gen_config, &mut |_| true, trace); - let decode_ended = std::time::Instant::now(); - let layers = if time_layers { - s.take_layer_timings() - } else { - None - }; - let measured = TracedTurn { - tracer, - layers, - num_layers: s.num_layers(), - }; - session - .on_gpu - .store(s.on_gpu(), std::sync::atomic::Ordering::Relaxed); - r.map(|turn| (turn, measured, decode_ended)) - .map_err(|e| e.to_string()) - }, - Err(_) => Err(POISONED.to_string()), + let turn = tokio::task::spawn_blocking(move || { + qwen35_turn(&session, &input_ids, &gen_config, top_n, tracer, time_layers) }) .await; - let (turn, measured, decode_ended) = match turn { - Ok(Ok(done)) => done, + let reply = match turn { + Ok(Ok(turn)) => turn, Ok(Err(e)) => { state.metrics.record_failure(); return Some(fail_response( @@ -240,24 +331,16 @@ async fn try_qwen35_backend( }, }; - let mut generated_ids = turn.tokens[prompt_token_count..].to_vec(); - if generated_ids - .last() - .is_some_and(|t| stop_tokens.contains(t)) - { - generated_ids.pop(); - } - let completion_tokens = generated_ids.len(); - let response_text = clean_chat_output(&decode_mapped.model.decode(&generated_ids)); + let completion_tokens = reply.tokens.len(); + let response_text = clean_chat_output(&decode_mapped.model.decode(&reply.tokens)); + let logprobs = top_n.map(|_| super::ChatLogprobs::from_steps(&tokenizer, &reply.steps)); + let measured = reply.measured; // #4954: the same phase split the stream reports; no stream, so no // first-content edge. - let timings = crate::api::PhaseTimings::from_marks( - turn.prefill_started, - turn.prefill_ended, - decode_ended, - ) - .to_timings_at(prompt_token_count, completion_tokens, start, None); + let timings = reply + .phases + .to_timings_at(prompt_token_count, completion_tokens, start, None); let duration = start.elapsed(); state.metrics.record_success(completion_tokens, duration); let traces = crate::api::serve_trace::traces_for(&crate::api::serve_trace::ServeTrace { @@ -284,8 +367,8 @@ async fn try_qwen35_backend( // #3719: what THIS turn ran on, measured by the session at its end. A forced-GPU // turn that fell back to the CPU still answers 200; this is how the client // (`apr code`'s document) can tell. - Some(turn.used_gpu), - None, + Some(reply.used_gpu), + logprobs, )) } diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs index 0ea3ba83f8..7019710c74 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs @@ -1026,3 +1026,135 @@ fn second_chat_turn_resumes_from_the_first_turns_checkpoint_4274() { ); } } + +/// #4971: the chat body asking for `logprobs` with two alternatives a step. +fn logprobs_body_4971(stream: bool) -> serde_json::Value { + let mut body = chat_body(stream, 16); + body["logprobs"] = true.into(); + body["top_logprobs"] = 2.into(); + body +} + +/// What a fresh hybrid server answers `body`: the reply, or each `data:` chunk +/// of the stream. `None` when the model file is absent. +async fn chat_json_4971(no_gpu: bool, body: serde_json::Value) -> Option> { + let (state, _) = state_or_skip(no_gpu)?; + let stream = body["stream"] == true; + let (status, text) = post(create_router(state), "/v1/chat/completions", body).await; + assert_eq!(status, StatusCode::OK, "{text}"); + if !stream { + return Some(vec![serde_json::from_str(&text).expect("JSON")]); + } + Some( + text.lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|data| *data != "[DONE]") + .map(|data| serde_json::from_str(data).expect("SSE chunk")) + .collect(), + ) +} + +/// The logprobs entries across `chunks`, in order. +fn entries_4971(chunks: &[serde_json::Value]) -> Vec<&serde_json::Value> { + chunks + .iter() + .filter_map(|c| c["choices"][0]["logprobs"]["content"].as_array()) + .flatten() + .collect() +} + +/// `usage.completion_tokens` of the reply, or of the stream's terminal chunk. +fn completion_tokens_4971(chunks: &[serde_json::Value]) -> u64 { + chunks + .iter() + .find_map(|c| c["usage"]["completion_tokens"].as_u64()) + .expect("usage.completion_tokens") +} + +fn logprob_4971(entry: &serde_json::Value) -> f64 { + entry["logprob"].as_f64().expect("logprob") +} + +/// Each entry is a greedy step's: two alternatives, every logprob <= 0, and +/// the chosen token's logprob the best one (compared by logprob: tied logits +/// make the token alone ambiguous). +fn assert_greedy_entries_4971(entries: &[&serde_json::Value]) { + assert!(!entries.is_empty(), "the reply carries no logprobs entries"); + for entry in entries { + let tops = entry["top_logprobs"].as_array().expect("top_logprobs"); + assert_eq!(tops.len(), 2, "{entry}"); + assert!(logprob_4971(entry) <= 0.0, "{entry}"); + assert!(tops.iter().all(|t| logprob_4971(t) <= 0.0), "{entry}"); + assert!( + (logprob_4971(entry) - logprob_4971(&tops[0])).abs() < 1e-4, + "{entry}" + ); + } +} + +/// #4971 (ASOC-INV-021) on the hybrid: one entry per reply token, asking for +/// them changes no token, and a reply that did not ask has no `logprobs` key. +async fn assert_logprobs_served_and_no_token_changed_4971(no_gpu: bool) { + let Some(with) = chat_json_4971(no_gpu, logprobs_body_4971(false)).await else { + return; + }; + let Some(without) = chat_json_4971(no_gpu, chat_body(false, 16)).await else { + return; + }; + assert_eq!( + with[0]["choices"][0]["message"]["content"], + without[0]["choices"][0]["message"]["content"] + ); + let entries = entries_4971(&with); + assert_eq!( + entries.len() as u64, + completion_tokens_4971(&with), + "{with:?}" + ); + assert_greedy_entries_4971(&entries); + let choice = without[0]["choices"][0].as_object().expect("choice"); + assert!(!choice.contains_key("logprobs"), "{without:?}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn logprobs_4971_one_entry_per_reply_token_and_no_token_changes() { + assert_logprobs_served_and_no_token_changed_4971(true).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn logprobs_4971_a_stream_carries_the_entries_of_the_same_reply_unstreamed() { + let Some(chunks) = chat_json_4971(true, logprobs_body_4971(true)).await else { + return; + }; + let Some(reply) = chat_json_4971(true, logprobs_body_4971(false)).await else { + return; + }; + let streamed = entries_4971(&chunks); + assert_eq!( + streamed.len() as u64, + completion_tokens_4971(&chunks), + "{chunks:?}" + ); + assert_greedy_entries_4971(&streamed); + let unstreamed = entries_4971(&reply); + assert_eq!(streamed.len(), unstreamed.len(), "{reply:?}"); + for (s, u) in streamed.iter().zip(&unstreamed) { + assert_eq!(s["token"], u["token"], "{s} vs {u}"); + assert!( + (logprob_4971(s) - logprob_4971(u)).abs() < 1e-4, + "{s} vs {u}" + ); + } +} + +/// The V3 witness model serves from the GPU session; its logprobs come from the +/// same logits the step chose from. +#[cfg(feature = "cuda")] +#[tokio::test(flavor = "multi_thread")] +async fn gpu_logprobs_4971_the_gpu_session_serves_them_and_changes_no_token() { + if !crate::cuda::CudaExecutor::is_available() { + eprintln!("SKIP: no CUDA device"); + return; + } + assert_logprobs_served_and_no_token_changed_4971(false).await; +} diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index 17dcb1d78e..857adc0b14 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -531,17 +531,35 @@ impl Session { on_token: &mut dyn FnMut(u32) -> bool, trace: Option>, ) -> Result { - let layer_timing = trace.as_ref().is_some_and(|t| t.layers); - let mut tracer = trace.map(|t| t.tracer); - // Only a backend that said it would time its layers has timing to undo. - let timing_on = layer_timing && self.forward.set_layer_timing(true); - let turn = self.generate_inner( + self.generate_traced_with_logprobs( prompt, config, None, &mut |token, _| on_token(token), - &mut tracer, - ); + trace, + ) + } + + /// [`Session::generate_traced`] and [`Session::generate_with_logprobs`] in + /// one turn: the trace records what was measured, and with `top_n` (#4971) + /// `on_token` gets each step's record. Tracing never changes which token is + /// chosen, and neither does asking for logprobs. + /// + /// # Errors + /// As [`Session::generate_with_logprobs`]. + pub fn generate_traced_with_logprobs( + &mut self, + prompt: &[u32], + config: &QuantizedGenerateConfig, + top_n: Option, + on_token: &mut dyn FnMut(u32, Option) -> bool, + trace: Option>, + ) -> Result { + let layer_timing = trace.as_ref().is_some_and(|t| t.layers); + let mut tracer = trace.map(|t| t.tracer); + // Only a backend that said it would time its layers has timing to undo. + let timing_on = layer_timing && self.forward.set_layer_timing(true); + let turn = self.generate_inner(prompt, config, top_n, on_token, &mut tracer); if timing_on { self.forward.set_layer_timing(false); } From b833380188ffee81831b74c3ec5969a3b6213212 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 06:42:13 +0200 Subject: [PATCH 06/35] fix(serve): the chat request names TopLogprobs in scope; DeviceGreedy test init gets its new fields (PMAT-4971) mod_create_demo.rs is include!d into api/mod.rs, so `super::` reached the crate root, where TopLogprobs is not. The new `reads`/`read_back` fields of the DeviceGreedy test engine were missing from one initializer. Both broke the aprender-serve lib and lib-test builds on lane S (ac9303795). Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 042d751ba194aa2f94ebb27d868dba292ec0640c) (cherry picked from commit cff2c2645e9de373ed4759b2f9971f66521bd77e) (cherry picked from commit 35eb5532a8a933b0faac4d8e59876ef04689bb8c) --- crates/aprender-serve/src/api/mod_create_demo.rs | 2 +- crates/aprender-serve/src/session_tests.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/aprender-serve/src/api/mod_create_demo.rs b/crates/aprender-serve/src/api/mod_create_demo.rs index a98a73bb69..83cf45bfe7 100644 --- a/crates/aprender-serve/src/api/mod_create_demo.rs +++ b/crates/aprender-serve/src/api/mod_create_demo.rs @@ -169,7 +169,7 @@ pub struct ChatCompletionRequest { /// #4971: OpenAI's `top_logprobs`, 0 to 20 alternatives per entry. Only /// with `logprobs: true`; alone it is refused with 400. #[serde(default, skip_serializing_if = "Option::is_none")] - pub top_logprobs: Option, + pub top_logprobs: Option, } /// The `chat_template_kwargs` apr honours (#3723). diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index 82352d7adb..091b200cba 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -598,6 +598,8 @@ fn a_device_argmax_from_below_the_checkpoint_drops_it() { inner: Scripted::checkpointing(3, 100, MARK), greedy_answer: 5, greedy_calls: 0, + reads: Vec::new(), + read_back: None, }); let prompt = [7861, 7862, MARK, 7863]; s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); From 3c71ad1fe9fbeae22e6c8b68a0079422cfd36379 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 06:46:37 +0200 Subject: [PATCH 07/35] refactor(serve): split the CUDA batch scheduler loop and process_cuda_batch into named steps (PMAT-4971) No behaviour change. cuda_batch_scheduler_loop (cognitive 85) and process_cuda_batch (cognitive 72) were each one function; the pre-commit hook fails any staged file holding a function over 25, so no change to this file could land. Each step is now its own function, same order, same lock scopes, same log lines: - loop: drain_adaptive (+ wait_for_stragglers), accumulate_window (Accumulated::Closed runs the last batch then stops, as before), log_batch_done. - batch: run_fast_path (+ ttft_trace), phase1_inputs, setup_and_prefill, decode_step (join_slot, recycle_done_slots), fail_all; the five copies of the per-token callback are token_callback. The one dropped line is `if !batch.is_empty()` before the channel-closed batch: the batch always holds the request that opened it. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b37642e0ecab2f65253680d4cd108641586972ce) (cherry picked from commit b1176649fc7c7c9a2ef1ffe134203dfec0f80ea9) (cherry picked from commit dfe1c9bf6395d4afd2c365d33301797f315a749d) --- .../src/api/cuda_batch_scheduler.rs | 647 +++++++++++------- 1 file changed, 385 insertions(+), 262 deletions(-) diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs index a309fd26e3..e7c8312466 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs @@ -254,64 +254,17 @@ async fn cuda_batch_scheduler_loop( // Phase 2: If drain found peers AND batch not full, short timed wait for stragglers. // This eliminates the c=1 TTFT penalty of fixed batch windows while giving // consistent M=max batches at c>1. - let mut batch = vec![first]; - if config.window_ms == 0 { - // PMAT-086/095: Zero-latency drain with cooperative yield + adaptive wait. - tokio::task::yield_now().await; - while batch.len() < config.max_batch { - match rx.try_recv() { - Ok(req) => batch.push(req), - Err(_) => break, - } - } - // PMAT-095/097: Adaptive wait for batch formation. - // Phase 2a: If we found peers, wait 3ms for stragglers. - // Phase 2b (PMAT-097): If we're singleton but recently saw concurrent traffic, - // wait 2ms for peers. Fixes c=4 TTFT P99.9 tail (m=1 batches block 1.7s). - // At true c=1, recent_batch_gt1 stays false → no wait → zero overhead. - let should_wait = if batch.len() > 1 && batch.len() < config.max_batch { - true // Phase 2a: found peers, wait for more - } else if batch.len() == 1 && recent_batch_gt1 && config.max_batch > 1 { - true // Phase 2b: singleton but concurrent traffic detected - } else { - false - }; - if should_wait { - let wait_ms = if batch.len() > 1 { 3 } else { 2 }; - let deadline = - tokio::time::Instant::now() + tokio::time::Duration::from_millis(wait_ms); - while batch.len() < config.max_batch { - match tokio::time::timeout_at(deadline, rx.recv()).await { - Ok(Some(req)) => batch.push(req), - Ok(None) => break, - Err(_timeout) => break, - } - } - } + let batch = if config.window_ms == 0 { + drain_adaptive(first, &mut rx, config.max_batch, recent_batch_gt1).await } else { - let deadline = - tokio::time::Instant::now() + tokio::time::Duration::from_millis(config.window_ms); - - while batch.len() < config.max_batch { - match tokio::time::timeout_at(deadline, rx.recv()).await { - Ok(Some(req)) => batch.push(req), - Ok(None) => { - eprintln!("[PMAT-044] Channel closed during accumulation"); - if !batch.is_empty() { - process_cuda_batch( - &model, - batch, - &mut rx, - config.max_batch, - &in_flight, - ); - } - return; - }, - Err(_timeout) => break, // Window expired - } + match accumulate_window(first, &mut rx, &config).await { + Accumulated::Ready(batch) => batch, + Accumulated::Closed(batch) => { + process_cuda_batch(&model, batch, &mut rx, config.max_batch, &in_flight); + return; + }, } - } + }; let batch_size = batch.len(); let batch_start = std::time::Instant::now(); @@ -322,20 +275,128 @@ async fn cuda_batch_scheduler_loop( // Process the batch (PMAT-073: pass rx for mid-batch joins) process_cuda_batch(&model, batch, &mut rx, config.max_batch, &in_flight); - let elapsed = batch_start.elapsed(); - eprintln!( - "[PMAT-044] Batch m={} done in {:.1}ms ({:.1} tok/s/slot)", - batch_size, - elapsed.as_secs_f64() * 1000.0, - if elapsed.as_secs_f64() > 0.0 { - 1000.0 / elapsed.as_secs_f64() / batch_size as f64 - } else { - 0.0 - } - ); + log_batch_done(batch_size, batch_start.elapsed()); + } +} + +/// PMAT-044: how a timed accumulation window ended. +#[cfg(feature = "cuda")] +enum Accumulated { + /// The window expired or the batch filled; the scheduler keeps going. + Ready(Vec), + /// The channel closed mid-window: run this batch, then stop. + Closed(Vec), +} + +/// PMAT-086/095/097: the zero-window batch — a cooperative yield, a +/// non-blocking drain, then a short adaptive wait for stragglers. +#[cfg(feature = "cuda")] +async fn drain_adaptive( + first: CudaBatchRequest, + rx: &mut tokio::sync::mpsc::Receiver, + max_batch: usize, + recent_batch_gt1: bool, +) -> Vec { + let mut batch = vec![first]; + // PMAT-086/095: Zero-latency drain with cooperative yield + adaptive wait. + tokio::task::yield_now().await; + while batch.len() < max_batch { + match rx.try_recv() { + Ok(req) => batch.push(req), + Err(_) => break, + } + } + // PMAT-095/097: Adaptive wait for batch formation. + // Phase 2a: If we found peers, wait 3ms for stragglers. + // Phase 2b (PMAT-097): If we're singleton but recently saw concurrent traffic, + // wait 2ms for peers. Fixes c=4 TTFT P99.9 tail (m=1 batches block 1.7s). + // At true c=1, recent_batch_gt1 stays false → no wait → zero overhead. + let should_wait = if batch.len() > 1 && batch.len() < max_batch { + true // Phase 2a: found peers, wait for more + } else if batch.len() == 1 && recent_batch_gt1 && max_batch > 1 { + true // Phase 2b: singleton but concurrent traffic detected + } else { + false + }; + if should_wait { + let wait_ms = if batch.len() > 1 { 3 } else { 2 }; + wait_for_stragglers(&mut batch, rx, max_batch, wait_ms).await; + } + batch +} + +/// PMAT-095: up to `max_batch`, whatever reaches `rx` within `wait_ms`; a +/// closed channel ends the wait early. +#[cfg(feature = "cuda")] +async fn wait_for_stragglers( + batch: &mut Vec, + rx: &mut tokio::sync::mpsc::Receiver, + max_batch: usize, + wait_ms: u64, +) { + let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(wait_ms); + while batch.len() < max_batch { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(req)) => batch.push(req), + Ok(None) => break, + Err(_timeout) => break, + } } } +/// PMAT-044: a fixed window of `config.window_ms` for peers to join `first`. +#[cfg(feature = "cuda")] +async fn accumulate_window( + first: CudaBatchRequest, + rx: &mut tokio::sync::mpsc::Receiver, + config: &CudaBatchConfig, +) -> Accumulated { + let mut batch = vec![first]; + let deadline = + tokio::time::Instant::now() + tokio::time::Duration::from_millis(config.window_ms); + + while batch.len() < config.max_batch { + match tokio::time::timeout_at(deadline, rx.recv()).await { + Ok(Some(req)) => batch.push(req), + Ok(None) => { + eprintln!("[PMAT-044] Channel closed during accumulation"); + return Accumulated::Closed(batch); + }, + Err(_timeout) => break, // Window expired + } + } + Accumulated::Ready(batch) +} + +/// PMAT-044: one line per batch, its wall time and per-slot rate. +#[cfg(feature = "cuda")] +fn log_batch_done(batch_size: usize, elapsed: std::time::Duration) { + eprintln!( + "[PMAT-044] Batch m={} done in {:.1}ms ({:.1} tok/s/slot)", + batch_size, + elapsed.as_secs_f64() * 1000.0, + if elapsed.as_secs_f64() > 0.0 { + 1000.0 / elapsed.as_secs_f64() / batch_size as f64 + } else { + 0.0 + } + ); +} + +/// The channel one request's tokens (or its error) go back on. +#[cfg(feature = "cuda")] +type TokenSender = tokio::sync::mpsc::Sender>; + +/// A slot's per-token callback; `false` means its caller has gone. +#[cfg(feature = "cuda")] +type TokenCallback = Box bool + Send>; + +/// The callback that streams a slot's tokens to `token_tx`. +#[cfg(feature = "cuda")] +fn token_callback(token_tx: TokenSender) -> TokenCallback { + Box::new(move |token_id: u32| -> bool { token_tx.try_send(Ok(token_id)).is_ok() }) +} + #[cfg(feature = "cuda")] fn process_cuda_batch( model: &Arc>, @@ -371,22 +432,7 @@ fn process_cuda_batch( // This is acceptable because the fast path's 3x better ITL outweighs the // latency benefit of mid-batch join for the second request. let req = batch.into_iter().next().unwrap(); - if std::env::var("TTFT_TRACE").is_ok() { - eprintln!( - "[TTFT] {:>20}: {:>7.2}ms", - "queue_latency", - req.enqueue_time.elapsed().as_secs_f64() * 1000.0 - ); - } - let mut cuda_model = model.write().expect("PMAT-044: model lock poisoned"); - if std::env::var("TTFT_TRACE").is_ok() { - eprintln!( - "[TTFT] {:>20}: {:>7.2}ms", - "lock_acquired", - req.enqueue_time.elapsed().as_secs_f64() * 1000.0 - ); - } - generate_single_request(&mut cuda_model, req); + run_fast_path(model, req); return; } @@ -397,76 +443,29 @@ fn process_cuda_batch( // Default OFF. Enable: STAGGERED_PREFILL=1. let staggered = std::env::var("STAGGERED_PREFILL").as_deref() == Ok("1") && m > 1; - let mut pending_joins: std::collections::VecDeque = - std::collections::VecDeque::new(); - // Build Phase 1 inputs: first request only (staggered) or all requests (batched) - let (phase1_prompts, phase1_configs, mut error_senders, phase1_callbacks) = if staggered { - // Split batch: first → immediate prefill, rest → pending joins - let mut batch_iter = batch.into_iter(); - let first_req = batch_iter.next().unwrap(); - pending_joins = batch_iter.collect(); - - let prompts = vec![first_req.prompt_ids.clone()]; - let configs = vec![first_req.config.clone()]; - let error_senders = vec![first_req.token_tx.clone()]; - let first_tx = first_req.token_tx; - let callbacks: Vec bool + Send>> = - vec![Box::new(move |token_id: u32| -> bool { - first_tx.try_send(Ok(token_id)).is_ok() - })]; - - eprintln!( - "[PMAT-099] Staggered prefill: 1 immediate + {} pending joins", - pending_joins.len() - ); - - (prompts, configs, error_senders, callbacks) - } else { - // All prompts prefilled together in Phase 1 (original PMAT-072 behavior) - let prompts: Vec> = batch.iter().map(|r| r.prompt_ids.clone()).collect(); - let configs: Vec = - batch.iter().map(|r| r.config.clone()).collect(); - let error_senders: Vec>> = - batch.iter().map(|r| r.token_tx.clone()).collect(); - let callbacks: Vec bool + Send>> = batch - .into_iter() - .map(|req| { - Box::new(move |token_id: u32| -> bool { - req.token_tx.try_send(Ok(token_id)).is_ok() - }) as Box bool + Send> - }) - .collect(); - (prompts, configs, error_senders, callbacks) - }; + let Phase1 { + prompts, + configs, + mut error_senders, + callbacks, + mut pending_joins, + } = phase1_inputs(batch, staggered); // Phase 1: Setup + Prefill (under lock) // Staggered: prefills 1 prompt, pre-allocates max_batch KV slots. // Non-staggered: prefills all M prompts (original behavior). - let mut state = { - let mut cuda_model = model.write().expect("PMAT-072: model lock poisoned"); - match cuda_model.batched_setup_and_prefill( - &phase1_prompts, - &phase1_configs, - phase1_callbacks, - max_batch, - ) { - Ok(s) => s, - Err(e) => { - eprintln!("[PMAT-072] Setup+prefill ERROR (m={m}): {e}"); - for tx in &error_senders { - let _ = tx.try_send(Err(e.to_string())); - } - // Also notify pending joins - for req in &pending_joins { - let _ = req.token_tx.try_send(Err(e.to_string())); - } - // PMAT-765: reset stale batched state (batched_kv_stride) before returning, so - // the NEXT batch doesn't reuse buffers with a stale stride → KV corruption. - cuda_model.reset_batched_state(); - return; - }, - } + let Some(mut state) = setup_and_prefill( + model, + &prompts, + &configs, + callbacks, + max_batch, + m, + &error_senders, + &pending_joins, + ) else { + return; }; // Phase 2: Decode loop with mid-batch joins (PMAT-073/099) and slot recycling (PMAT-074) @@ -475,120 +474,15 @@ fn process_cuda_batch( // PP-24: the initial batch is live from here, joins or not. in_flight.sync(state.m); while !state.all_done() && state.gen_idx < state.max_tokens_max { - let token_ids = { - let mut cuda_model = model.write().expect("PMAT-072: model lock poisoned"); - - // PMAT-099: Join one pending staggered slot per step (progressive ramp-up). - // This limits decode stall to one prefill per step instead of blocking all at once. - if !pending_joins.is_empty() && state.m < state.max_kv_slots { - let req = pending_joins.pop_front().unwrap(); - let error_tx = req.token_tx.clone(); - let prompt_ids = req.prompt_ids; - let config = req.config; - let token_tx = req.token_tx; - let on_token: Box bool + Send> = - Box::new(move |token_id: u32| -> bool { - token_tx.try_send(Ok(token_id)).is_ok() - }); - match cuda_model.add_slot_to_batch(&mut state, prompt_ids, config, on_token) { - Ok(()) => { - error_senders.push(error_tx); - }, - Err(e) => { - eprintln!("[PMAT-099] Staggered join FAILED: {e}"); - let _ = error_tx.try_send(Err(e.to_string())); - }, - } - } - - // PMAT-073: Check for pending requests to join mid-batch (fill empty slots). - while state.m < state.max_kv_slots { - match rx.try_recv() { - Ok(req) => { - let error_tx = req.token_tx.clone(); - let prompt_ids = req.prompt_ids; - let config = req.config; - let token_tx = req.token_tx; - let on_token: Box bool + Send> = - Box::new(move |token_id: u32| -> bool { - token_tx.try_send(Ok(token_id)).is_ok() - }); - match cuda_model.add_slot_to_batch(&mut state, prompt_ids, config, on_token) - { - Ok(()) => { - error_senders.push(error_tx); - }, - Err(e) => { - eprintln!("[PMAT-073] Mid-batch join FAILED: {e}"); - let _ = error_tx.try_send(Err(e.to_string())); - }, - } - }, - Err(_) => break, // No pending requests - } - } - - // PP-24: publish what this batch actually holds, after every join - // and recycle path above has run and before the step decodes it. - in_flight.sync(state.m); - - // PMAT-074: Slot recycling — reuse finished slots for pending requests. - // Check staggered pending joins first, then external channel. - for slot_idx in 0..state.m { - if !state.done[slot_idx] { - continue; - } - // Try pending staggered joins first (they arrived with the initial batch) - let req = if !pending_joins.is_empty() { - Some(pending_joins.pop_front().unwrap()) - } else { - rx.try_recv().ok() - }; - match req { - Some(req) => { - let error_tx = req.token_tx.clone(); - let prompt_ids = req.prompt_ids; - let config = req.config; - let token_tx = req.token_tx; - let on_token: Box bool + Send> = - Box::new(move |token_id: u32| -> bool { - token_tx.try_send(Ok(token_id)).is_ok() - }); - match cuda_model - .recycle_slot(&mut state, slot_idx, prompt_ids, config, on_token) - { - Ok(()) => { - error_senders[slot_idx] = error_tx; - }, - Err(e) => { - eprintln!("[PMAT-074] Slot recycle FAILED (slot {slot_idx}): {e}"); - let _ = error_tx.try_send(Err(e.to_string())); - }, - } - }, - None => break, // No pending requests - } - } - - match cuda_model.batched_decode_step(&mut state) { - Ok(ids) => ids, - Err(e) => { - eprintln!( - "[PMAT-074] Decode step ERROR (m={}, step={}): {e}", - state.m, state.gen_idx - ); - for tx in &error_senders { - let _ = tx.try_send(Err(e.to_string())); - } - // Notify any remaining pending joins - for req in &pending_joins { - let _ = req.token_tx.try_send(Err(e.to_string())); - } - // Still need cleanup under lock - cuda_model.batched_cleanup(&state); - return; - }, - } + let Some(token_ids) = decode_step( + model, + &mut state, + rx, + &mut pending_joins, + &mut error_senders, + &in_flight, + ) else { + return; }; // Token distribution runs WITHOUT model lock — SSE callbacks only @@ -602,6 +496,235 @@ fn process_cuda_batch( } } +/// PMAT-044: the m=1 fast path, holding the model lock for the whole turn. +#[cfg(feature = "cuda")] +fn run_fast_path(model: &Arc>, req: CudaBatchRequest) { + ttft_trace("queue_latency", &req); + let mut cuda_model = model.write().expect("PMAT-044: model lock poisoned"); + ttft_trace("lock_acquired", &req); + generate_single_request(&mut cuda_model, req); +} + +/// TTFT_TRACE: how long `req` has waited by `stage`. +#[cfg(feature = "cuda")] +fn ttft_trace(stage: &str, req: &CudaBatchRequest) { + if std::env::var("TTFT_TRACE").is_ok() { + eprintln!( + "[TTFT] {:>20}: {:>7.2}ms", + stage, + req.enqueue_time.elapsed().as_secs_f64() * 1000.0 + ); + } +} + +/// PMAT-072/099: what Phase 1 prefills, who hears about its tokens and +/// errors, and (staggered) who joins later. +#[cfg(feature = "cuda")] +struct Phase1 { + prompts: Vec>, + configs: Vec, + error_senders: Vec, + callbacks: Vec, + pending_joins: std::collections::VecDeque, +} + +#[cfg(feature = "cuda")] +fn phase1_inputs(batch: Vec, staggered: bool) -> Phase1 { + if staggered { + // Split batch: first → immediate prefill, rest → pending joins + let mut batch_iter = batch.into_iter(); + let first_req = batch_iter.next().unwrap(); + let pending_joins: std::collections::VecDeque = batch_iter.collect(); + + eprintln!( + "[PMAT-099] Staggered prefill: 1 immediate + {} pending joins", + pending_joins.len() + ); + + Phase1 { + prompts: vec![first_req.prompt_ids.clone()], + configs: vec![first_req.config.clone()], + error_senders: vec![first_req.token_tx.clone()], + callbacks: vec![token_callback(first_req.token_tx)], + pending_joins, + } + } else { + // All prompts prefilled together in Phase 1 (original PMAT-072 behavior) + Phase1 { + prompts: batch.iter().map(|r| r.prompt_ids.clone()).collect(), + configs: batch.iter().map(|r| r.config.clone()).collect(), + error_senders: batch.iter().map(|r| r.token_tx.clone()).collect(), + callbacks: batch + .into_iter() + .map(|req| token_callback(req.token_tx)) + .collect(), + pending_joins: std::collections::VecDeque::new(), + } + } +} + +/// Phase 1 under the model lock. `None` when it failed: every caller has +/// been told, and the batched state reset. +#[cfg(feature = "cuda")] +fn setup_and_prefill( + model: &Arc>, + prompts: &[Vec], + configs: &[QuantizedGenerateConfig], + callbacks: Vec, + max_batch: usize, + m: usize, + error_senders: &[TokenSender], + pending_joins: &std::collections::VecDeque, +) -> Option { + let mut cuda_model = model.write().expect("PMAT-072: model lock poisoned"); + match cuda_model.batched_setup_and_prefill(prompts, configs, callbacks, max_batch) { + Ok(s) => Some(s), + Err(e) => { + eprintln!("[PMAT-072] Setup+prefill ERROR (m={m}): {e}"); + // Every slot, and the pending joins too + fail_all(error_senders, pending_joins, &e.to_string()); + // PMAT-765: reset stale batched state (batched_kv_stride) before returning, so + // the NEXT batch doesn't reuse buffers with a stale stride → KV corruption. + cuda_model.reset_batched_state(); + None + }, + } +} + +/// Send `error` to every slot's caller and every pending join's. +#[cfg(feature = "cuda")] +fn fail_all( + error_senders: &[TokenSender], + pending_joins: &std::collections::VecDeque, + error: &str, +) { + for tx in error_senders { + let _ = tx.try_send(Err(error.to_string())); + } + for req in pending_joins { + let _ = req.token_tx.try_send(Err(error.to_string())); + } +} + +/// One Phase 2 step under the model lock: joins (one pending staggered +/// slot, then waiting requests), slot recycling, then the decode itself. +/// `None` when the step failed: every caller has been told, and the batch +/// cleaned up. +#[cfg(feature = "cuda")] +fn decode_step( + model: &Arc>, + state: &mut crate::gguf::BatchedDecodeState, + rx: &mut tokio::sync::mpsc::Receiver, + pending_joins: &mut std::collections::VecDeque, + error_senders: &mut Vec, + in_flight: &BatchInFlight<'_>, +) -> Option> { + let mut cuda_model = model.write().expect("PMAT-072: model lock poisoned"); + + // PMAT-099: Join one pending staggered slot per step (progressive ramp-up). + // This limits decode stall to one prefill per step instead of blocking all at once. + if !pending_joins.is_empty() && state.m < state.max_kv_slots { + let req = pending_joins.pop_front().unwrap(); + join_slot( + &mut cuda_model, + state, + req, + error_senders, + "[PMAT-099] Staggered join FAILED", + ); + } + + // PMAT-073: Check for pending requests to join mid-batch (fill empty slots). + while state.m < state.max_kv_slots { + match rx.try_recv() { + Ok(req) => join_slot( + &mut cuda_model, + state, + req, + error_senders, + "[PMAT-073] Mid-batch join FAILED", + ), + Err(_) => break, // No pending requests + } + } + + // PP-24: publish what this batch actually holds, after every join + // and recycle path above has run and before the step decodes it. + in_flight.sync(state.m); + + recycle_done_slots(&mut cuda_model, state, rx, pending_joins, error_senders); + + match cuda_model.batched_decode_step(state) { + Ok(ids) => Some(ids), + Err(e) => { + eprintln!( + "[PMAT-074] Decode step ERROR (m={}, step={}): {e}", + state.m, state.gen_idx + ); + // Every slot, and any remaining pending joins + fail_all(error_senders, pending_joins, &e.to_string()); + // Still need cleanup under lock + cuda_model.batched_cleanup(state); + None + }, + } +} + +/// PMAT-073/099: a new slot in the running batch for `req`; on failure its +/// caller hears why, and `failed` heads the log line. +#[cfg(feature = "cuda")] +fn join_slot( + cuda_model: &mut OwnedQuantizedModelCuda, + state: &mut crate::gguf::BatchedDecodeState, + req: CudaBatchRequest, + error_senders: &mut Vec, + failed: &str, +) { + let error_tx = req.token_tx.clone(); + let on_token = token_callback(req.token_tx); + match cuda_model.add_slot_to_batch(state, req.prompt_ids, req.config, on_token) { + Ok(()) => { + error_senders.push(error_tx); + }, + Err(e) => { + eprintln!("{failed}: {e}"); + let _ = error_tx.try_send(Err(e.to_string())); + }, + } +} + +/// PMAT-074: Slot recycling — reuse finished slots for pending requests. +/// Staggered pending joins go first (they arrived with the initial batch), +/// then the channel. +#[cfg(feature = "cuda")] +fn recycle_done_slots( + cuda_model: &mut OwnedQuantizedModelCuda, + state: &mut crate::gguf::BatchedDecodeState, + rx: &mut tokio::sync::mpsc::Receiver, + pending_joins: &mut std::collections::VecDeque, + error_senders: &mut [TokenSender], +) { + for slot_idx in 0..state.m { + if !state.done[slot_idx] { + continue; + } + let Some(req) = pending_joins.pop_front().or_else(|| rx.try_recv().ok()) else { + break; // No pending requests + }; + let error_tx = req.token_tx.clone(); + let on_token = token_callback(req.token_tx); + match cuda_model.recycle_slot(state, slot_idx, req.prompt_ids, req.config, on_token) { + Ok(()) => { + error_senders[slot_idx] = error_tx; + }, + Err(e) => { + eprintln!("[PMAT-074] Slot recycle FAILED (slot {slot_idx}): {e}"); + let _ = error_tx.try_send(Err(e.to_string())); + }, + } + } +} + #[cfg(all(test, feature = "cuda"))] #[path = "cuda_batch_scheduler_tests.rs"] mod tests; From cb7b27ed627adc10bc82d5a3feaf231e67da9cbd Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 06:47:47 +0200 Subject: [PATCH 08/35] feat(serve): chat logprobs on the dense CUDA backend, direct and through the scheduler's m=1 path (PMAT-4971) The CUDA chat backend answered `logprobs: true` with 501. It now serves ASOC-INV-021 the way the quantized CPU and Qwen3.5 paths do: - The request carries a RecordSink (top_n + a record channel). The dense turn sends each step's record before its token, on the direct path (dense_cuda_turn) and on the batch scheduler's m=1 fast path (dense_stream_recording), streamed and not. - The handler takes a record per token with a non-blocking try_recv, so a missing record is an error chunk / 500, never a hang. A batched (m>1) turn records nothing yet, so it fails closed until the batched slice. - The traced CUDA loop records none: 501 naming `CUDA (traced)`. - try_cuda_backend's streamed branch moves to cuda_streamed_reply, which keeps it under the cognitive cap. FALSIFY-LOGPROBS-CUDA-4971 drives the real router on tinyllama, direct then scheduled: one entry per completion token, the same reply as without logprobs, the same entries streamed, and the traced 501. No device or no model prints SKIP, which is not_measured. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 587263534db228ef3717b2dc11f6a8b35ac18431) (cherry picked from commit 371678507794e5b229efcd867cc3919bb2bf53e0) (cherry picked from commit 4cda1f82484f6bdc7faf010a3f0663dbca066f63) --- contracts/apr-serve-openai-compat-v1.yaml | 8 +- .../aprender-serve/src/api/chat_logprobs.rs | 109 +++++++--- .../src/api/cuda_batch_scheduler.rs | 46 ++++- .../src/api/cuda_batch_scheduler_tests.rs | 1 + .../src/api/cuda_chat_backend.rs | 189 +++++++++++++---- .../src/api/gpu_completions_handler.rs | 2 + .../aprender-serve/src/api/openai_handlers.rs | 2 +- .../src/api/tests/logprobs_cuda_4971.rs | 191 ++++++++++++++++++ crates/aprender-serve/src/api/tests/mod.rs | 2 + 9 files changed, 476 insertions(+), 74 deletions(-) create mode 100644 crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 1fe485fbd1..d45d11a5da 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -660,7 +660,7 @@ falsification_tests: if_fails: 'Asking for the margin changes the kernels that choose the token. The witness then measures a different run from the one it claims, which is the defect V3 exists to rule out.' - id: FALSIFY-LOGPROBS-WIRE-4971 name: chat_logprobs_are_served_or_refused_by_name - prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. Streamed, the same request answers 200 with the entries on its chunks: their count over the chunks equals the terminal chunk''s usage.completion_tokens, and their tokens and logprobs equal the unstreamed reply''s. A stream without `logprobs` has no `logprobs` key on any chunk. A path with no logprobs answers 501 and names itself: the traced quantized loop (`quantized (traced)`), and every other chat backend but the Qwen3.5 hybrid (FALSIFY-LOGPROBS-QWEN35-4971).' + prediction: 'On a dense CPU server, `logprobs: true, top_logprobs: 3` at temperature 0 answers 200. choices[0].logprobs.content has usage.completion_tokens entries, each with 3 top_logprobs, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob (compared by logprob: tied logits make the token alone ambiguous). `logprobs: true` alone gives entries with empty top_logprobs. The same request without `logprobs` gives the same text and no `logprobs` key. `top_logprobs: 21` or `-1`, and `top_logprobs: 2` without `logprobs: true`, are 4xx and name top_logprobs. Streamed, the same request answers 200 with the entries on its chunks: their count over the chunks equals the terminal chunk''s usage.completion_tokens, and their tokens and logprobs equal the unstreamed reply''s. A stream without `logprobs` has no `logprobs` key on any chunk. A path with no logprobs answers 501 and names itself: the traced quantized loop (`quantized (traced)`), the traced CUDA loop (`CUDA (traced)`), and every other chat backend but the Qwen3.5 hybrid (FALSIFY-LOGPROBS-QWEN35-4971) and the dense CUDA one (FALSIFY-LOGPROBS-CUDA-4971).' test_harness: 'cargo test -p aprender-serve --lib api::tests::logprobs_4971' expected_output: "test result: ok" if_fails: 'A client that asks for logprobs gets a 200 without them (serde ignores unknown keys, which was main''s behaviour before #4971). The perf041 witness then records no margin and S5 can never pass.' @@ -670,3 +670,9 @@ falsification_tests: test_harness: 'cargo test -p aprender-serve --lib qwen35_serve_tests::logprobs_4971 -- --nocapture' expected_output: "test result: ok" if_fails: 'The V3 witness model is served by the Qwen3.5 backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the model it is specified against.' + - id: FALSIFY-LOGPROBS-CUDA-4971 + name: the_dense_cuda_backend_serves_chat_logprobs + prediction: 'With --features cuda, tinyllama Q4_K_M on CUDA device 0 behind the router, first on the direct path and then behind a running batch scheduler (one request at a time, so its m=1 path): `logprobs: true, top_logprobs: 3` at temperature 0 answers 200 with usage.completion_tokens entries, each with 3 top_logprobs, best first, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and completion_tokens and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob, and no chunk is an error. On the direct path a traced request (X-Trace-Level) for logprobs answers 501 naming `CUDA (traced)`. A batched turn (m > 1) records none yet, so a token without its record ends the stream with an error chunk and fails an unstreamed reply with 500; it never answers 200 without them (chat_logprobs tests). No device or no model file prints a SKIP line, which is not_measured and never a pass.' + test_harness: 'cargo test -p aprender-serve --features cuda --lib api::tests::logprobs_cuda_4971 -- --nocapture' + expected_output: "test result: ok" + if_fails: 'apr serve on a CUDA host routes dense chat through this backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the GPU leg.' diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs index 3f3662aef4..0b4d48f760 100644 --- a/crates/aprender-serve/src/api/chat_logprobs.rs +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -161,9 +161,10 @@ impl ChatTokenLogprob { /// #4971: the logprobs of a live stream. The engine sends each token's record /// ahead of the token, so the record is here by the time the stream receives -/// the token. The entries are held until a chunk carries text, and that chunk -/// carries all of them; what is still held when the stream ends goes on its -/// terminal chunk. So every entry is sent once, in the order of the tokens. +/// the token, and taking it never waits. The entries are held until a chunk +/// carries text, and that chunk carries all of them; what is still held when +/// the stream ends goes on its terminal chunk. So every entry is sent once, in +/// the order of the tokens. A finished turn's entries are [`Self::collect`]ed. pub(crate) struct StreamLogprobs { records: tokio::sync::mpsc::UnboundedReceiver, held: Vec, @@ -179,31 +180,47 @@ impl StreamLogprobs { /// Holds the entry of `token`, which the stream has just received. /// + /// It does not wait for the record: one that is not here when its token is + /// was never sent, and waiting would hang a stream whose engine keeps the + /// sender (a batched turn, which records none). + /// /// # Errors /// The engine sent no record for `token`, or the record of another token. /// The stream then ends with an error rather than go on with entries that /// do not match its tokens. - pub(crate) async fn take( - &mut self, - tokenizer: &BPETokenizer, - token: u32, - ) -> Result<(), String> { - match self.records.recv().await { - Some(step) if step.chosen == token => { + pub(crate) fn take(&mut self, tokenizer: &BPETokenizer, token: u32) -> Result<(), String> { + match self.records.try_recv() { + Ok(step) if step.chosen == token => { self.held .push(ChatTokenLogprob::from_step(tokenizer, &step)); Ok(()) }, - Some(step) => Err(format!( + Ok(step) => Err(format!( "token {token} arrived with the logprobs of token {} (#4971)", step.chosen )), - None => Err(format!( + Err(_) => Err(format!( "token {token} arrived without its logprobs (#4971)" )), } } + /// The entries of a finished turn's reply `tokens`, whose records were all + /// sent ahead of them. + /// + /// # Errors + /// As [`Self::take`], for the first token without its own record. + pub(crate) fn collect( + mut self, + tokenizer: &BPETokenizer, + tokens: &[u32], + ) -> Result { + for &token in tokens { + self.take(tokenizer, token)?; + } + Ok(ChatLogprobs { content: self.held }) + } + /// The entries held until now, for the chunk about to be sent, or `None` /// when none are held. pub(crate) fn release(&mut self) -> Option { @@ -276,17 +293,19 @@ pub(crate) fn logprobs_refusal(request: &ChatCompletionRequest, backend: &str) - }) } -/// [`logprobs_refusal`] for the quantized CPU backend, which computes them on -/// its dense turn, streamed or not: only its traced loop refuses. +/// [`logprobs_refusal`] for a backend that computes them on its dense turn, +/// streamed or not (the quantized CPU and dense CUDA backends): only its +/// traced loop refuses, as `{backend} (traced)`. #[must_use] -pub(crate) fn quantized_logprobs_refusal( +pub(crate) fn traced_logprobs_refusal( request: &ChatCompletionRequest, + backend: &str, traced: bool, ) -> Option { if !traced { return None; } - logprobs_refusal(request, "quantized (traced)") + logprobs_refusal(request, &format!("{backend} (traced)")) } #[cfg(test)] @@ -473,29 +492,63 @@ mod tests { } #[test] - fn the_quantized_backend_refuses_only_its_traced_loop() { + fn a_dense_backend_refuses_only_its_traced_loop() { let mut asks = request(r#","logprobs":true"#).expect("valid"); - // (stream, traced, refused as) - let table: &[(bool, bool, Option<&str>)] = &[ - (false, false, None), - (true, false, None), - (false, true, Some("quantized (traced)")), - (true, true, Some("quantized (traced)")), + // (backend, stream, traced, refused as) + let table: &[(&str, bool, bool, Option<&str>)] = &[ + ("quantized", false, false, None), + ("quantized", true, false, None), + ("quantized", false, true, Some("quantized (traced)")), + ("quantized", true, true, Some("quantized (traced)")), + ("CUDA", false, false, None), + ("CUDA", true, false, None), + ("CUDA", false, true, Some("CUDA (traced)")), + ("CUDA", true, true, Some("CUDA (traced)")), ]; - for &(stream, traced, want) in table { + for &(backend, stream, traced, want) in table { asks.stream = stream; - let got = quantized_logprobs_refusal(&asks, traced); + let got = traced_logprobs_refusal(&asks, backend, traced); match want { - None => assert_eq!(got, None, "stream={stream} traced={traced}"), + None => assert_eq!(got, None, "{backend} stream={stream} traced={traced}"), Some(path) => assert!( got.as_deref().is_some_and(|r| r.contains(path)), - "stream={stream} traced={traced}: {got:?}" + "{backend} stream={stream} traced={traced}: {got:?}" ), } } let mut plain = request("").expect("valid"); plain.stream = true; - assert_eq!(quantized_logprobs_refusal(&plain, true), None); + assert_eq!(traced_logprobs_refusal(&plain, "CUDA", true), None); + } + + #[test] + fn a_finished_turn_collects_one_entry_per_token_from_records_sent_ahead() { + let (records, logprobs) = StreamLogprobs::channel(); + for (chosen, logits) in [(1, [0.0, 2.0, 1.0]), (2, [0.0, 1.0, 3.0])] { + records.send(step(chosen, &logits, 2)).expect("open"); + } + let lp = logprobs.collect(&tokenizer(), &[1, 2]).expect("collected"); + assert_eq!(lp.content.len(), 2); + assert!(lp.content.iter().all(|e| e.top_logprobs.len() == 2)); + // The sender is still held: collect read what was sent and did not wait. + drop(records); + } + + #[test] + fn a_token_without_its_own_record_is_an_error_and_never_a_wait() { + // The engine keeps its sender and sends nothing, as a batched turn + // would: the first token is refused at once. + let (records, logprobs) = StreamLogprobs::channel(); + let err = logprobs.collect(&tokenizer(), &[1]).expect_err("no record"); + assert!(err.contains("without its logprobs"), "{err}"); + drop(records); + let (other, mut held) = StreamLogprobs::channel(); + other.send(step(2, &[0.0, 1.0, 3.0], 1)).expect("open"); + let err = held + .take(&tokenizer(), 1) + .expect_err("another token's record"); + assert!(err.contains("logprobs of token 2"), "{err}"); + assert_eq!(held.release(), None, "a refused entry is never held"); } #[test] diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs index e7c8312466..f2fdf2e2ed 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs @@ -30,6 +30,43 @@ pub struct CudaBatchRequest { /// phase split to report and inventing one would put a fabricated numerator /// into a gated ratio. pub timing_tx: Option>, + /// #4971: `Some` when the request asked for logprobs. The single-request + /// path sends each token's record here ahead of the token. The batched + /// path records none, so the handler meets a token without its record and + /// fails the request: never a reply without what it asked for. + pub logprobs: Option, +} + +/// #4971: where a request that asked for logprobs gets each token's record, +/// sent ahead of the token, with the `top_n` most likely tokens of its step. +#[cfg(feature = "cuda")] +pub struct RecordSink { + /// How many of each step's best tokens a record carries. + pub top_n: usize, + /// The handler's end, a [`StreamLogprobs`](crate::api::chat_logprobs::StreamLogprobs). + pub records: tokio::sync::mpsc::UnboundedSender, +} + +/// One dense turn of `session`, as [`dense_stream`](crate::gguf::dense_session::dense_stream), +/// and with `logprobs` (#4971) each token's record sent ahead of the token. A +/// record that cannot be sent means the handler is gone, which `on_token` +/// sees too. +#[cfg(feature = "cuda")] +pub(crate) fn dense_stream_recording( + session: &mut crate::session::Session, + prompt: &[u32], + config: &QuantizedGenerateConfig, + logprobs: Option<&RecordSink>, + on_token: &mut dyn FnMut(u32) -> bool, +) -> crate::error::Result<(Vec, bool)> { + use crate::gguf::dense_session::{dense_stream, dense_stream_with_logprobs}; + let Some(sink) = logprobs else { + return dense_stream(session, prompt, config, on_token); + }; + dense_stream_with_logprobs(session, prompt, config, sink.top_n, &mut |token, record| { + let _ = sink.records.send(record); + on_token(token) + }) } /// PMAT-044: Batch scheduler configuration @@ -129,11 +166,12 @@ fn generate_single_request_inner(cuda_model: &mut OwnedQuantizedModelCuda, req: crate::gguf::dense_session_borrowed::BorrowedCudaForward::new(cuda_model), ); // `dense_stream` never hands on the stop token that ends the turn, as the - // pre-port loop checked it before emitting. - let stream = crate::gguf::dense_session::dense_stream; + // pre-port loop checked it before emitting; with logprobs (#4971) each + // token's record goes to the handler ahead of the token. + let (prompt, config, logprobs) = (&req.prompt_ids, &req.config, req.logprobs.as_ref()); if req.non_streaming { let mut tokens = Vec::new(); - let result = stream(&mut session, &req.prompt_ids, &req.config, &mut |tid| { + let result = dense_stream_recording(&mut session, prompt, config, logprobs, &mut |tid| { tokens.push(tid); true }); @@ -150,7 +188,7 @@ fn generate_single_request_inner(cuda_model: &mut OwnedQuantizedModelCuda, req: }, } } else { - let result = stream(&mut session, &req.prompt_ids, &req.config, &mut |tid| { + let result = dense_stream_recording(&mut session, prompt, config, logprobs, &mut |tid| { req.token_tx.try_send(Ok(tid)).is_ok() }); if let Err(e) = result { diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler_tests.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler_tests.rs index 0d9d7e8890..e118cd65d9 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler_tests.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler_tests.rs @@ -46,6 +46,7 @@ fn scheduled( non_streaming, enqueue_time: std::time::Instant::now(), timing_tx: None, + logprobs: None, }, ); let mut out = Vec::new(); diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 1eaeae260f..aca8c24f32 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -113,7 +113,7 @@ async fn try_cuda_backend( let t0 = if ttft_trace { Some(std::time::Instant::now()) } else { None }; let cuda_model_lock = state.cuda_model()?; - let tokenizer = match require_tokenizer_refusing_logprobs(state, request, "CUDA") { + let tokenizer = match require_tokenizer(state) { Ok(t) => t, Err(r) => return Some(r), }; @@ -142,49 +142,61 @@ async fn try_cuda_backend( state.should_trace(trace_level), cancel, ); + // #4971: logprobs come from the dense turn; the traced loop records none. + if let Some(reason) = + super::chat_logprobs::traced_logprobs_refusal(request, "CUDA", q_config.trace) + { + return Some(fail_response(state, StatusCode::NOT_IMPLEMENTED, reason)); + } let max_tokens = q_config.max_tokens; + let (sink, logprobs) = cuda_record_channel(request.logprobs_top_n()); if request.stream { - let (tx, rx) = tokio::sync::mpsc::channel::>(16); - // §3: the engine's return path for the server-measured phase split. - // The handler returns the SSE response before generation finishes, so - // the measurement cannot be a return value; the terminal chunk awaits - // this oneshot after the token channel closes. - let (timing_tx, timing_rx) = - tokio::sync::oneshot::channel::(); - - if let Err(r) = - dispatch_cuda_stream(state, cuda_model_lock, prompt_ids, q_config, tx, timing_tx) - { - return Some(r); - } - - return Some(true_streaming_sse_response( - rx, - tokenizer, - request_id.to_string(), - request.model.clone(), - state.metrics.clone(), + return Some(cuda_streamed_reply( + state, + request, + request_id, start, - max_tokens, - prompt_tokens, - Some(timing_rx), - request.stop.as_deref(), - crate::api::stream_tool_calls::StreamTools::from_request(request), + cuda_model_lock, + prompt_ids, + q_config, + tokenizer, + sink, + logprobs, )); } // Non-streaming CUDA — route through batch scheduler when available (realizr#211) let (timing_tx, timing_rx) = tokio::sync::oneshot::channel::(); let generated = if let Some(batch_tx) = state.cuda_batch_tx() { - cuda_batch_collect(state, batch_tx, prompt_ids, q_config, timing_tx, &tokenizer).await + cuda_batch_collect( + state, batch_tx, prompt_ids, q_config, timing_tx, &tokenizer, sink, + ) + .await } else { - cuda_direct_collect(state, cuda_model_lock, &prompt_ids, &q_config, timing_tx, &tokenizer, prompt_tokens) + cuda_direct_collect( + state, + cuda_model_lock, + &prompt_ids, + &q_config, + timing_tx, + &tokenizer, + sink.as_ref(), + ) }; let (token_ids, completion_tokens, response_text) = match generated { Ok(g) => g, Err(r) => return Some(r), }; + // #4971: every record was sent ahead of its token, so all of them are here. + // A token without its own (a batched turn records none) fails the request. + let logprobs = match logprobs + .map(|l| l.collect(&tokenizer, &token_ids)) + .transpose() + { + Ok(l) => l, + Err(e) => return Some(fail_response(state, StatusCode::INTERNAL_SERVER_ERROR, e)), + }; let latency = start.elapsed(); state.metrics.record_success(completion_tokens, latency); @@ -207,13 +219,86 @@ async fn try_cuda_backend( request_tool_choice(request), timings, None, - None, + logprobs, )) } +/// `try_cuda_backend`'s streamed reply: dispatch the turn, then hand its +/// token channel to the SSE stream. A refused dispatch is the reply. +#[cfg(feature = "cuda")] +fn cuda_streamed_reply( + state: &AppState, + request: &ChatCompletionRequest, + request_id: &str, + start: Instant, + cuda_model_lock: &Arc>, + prompt_ids: Vec, + q_config: crate::gguf::QuantizedGenerateConfig, + tokenizer: Arc, + sink: Option, + logprobs: Option, +) -> Response { + let max_tokens = q_config.max_tokens; + let prompt_tokens = prompt_ids.len(); + let (tx, rx) = tokio::sync::mpsc::channel::>(16); + // §3: the engine's return path for the server-measured phase split. + // The handler returns the SSE response before generation finishes, so + // the measurement cannot be a return value; the terminal chunk awaits + // this oneshot after the token channel closes. + let (timing_tx, timing_rx) = tokio::sync::oneshot::channel::(); + + if let Err(r) = dispatch_cuda_stream( + state, + cuda_model_lock, + prompt_ids, + q_config, + tx, + timing_tx, + sink, + ) { + return r; + } + + live_sse_response( + rx, + tokenizer, + request_id.to_string(), + request.model.clone(), + state.metrics.clone(), + start, + max_tokens, + prompt_tokens, + Some(timing_rx), + request.stop.as_deref(), + crate::api::stream_tool_calls::StreamTools::from_request(request), + logprobs, + ) +} + +/// #4971: the engine's and the handler's ends of a request's logprobs records, +/// when it asked for them with `top_n` alternatives per token. +#[cfg(feature = "cuda")] +fn cuda_record_channel( + top_n: Option, +) -> ( + Option, + Option, +) { + top_n + .map(|top_n| { + let (records, logprobs) = super::chat_logprobs::StreamLogprobs::channel(); + ( + super::cuda_batch_scheduler::RecordSink { top_n, records }, + logprobs, + ) + }) + .unzip() +} + /// `try_cuda_backend`'s streaming dispatch: the batch scheduler when one is /// running, else the direct RwLock path on a blocking task. `Err` is the 503 -/// for a full batch queue. +/// for a full batch queue. With `logprobs` (#4971) each token's record goes +/// ahead of the token. #[cfg(feature = "cuda")] #[allow(clippy::result_large_err)] fn dispatch_cuda_stream( @@ -223,6 +308,7 @@ fn dispatch_cuda_stream( q_config: crate::gguf::QuantizedGenerateConfig, tx: tokio::sync::mpsc::Sender>, timing_tx: tokio::sync::oneshot::Sender, + logprobs: Option, ) -> Result<(), Response> { // PMAT-044: Use batch scheduler if available (continuous batching) if let Some(batch_tx) = state.cuda_batch_tx() { @@ -233,6 +319,7 @@ fn dispatch_cuda_stream( non_streaming: false, enqueue_time: std::time::Instant::now(), timing_tx: Some(timing_tx), + logprobs, }; if let Err(e) = batch_tx.try_send(batch_req) { // §5.2: this 503 is the one admission REFUSAL this server has @@ -256,7 +343,13 @@ fn dispatch_cuda_stream( // Stops when the client goes away — see `streaming_token_sink`. let sink = crate::api::openai_handlers::streaming_token_sink(tx.clone(), sink_metrics); - let result = dense_cuda_turn(&mut cuda_model, &prompt_ids, &q_config, sink); + let result = dense_cuda_turn( + &mut cuda_model, + &prompt_ids, + &q_config, + logprobs.as_ref(), + sink, + ); // Taken under the SAME write lock the request ran under, so the // split belongs to this request and to no other. let _ = timing_tx.send(phase_split(&mut cuda_model, generate_start)); @@ -270,7 +363,8 @@ fn dispatch_cuda_stream( /// `try_cuda_backend`'s non-streaming turn through the batch scheduler /// (realizr#211): submit, then collect every token. Returns -/// `(tokens, completion_tokens, cleaned text)`. +/// `(tokens, completion_tokens, cleaned text)`. With `logprobs` (#4971) each +/// token's record goes ahead of the token. #[cfg(feature = "cuda")] async fn cuda_batch_collect( state: &AppState, @@ -279,6 +373,7 @@ async fn cuda_batch_collect( q_config: crate::gguf::QuantizedGenerateConfig, timing_tx: tokio::sync::oneshot::Sender, tokenizer: &BPETokenizer, + logprobs: Option, ) -> Result<(Vec, usize, String), Response> { // Use batch scheduler: submit request and collect all tokens // realizr#212: capacity 512 for bulk-send after non-streaming generation @@ -290,6 +385,7 @@ async fn cuda_batch_collect( non_streaming: true, // realizr#212: scheduler accumulates + bulk-sends enqueue_time: std::time::Instant::now(), timing_tx: Some(timing_tx), + logprobs, }; if let Err(e) = batch_tx.try_send(batch_req) { // §5.2: counted at the refusal, as in `dispatch_cuda_stream`. @@ -317,7 +413,8 @@ async fn cuda_batch_collect( } /// `try_cuda_backend`'s non-streaming turn when no batch scheduler runs: the -/// direct RwLock path (serialized). Same return shape as `cuda_batch_collect`. +/// direct RwLock path (serialized). Same return shape and `logprobs` as +/// `cuda_batch_collect`. #[cfg(feature = "cuda")] #[allow(clippy::result_large_err)] fn cuda_direct_collect( @@ -327,11 +424,13 @@ fn cuda_direct_collect( q_config: &crate::gguf::QuantizedGenerateConfig, timing_tx: tokio::sync::oneshot::Sender, tokenizer: &BPETokenizer, - prompt_tokens: usize, + logprobs: Option<&super::cuda_batch_scheduler::RecordSink>, ) -> Result<(Vec, usize, String), Response> { + let prompt_tokens = prompt_ids.len(); let mut cuda_model = cuda_model_lock.write().expect("operation failed"); let generate_start = std::time::Instant::now(); - let generated = match dense_cuda_turn(&mut cuda_model, prompt_ids, q_config, |_| true) { + let generated = match dense_cuda_turn(&mut cuda_model, prompt_ids, q_config, logprobs, |_| true) + { Ok(g) => g, Err(e) => return Err(fail_response(state, crate::api::generation_error_status(&e), e)), }; @@ -352,13 +451,16 @@ fn cuda_direct_collect( /// #4268: one serve request's dense CUDA turn, run on the one engine over the /// model borrowed under the request's write lock. A GPU failure is an error, /// as it always was here: serve has no CPU copy to fall back to. `--trace` -/// keeps the instrumented loop. Stale phase timings are cleared first so -/// [`phase_split`] reads this request's. +/// keeps the instrumented loop, which records no logprobs (a traced request +/// for them is refused before this). Stale phase timings are cleared first so +/// [`phase_split`] reads this request's. With `logprobs` (#4971) each token's +/// record goes ahead of the token. #[cfg(feature = "cuda")] fn dense_cuda_turn( cuda_model: &mut crate::gguf::OwnedQuantizedModelCuda, prompt: &[u32], config: &crate::gguf::QuantizedGenerateConfig, + logprobs: Option<&super::cuda_batch_scheduler::RecordSink>, mut on_token: impl FnMut(u32) -> bool, ) -> crate::error::Result> { if config.trace { @@ -368,8 +470,14 @@ fn dense_cuda_turn( let mut session = crate::session::Session::new( crate::gguf::dense_session_borrowed::BorrowedCudaForward::new(cuda_model), ); - crate::gguf::dense_session::dense_stream(&mut session, prompt, config, &mut on_token) - .map(|(tokens, _)| tokens) + super::cuda_batch_scheduler::dense_stream_recording( + &mut session, + prompt, + config, + logprobs, + &mut on_token, + ) + .map(|(tokens, _)| tokens) } #[cfg(feature = "cuda")] @@ -509,7 +617,8 @@ fn try_quantized_backend( cancel, ); // #4971: logprobs come from the dense turn; the traced loop records none. - if let Some(reason) = super::chat_logprobs::quantized_logprobs_refusal(request, q_config.trace) + if let Some(reason) = + super::chat_logprobs::traced_logprobs_refusal(request, "quantized", q_config.trace) { return Some(fail_response(state, StatusCode::NOT_IMPLEMENTED, reason)); } diff --git a/crates/aprender-serve/src/api/gpu_completions_handler.rs b/crates/aprender-serve/src/api/gpu_completions_handler.rs index 14c1ad0cc1..42531fe80a 100644 --- a/crates/aprender-serve/src/api/gpu_completions_handler.rs +++ b/crates/aprender-serve/src/api/gpu_completions_handler.rs @@ -419,6 +419,8 @@ async fn try_cuda_gguf_completions( // so there is nothing for the engine's measurement to reach. Dropped // rather than measured-and-discarded. timing_tx: None, + // #4971: `/v1/completions` serves no logprobs. + logprobs: None, }; batch_tx.try_send(batch_req).map_err(|_| { diff --git a/crates/aprender-serve/src/api/openai_handlers.rs b/crates/aprender-serve/src/api/openai_handlers.rs index dae84653d2..56c1482002 100644 --- a/crates/aprender-serve/src/api/openai_handlers.rs +++ b/crates/aprender-serve/src/api/openai_handlers.rs @@ -1170,7 +1170,7 @@ pub(crate) fn live_sse_response( // #4971: the token's entry is held for the chunk that // releases its text, and a token without one ends the stream. let taken = match logprobs.as_mut() { - Some(held) => held.take(&tokenizer, token_id).await, + Some(held) => held.take(&tokenizer, token_id), None => Ok(()), }; if let Err(e) = taken { diff --git a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs new file mode 100644 index 0000000000..d97dc184f9 --- /dev/null +++ b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs @@ -0,0 +1,191 @@ +//! FALSIFY-LOGPROBS-CUDA-4971: chat `logprobs` on the dense CUDA backend +//! (ASOC-INV-021), through the real router, on the direct path and through the +//! batch scheduler (one request at a time, so its m=1 path). +//! +//! Before this the CUDA backend answered a request for logprobs with 501. It +//! needs a CUDA device and tinyllama; without either the test prints SKIP, +//! which is not_measured, never a pass. + +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use serde_json::Value; +use tower::util::ServiceExt; + +use super::native_routes_2376::{body_string, post}; +use crate::api::{create_router, AppState}; + +const CHAT: &str = "/v1/chat/completions"; + +const TINYLLAMA: [&str; 2] = [ + "/home/noah/.apr/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", + "/mnt/nvme-raid0/cache/apr-home/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", +]; + +/// tinyllama on CUDA device 0 behind the direct path, or `None` (and a SKIP +/// line) when the model or the device is missing. +fn cuda_state() -> Option { + use crate::gguf::{MappedGGUFModel, OwnedQuantizedModel, OwnedQuantizedModelCuda}; + let Some(path) = TINYLLAMA + .iter() + .map(std::path::Path::new) + .find(|p| p.exists()) + else { + eprintln!("SKIP: tinyllama not present"); + return None; + }; + let mapped = MappedGGUFModel::from_path(path).expect("map tinyllama"); + let vocab = mapped + .model + .vocabulary() + .expect("tinyllama has a vocabulary"); + let model = OwnedQuantizedModel::from_mapped(&mapped).expect("load tinyllama"); + let cuda = match OwnedQuantizedModelCuda::new(model, 0) { + Ok(cuda) => cuda, + Err(e) => { + eprintln!("SKIP: no CUDA device: {e}"); + return None; + }, + }; + Some(AppState::with_cuda_model_and_vocab(cuda, vocab).expect("CUDA state")) +} + +/// The same model behind a running batch scheduler, as `apr serve` builds it. +fn scheduled(state: &AppState) -> AppState { + use crate::api::cuda_batch_scheduler::{spawn_cuda_batch_scheduler, CudaBatchConfig}; + let model = state.cuda_model().expect("a CUDA state").clone(); + let tx = spawn_cuda_batch_scheduler( + model, + CudaBatchConfig::default(), + crate::api::InFlightCounter::new(), + ); + state.clone().with_cuda_batch_tx(tx) +} + +fn chat_body(extra: &str) -> String { + format!( + r#"{{"model":"default","messages":[{{"role":"user","content":"The capital of France is"}}],"max_tokens":8,"temperature":0{extra}}}"# + ) +} + +fn logprob(entry: &Value) -> f64 { + entry["logprob"].as_f64().expect("logprob is a number") +} + +async fn chat_json(state: &AppState, extra: &str) -> Value { + let (status, body) = post(state.clone(), CHAT, &chat_body(extra)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + serde_json::from_str(&body).expect("chat reply is JSON") +} + +/// The `logprobs.content` entries of a streamed reply, in order, and its +/// `usage.completion_tokens`. +async fn streamed_entries(state: &AppState, extra: &str) -> (Vec, u64) { + let body = chat_body(&format!(r#","stream":true{extra}"#)); + let (status, body) = post(state.clone(), CHAT, &body).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let chunks: Vec = body + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|data| *data != "[DONE]") + .map(|data| serde_json::from_str(data).expect("an SSE chunk is JSON")) + .collect(); + assert!( + chunks.iter().all(|c| c.get("error").is_none()), + "the stream carried an error: {body}" + ); + let completion_tokens = chunks + .iter() + .find_map(|c| c["usage"]["completion_tokens"].as_u64()) + .expect("the terminal chunk carries usage"); + let entries = chunks + .iter() + .filter_map(|c| c["choices"][0]["logprobs"]["content"].as_array()) + .flatten() + .cloned() + .collect(); + (entries, completion_tokens) +} + +/// One entry per completion token, each with its best alternatives, the +/// chosen one a best one (greedy); the same reply as without logprobs; and the +/// same entries streamed. +async fn logprobs_are_served(state: &AppState, path: &str) { + let reply = chat_json(state, r#","logprobs":true,"top_logprobs":3"#).await; + let completion_tokens = reply["usage"]["completion_tokens"] + .as_u64() + .expect("usage.completion_tokens"); + let content = reply["choices"][0]["logprobs"]["content"] + .as_array() + .unwrap_or_else(|| panic!("{path}: no choices[0].logprobs.content: {reply}")); + assert!(completion_tokens > 0, "{path}: generated nothing: {reply}"); + assert_eq!(content.len() as u64, completion_tokens, "{path}: {reply}"); + for entry in content { + let tops = entry["top_logprobs"].as_array().expect("top_logprobs"); + assert_eq!(tops.len(), 3, "{path}: {entry}"); + assert!(logprob(entry) <= 0.0, "{path}: {entry}"); + assert!( + tops.windows(2).all(|w| logprob(&w[0]) >= logprob(&w[1])), + "{path}: {entry}" + ); + assert!( + (logprob(entry) - logprob(&tops[0])).abs() < 1e-4, + "{path}: {entry}" + ); + } + + let without = chat_json(state, "").await; + assert_eq!( + reply["choices"][0]["message"]["content"], without["choices"][0]["message"]["content"], + "{path}: asking for logprobs changed the reply" + ); + assert_eq!( + reply["usage"]["completion_tokens"], without["usage"]["completion_tokens"], + "{path}" + ); + let choice = without["choices"][0].as_object().expect("choice object"); + assert!(!choice.contains_key("logprobs"), "{path}: {without}"); + + let (streamed, streamed_tokens) = + streamed_entries(state, r#","logprobs":true,"top_logprobs":3"#).await; + assert_eq!( + streamed.len() as u64, + streamed_tokens, + "{path}: {streamed:?}" + ); + assert_eq!(streamed.len(), content.len(), "{path}: {streamed:?}"); + for (s, u) in streamed.iter().zip(content) { + assert_eq!(s["token"], u["token"], "{path}: {s} vs {u}"); + assert!((logprob(s) - logprob(u)).abs() < 1e-3, "{path}: {s} vs {u}"); + } +} + +async fn traced_logprobs_are_refused(state: &AppState) { + let request = Request::builder() + .method("POST") + .uri(CHAT) + .header("content-type", "application/json") + .header("X-Trace-Level", "brick") + .body(Body::from(chat_body(r#","logprobs":true"#))) + .expect("build request"); + let response = create_router(state.clone()) + .oneshot(request) + .await + .expect("dispatch"); + let status = response.status(); + let body = body_string(response).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{body}"); + assert!(body.contains("CUDA (traced)"), "{body}"); +} + +/// One test, so one model on the device and one request at a time on it. +#[tokio::test] +async fn the_dense_cuda_backend_serves_chat_logprobs_direct_and_scheduled() { + let Some(direct) = cuda_state() else { + return; + }; + logprobs_are_served(&direct, "direct").await; + traced_logprobs_are_refused(&direct).await; + logprobs_are_served(&scheduled(&direct), "scheduled").await; +} diff --git a/crates/aprender-serve/src/api/tests/mod.rs b/crates/aprender-serve/src/api/tests/mod.rs index 416abd7ad9..57427d9c10 100644 --- a/crates/aprender-serve/src/api/tests/mod.rs +++ b/crates/aprender-serve/src/api/tests/mod.rs @@ -80,3 +80,5 @@ mod completions_eog_stop_4345; // aprender#4345: the wgpu GpuModel /v1/completio mod stream_tool_calls_4918; // aprender#4918: streamed tool calls arrive as delta.tool_calls, finish tool_calls (T1-T12) #[cfg(feature = "gpu")] // create_test_quantized_model is gpu-gated mod logprobs_4971; // aprender#4971: chat logprobs/top_logprobs honoured on the quantized CPU path, refused 4xx/501 elsewhere (ASOC-INV-021) +#[cfg(feature = "cuda")] +mod logprobs_cuda_4971; // aprender#4971: chat logprobs on the dense CUDA backend, direct and through the batch scheduler (needs a device + tinyllama; SKIP = not_measured) From 42d739ea4fdd2d340ba784d9f5667ab440a3bdc2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:05:34 +0200 Subject: [PATCH 09/35] feat(serve): chat logprobs on a batched CUDA turn, m > 1 (PMAT-4971) The batch scheduler's batched path recorded no logprobs, so a request that asked for them failed whenever it shared a batch. Each slot now carries an optional recorder. The decode step records a live slot's logprobs from the same logits its token came from: the sampled path from the logits it downloads, the device-argmax path from logits_buf, downloaded after the argmax so the token stays the device's. A recording step runs eager, since BATCHED_GRAPH replays into its own buffers. A record reaches the handler before its token; a stop token's record is dropped. A joined or recycled slot records nothing until the scheduler gives it its request's recorder. FALSIFY-LOGPROBS-CUDA-4971 gains a pair decoded as one m=2 batch, shown by the scheduler's in-flight peak: one slot with logprobs and one without (same text, entries only where asked), then both asking, one streamed (equal entries). Only the iteration scheduler still records none. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 943a091d68bf2af7d53e1146334337647c0a2f10) (cherry picked from commit 294cf0d6a62d1d4210d6fc5acc06d636adc163bf) (cherry picked from commit 2a70ee237f99ae64c790c951efdfab447e35a44c) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- .../aprender-serve/src/api/chat_logprobs.rs | 2 +- .../src/api/cuda_batch_scheduler.rs | 37 +++- .../src/api/cuda_chat_backend.rs | 2 +- .../src/api/tests/logprobs_cuda_4971.rs | 98 ++++++++-- .../cuda/executor/layers/batched_forward.rs | 25 +++ .../gguf/cuda/generate_batched_streaming.rs | 183 ++++++++++++++---- crates/aprender-serve/src/gguf/cuda/mod.rs | 2 +- crates/aprender-serve/src/gguf/mod.rs | 2 +- 9 files changed, 288 insertions(+), 65 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index d45d11a5da..34f3e376a0 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -672,7 +672,7 @@ falsification_tests: if_fails: 'The V3 witness model is served by the Qwen3.5 backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the model it is specified against.' - id: FALSIFY-LOGPROBS-CUDA-4971 name: the_dense_cuda_backend_serves_chat_logprobs - prediction: 'With --features cuda, tinyllama Q4_K_M on CUDA device 0 behind the router, first on the direct path and then behind a running batch scheduler (one request at a time, so its m=1 path): `logprobs: true, top_logprobs: 3` at temperature 0 answers 200 with usage.completion_tokens entries, each with 3 top_logprobs, best first, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and completion_tokens and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob, and no chunk is an error. On the direct path a traced request (X-Trace-Level) for logprobs answers 501 naming `CUDA (traced)`. A batched turn (m > 1) records none yet, so a token without its record ends the stream with an error chunk and fails an unstreamed reply with 500; it never answers 200 without them (chat_logprobs tests). No device or no model file prints a SKIP line, which is not_measured and never a pass.' + prediction: 'With --features cuda, tinyllama Q4_K_M on CUDA device 0 behind the router, first on the direct path, then behind a running batch scheduler one request at a time (its m=1 path), then two at once in one batch: `logprobs: true, top_logprobs: 3` at temperature 0 answers 200 with usage.completion_tokens entries, each with 3 top_logprobs, best first, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and completion_tokens and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob, and no chunk is an error. On the direct path a traced request (X-Trace-Level) for logprobs answers 501 naming `CUDA (traced)`. Behind a scheduler that holds a request until a second fills a batch of two, the scheduler''s in-flight peak is 2, so the pair decoded as one m=2 batch. Of a pair where one asks for logprobs and one does not, the one asking gets the same well-formed entries, and both get the same text and completion_tokens, with no `logprobs` key on the one not asking. Of a pair that both ask, one streamed, the streamed entries equal the unstreamed ones and the first pair''s, token and logprob. A turn that records none (the iteration scheduler) never answers 200 without them: a token without its record ends the stream with an error chunk and fails an unstreamed reply with 500 (chat_logprobs tests). No device or no model file prints a SKIP line, which is not_measured and never a pass.' test_harness: 'cargo test -p aprender-serve --features cuda --lib api::tests::logprobs_cuda_4971 -- --nocapture' expected_output: "test result: ok" if_fails: 'apr serve on a CUDA host routes dense chat through this backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the GPU leg.' diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs index 0b4d48f760..1fa257642e 100644 --- a/crates/aprender-serve/src/api/chat_logprobs.rs +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -182,7 +182,7 @@ impl StreamLogprobs { /// /// It does not wait for the record: one that is not here when its token is /// was never sent, and waiting would hang a stream whose engine keeps the - /// sender (a batched turn, which records none). + /// sender (a turn of the iteration scheduler, which records none). /// /// # Errors /// The engine sent no record for `token`, or the record of another token. diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs index f2fdf2e2ed..0c53e5447b 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs @@ -30,10 +30,10 @@ pub struct CudaBatchRequest { /// phase split to report and inventing one would put a fabricated numerator /// into a gated ratio. pub timing_tx: Option>, - /// #4971: `Some` when the request asked for logprobs. The single-request - /// path sends each token's record here ahead of the token. The batched - /// path records none, so the handler meets a token without its record and - /// fails the request: never a reply without what it asked for. + /// #4971: `Some` when the request asked for logprobs. Each token's record + /// is sent here ahead of the token, by the single-request path and by a + /// batched slot alike. A token the handler meets without its record fails + /// the request: never a reply without what it asked for. pub logprobs: Option, } @@ -435,6 +435,19 @@ fn token_callback(token_tx: TokenSender) -> TokenCallback { Box::new(move |token_id: u32| -> bool { token_tx.try_send(Ok(token_id)).is_ok() }) } +/// #4971: a batched slot's recorder, which sends each record to the handler +/// ahead of the token `token_callback` sends. +#[cfg(feature = "cuda")] +fn slot_recorder(sink: RecordSink) -> crate::gguf::SlotRecorder { + let RecordSink { top_n, records } = sink; + crate::gguf::SlotRecorder { + top_n, + on_record: Box::new(move |record| { + let _ = records.send(record); + }), + } +} + #[cfg(feature = "cuda")] fn process_cuda_batch( model: &Arc>, @@ -487,6 +500,7 @@ fn process_cuda_batch( configs, mut error_senders, callbacks, + recorders, mut pending_joins, } = phase1_inputs(batch, staggered); @@ -505,6 +519,8 @@ fn process_cuda_batch( ) else { return; }; + // #4971: one per prompt prefilled, in its slot + state.recorders = recorders; // Phase 2: Decode loop with mid-batch joins (PMAT-073/099) and slot recycling (PMAT-074) // Lock per step (~19ms per acquire vs ~660ms total). @@ -563,11 +579,13 @@ struct Phase1 { configs: Vec, error_senders: Vec, callbacks: Vec, + /// #4971: each prompt's recorder, `None` when it did not ask for logprobs + recorders: Vec>, pending_joins: std::collections::VecDeque, } #[cfg(feature = "cuda")] -fn phase1_inputs(batch: Vec, staggered: bool) -> Phase1 { +fn phase1_inputs(mut batch: Vec, staggered: bool) -> Phase1 { if staggered { // Split batch: first → immediate prefill, rest → pending joins let mut batch_iter = batch.into_iter(); @@ -584,6 +602,7 @@ fn phase1_inputs(batch: Vec, staggered: bool) -> Phase1 { configs: vec![first_req.config.clone()], error_senders: vec![first_req.token_tx.clone()], callbacks: vec![token_callback(first_req.token_tx)], + recorders: vec![first_req.logprobs.map(slot_recorder)], pending_joins, } } else { @@ -592,6 +611,10 @@ fn phase1_inputs(batch: Vec, staggered: bool) -> Phase1 { prompts: batch.iter().map(|r| r.prompt_ids.clone()).collect(), configs: batch.iter().map(|r| r.config.clone()).collect(), error_senders: batch.iter().map(|r| r.token_tx.clone()).collect(), + recorders: batch + .iter_mut() + .map(|r| r.logprobs.take().map(slot_recorder)) + .collect(), callbacks: batch .into_iter() .map(|req| token_callback(req.token_tx)) @@ -720,9 +743,11 @@ fn join_slot( ) { let error_tx = req.token_tx.clone(); let on_token = token_callback(req.token_tx); + let recorder = req.logprobs.map(slot_recorder); match cuda_model.add_slot_to_batch(state, req.prompt_ids, req.config, on_token) { Ok(()) => { error_senders.push(error_tx); + state.recorders[state.m - 1] = recorder; }, Err(e) => { eprintln!("{failed}: {e}"); @@ -751,9 +776,11 @@ fn recycle_done_slots( }; let error_tx = req.token_tx.clone(); let on_token = token_callback(req.token_tx); + let recorder = req.logprobs.map(slot_recorder); match cuda_model.recycle_slot(state, slot_idx, req.prompt_ids, req.config, on_token) { Ok(()) => { error_senders[slot_idx] = error_tx; + state.recorders[slot_idx] = recorder; }, Err(e) => { eprintln!("[PMAT-074] Slot recycle FAILED (slot {slot_idx}): {e}"); diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index aca8c24f32..2e17791b38 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -189,7 +189,7 @@ async fn try_cuda_backend( Err(r) => return Some(r), }; // #4971: every record was sent ahead of its token, so all of them are here. - // A token without its own (a batched turn records none) fails the request. + // A token without its own (the iteration scheduler records none) fails the request. let logprobs = match logprobs .map(|l| l.collect(&tokenizer, &token_ids)) .transpose() diff --git a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs index d97dc184f9..be2fb8ff20 100644 --- a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs +++ b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs @@ -1,6 +1,7 @@ //! FALSIFY-LOGPROBS-CUDA-4971: chat `logprobs` on the dense CUDA backend //! (ASOC-INV-021), through the real router, on the direct path and through the -//! batch scheduler (one request at a time, so its m=1 path). +//! batch scheduler: one request at a time (its m=1 path), and two at once in +//! one m=2 batch. //! //! Before this the CUDA backend answered a request for logprobs with 501. It //! needs a CUDA device and tinyllama; without either the test prints SKIP, @@ -63,6 +64,24 @@ fn scheduled(state: &AppState) -> AppState { state.clone().with_cuda_batch_tx(tx) } +/// The same model behind a batch scheduler that holds its first request until +/// a second fills a batch of two, and the counter whose peak shows the pair +/// decoded as one m=2 batch. +fn batch_of_two(state: &AppState) -> (AppState, std::sync::Arc) { + use crate::api::cuda_batch_scheduler::{spawn_cuda_batch_scheduler, CudaBatchConfig}; + let model = state.cuda_model().expect("a CUDA state").clone(); + let counter = crate::api::InFlightCounter::new(); + let config = CudaBatchConfig { + max_batch: 2, + window_ms: 10_000, + }; + let tx = spawn_cuda_batch_scheduler(model, config, counter.clone()); + (state.clone().with_cuda_batch_tx(tx), counter) +} + +/// The request fields that ask for logprobs with 3 alternatives a step. +const WITH: &str = r#","logprobs":true,"top_logprobs":3"#; + fn chat_body(extra: &str) -> String { format!( r#"{{"model":"default","messages":[{{"role":"user","content":"The capital of France is"}}],"max_tokens":8,"temperature":0{extra}}}"# @@ -108,11 +127,9 @@ async fn streamed_entries(state: &AppState, extra: &str) -> (Vec, u64) { (entries, completion_tokens) } -/// One entry per completion token, each with its best alternatives, the -/// chosen one a best one (greedy); the same reply as without logprobs; and the -/// same entries streamed. -async fn logprobs_are_served(state: &AppState, path: &str) { - let reply = chat_json(state, r#","logprobs":true,"top_logprobs":3"#).await; +/// `reply`'s `logprobs.content`, checked: one entry per completion token, each +/// with its 3 best alternatives, best first, the chosen one the best (greedy). +fn checked_entries<'a>(reply: &'a Value, path: &str) -> &'a [Value] { let completion_tokens = reply["usage"]["completion_tokens"] .as_u64() .expect("usage.completion_tokens"); @@ -134,6 +151,24 @@ async fn logprobs_are_served(state: &AppState, path: &str) { "{path}: {entry}" ); } + content +} + +/// The tokens and logprobs of `a` and `b`, which must be the same. +fn same_entries(a: &[Value], b: &[Value], path: &str) { + assert_eq!(a.len(), b.len(), "{path}: {a:?} vs {b:?}"); + for (x, y) in a.iter().zip(b) { + assert_eq!(x["token"], y["token"], "{path}: {x} vs {y}"); + assert!((logprob(x) - logprob(y)).abs() < 1e-3, "{path}: {x} vs {y}"); + } +} + +/// One entry per completion token, each with its best alternatives, the +/// chosen one a best one (greedy); the same reply as without logprobs; and the +/// same entries streamed. +async fn logprobs_are_served(state: &AppState, path: &str) { + let reply = chat_json(state, WITH).await; + let content = checked_entries(&reply, path); let without = chat_json(state, "").await; assert_eq!( @@ -147,18 +182,54 @@ async fn logprobs_are_served(state: &AppState, path: &str) { let choice = without["choices"][0].as_object().expect("choice object"); assert!(!choice.contains_key("logprobs"), "{path}: {without}"); - let (streamed, streamed_tokens) = - streamed_entries(state, r#","logprobs":true,"top_logprobs":3"#).await; + let (streamed, streamed_tokens) = streamed_entries(state, WITH).await; assert_eq!( streamed.len() as u64, streamed_tokens, "{path}: {streamed:?}" ); - assert_eq!(streamed.len(), content.len(), "{path}: {streamed:?}"); - for (s, u) in streamed.iter().zip(content) { - assert_eq!(s["token"], u["token"], "{path}: {s} vs {u}"); - assert!((logprob(s) - logprob(u)).abs() < 1e-3, "{path}: {s} vs {u}"); - } + same_entries(&streamed, content, path); +} + +/// A batched turn (m = 2) records its logprobs too. One slot asking for them +/// and one not: the same reply, entries only on the one that asked. Both +/// asking, one streamed: the same entries, and the first pair's. +async fn batched_logprobs_are_served(state: &AppState) { + let (pair, counter) = batch_of_two(state); + let (reply, without) = tokio::join!(chat_json(&pair, WITH), chat_json(&pair, "")); + assert_eq!( + counter.peak_in_flight(), + 2, + "the pair did not decode as one batch" + ); + let content = checked_entries(&reply, "batched"); + assert_eq!( + reply["choices"][0]["message"]["content"], without["choices"][0]["message"]["content"], + "batched: asking for logprobs changed the reply" + ); + assert_eq!( + reply["usage"]["completion_tokens"], without["usage"]["completion_tokens"], + "batched" + ); + let choice = without["choices"][0].as_object().expect("choice object"); + assert!(!choice.contains_key("logprobs"), "batched: {without}"); + + let (pair, counter) = batch_of_two(state); + let ((streamed, streamed_tokens), again) = + tokio::join!(streamed_entries(&pair, WITH), chat_json(&pair, WITH)); + assert_eq!( + counter.peak_in_flight(), + 2, + "the streamed pair did not decode as one batch" + ); + assert_eq!( + streamed.len() as u64, + streamed_tokens, + "batched: {streamed:?}" + ); + let again = checked_entries(&again, "batched again"); + same_entries(&streamed, again, "batched streamed"); + same_entries(again, content, "batched again"); } async fn traced_logprobs_are_refused(state: &AppState) { @@ -188,4 +259,5 @@ async fn the_dense_cuda_backend_serves_chat_logprobs_direct_and_scheduled() { logprobs_are_served(&direct, "direct").await; traced_logprobs_are_refused(&direct).await; logprobs_are_served(&scheduled(&direct), "scheduled").await; + batched_logprobs_are_served(&direct).await; } diff --git a/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs b/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs index 3c9fa72159..1f0211c79f 100644 --- a/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs +++ b/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs @@ -462,6 +462,31 @@ impl CudaExecutor { ) -> Result, GpuError> { let (logits_ptr, len) = self.batched_output_norm_lm_head_into_logits(m, hidden_dim, vocab_size, epsilon)?; + Self::download_logits_at(logits_ptr, len) + } + + /// #4971: the logits the last `forward_batched_to_token_ids` argmaxed (m × vocab_size, + /// slot s at s*vocab_size), downloaded after it, so a slot that records logprobs keeps + /// the token the device argmax chose. `batched_gpu_argmax` only reads `logits_buf` and + /// syncs the stream before it returns. + pub fn download_batched_logits(&mut self, m: usize, vocab_size: u32) -> Result, GpuError> { + let len = m * vocab_size as usize; + let logits_ptr = self + .workspace + .logits_buf + .as_ref() + .filter(|buf| buf.len() >= len) + .ok_or_else(|| { + GpuError::InvalidLaunchConfig(format!( + "#4971: no batched logits of {len} elements to download" + )) + })? + .as_ptr(); + Self::download_logits_at(logits_ptr, len) + } + + /// Copies `len` logits from workspace.logits_buf at `logits_ptr` to the host. + fn download_logits_at(logits_ptr: u64, len: usize) -> Result, GpuError> { // SAFETY: logits_ptr is workspace.logits_buf (valid, `len` elements); non-owning wrapper. let logits_buf = unsafe { GpuBuffer::::from_raw_parts(logits_ptr, len) }; let mut host = vec![0.0f32; len]; diff --git a/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs b/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs index d835959759..cf7fa7a9bc 100644 --- a/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs +++ b/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs @@ -134,9 +134,64 @@ pub struct BatchedDecodeState { pub embed_buf: Vec, /// PMAT-086: Pre-allocated position buffer `[m]` (avoids Vec alloc per decode step) pub pos_buf: Vec, + /// #4971: per slot, where its logprobs go, `None` when it did not ask. + /// Every slot starts `None`, joined and recycled ones too; the caller sets + /// it after the slot is in. + pub recorders: Vec>, + /// #4971: each recording slot's record from the last decode step, handed + /// to its recorder by `distribute_tokens` just before the token. + pub records: Vec>, +} + +/// #4971: a slot that asked for logprobs: how many alternatives each step +/// records, and where the record goes. +pub struct SlotRecorder { + /// The `top_logprobs` the request asked for + pub top_n: usize, + /// Called with each step's record, before the step's token callback + pub on_record: Box, } impl BatchedDecodeState { + /// #4971: whether a live slot asked for logprobs, so this step's logits + /// must reach the host. + pub fn recording(&self) -> bool { + (0..self.m).any(|slot| !self.done[slot] && self.recorders[slot].is_some()) + } + + /// #4971: each live recording slot's record of this step: `token_ids` are + /// the chosen tokens and `logits` the `m × vocab_size` they were chosen + /// from, slot `s` at `s * vocab_size`. + pub fn record(&mut self, token_ids: &[u32], logits: &[f32]) { + let vocab = self.vocab_size; + for slot in 0..self.m { + let top_n = self.recorders[slot].as_ref().map(|r| r.top_n); + let Some(top_n) = top_n.filter(|_| !self.done[slot]) else { + continue; + }; + let step = self.sequences[slot].len() - self.prompts[slot].len(); + let slot_logits = &logits[slot * vocab..(slot + 1) * vocab]; + self.records[slot] = Some(crate::gguf::logprobs::StepLogprobs::of( + step, + token_ids[slot], + slot_logits, + top_n, + )); + } + } + + /// #4971: a slot just added or recycled starts with no recorder and no + /// held record. + fn clear_recording(&mut self, slot: usize) { + if slot == self.recorders.len() { + self.recorders.push(None); + self.records.push(None); + } else { + self.recorders[slot] = None; + self.records[slot] = None; + } + } + /// Distribute generated tokens to SSE callbacks. No model lock needed. /// /// Returns true if all slots are done. @@ -147,6 +202,9 @@ impl BatchedDecodeState { } let next_token = token_ids[slot_idx]; + // #4971: the record goes ahead of its token, as on the m=1 path; a + // stop token is never sent, so neither is its record. + let record = self.records[slot_idx].take(); if self.configs[slot_idx].stop_tokens.contains(&next_token) { self.done[slot_idx] = true; @@ -155,6 +213,10 @@ impl BatchedDecodeState { self.sequences[slot_idx].push(next_token); + if let (Some(record), Some(recorder)) = (record, self.recorders[slot_idx].as_mut()) { + (recorder.on_record)(record); + } + if !self.on_tokens[slot_idx](next_token) { self.done[slot_idx] = true; continue; @@ -394,6 +456,8 @@ impl OwnedQuantizedModelCuda { done, embed_buf, pos_buf, + recorders: (0..m).map(|_| None).collect(), + records: (0..m).map(|_| None).collect(), }) } @@ -454,6 +518,39 @@ impl OwnedQuantizedModelCuda { timer.mark("prep"); + let token_ids = self.batched_step_token_ids(state)?; + + timer.mark("fwd+sync+argmax"); + timer.emit(state.gen_idx as u64, state.m); + + if state.gen_idx < 3 { + eprintln!( + "[PMAT-072] decode step {}: token_ids={token_ids:?}, done={:?}", + state.gen_idx, state.done + ); + } + + // FALSIFY-CB-008 evidence, every step rather than the first three. + if batched_decode_trace() { + eprintln!( + "[CB-008] step={} m={} token_ids={token_ids:?} positions={:?} done={:?}", + state.gen_idx, + state.m, + &state.pos_buf[..state.m], + state.done + ); + } + + Ok(token_ids) + } + + /// The decode step's tokens, by the path the batch needs, from the + /// embeddings and positions already in `state`. + /// + /// #4971: a live slot that records logprobs gets this step's record from the + /// same logits its token came from. The graphed path (BATCHED_GRAPH=1) + /// replays into its own buffers, so a recording step runs eager. + fn batched_step_token_ids(&mut self, state: &mut BatchedDecodeState) -> Result> { // PMAT-056: Multi-stream root cause fixed (scatter moved to self.stream), // but graph replay still 25% slower than eager due to capture overhead. // Keep eager by default until graph replay is optimized. @@ -470,7 +567,9 @@ impl OwnedQuantizedModelCuda { .take(state.m) .any(|c| c.temperature > 0.0); - let token_ids: Vec = if any_sampling { + let recording = state.recording(); + + if any_sampling { let vocab = state.vocab_size; let logits = self .executor @@ -487,16 +586,22 @@ impl OwnedQuantizedModelCuda { operation: "forward_batched_to_logits".to_string(), reason: format!("Batched forward failed: {e}"), })?; - (0..state.m) + let token_ids: Vec = (0..state.m) .map(|slot| { let cfg = &state.configs[slot]; let base = slot * vocab; let slot_logits = &logits[base..base + vocab]; select_batched_token(slot_logits, cfg.temperature, cfg.top_k, cfg.top_p) }) - .collect() - } else if use_graph { - self.executor + .collect(); + if recording { + state.record(&token_ids, &logits); + } + return Ok(token_ids); + } + if use_graph && !recording { + return self + .executor .forward_batched_to_token_ids_graphed( &state.embed_buf, &state.pos_buf, @@ -509,45 +614,33 @@ impl OwnedQuantizedModelCuda { .map_err(|e| RealizarError::UnsupportedOperation { operation: "forward_batched_to_token_ids_graphed".to_string(), reason: format!("Batched forward failed: {e}"), - })? - } else { - self.executor - .forward_batched_to_token_ids( - &state.embed_buf, - &state.pos_buf, - state.num_layers, - state.hidden_dim as u32, - state.intermediate_dim as u32, - state.vocab_size as u32, - state.eps, - ) - .map_err(|e| RealizarError::UnsupportedOperation { - operation: "forward_batched_to_token_ids".to_string(), - reason: format!("Batched forward failed: {e}"), - })? - }; - - timer.mark("fwd+sync+argmax"); - timer.emit(state.gen_idx as u64, state.m); - - if state.gen_idx < 3 { - eprintln!( - "[PMAT-072] decode step {}: token_ids={token_ids:?}, done={:?}", - state.gen_idx, state.done - ); + }); } - - // FALSIFY-CB-008 evidence, every step rather than the first three. - if batched_decode_trace() { - eprintln!( - "[CB-008] step={} m={} token_ids={token_ids:?} positions={:?} done={:?}", - state.gen_idx, - state.m, - &state.pos_buf[..state.m], - state.done - ); + let token_ids = self + .executor + .forward_batched_to_token_ids( + &state.embed_buf, + &state.pos_buf, + state.num_layers, + state.hidden_dim as u32, + state.intermediate_dim as u32, + state.vocab_size as u32, + state.eps, + ) + .map_err(|e| RealizarError::UnsupportedOperation { + operation: "forward_batched_to_token_ids".to_string(), + reason: format!("Batched forward failed: {e}"), + })?; + if recording { + let logits = self + .executor + .download_batched_logits(state.m, state.vocab_size as u32) + .map_err(|e| RealizarError::UnsupportedOperation { + operation: "download_batched_logits".to_string(), + reason: format!("#4971 logprobs: {e}"), + })?; + state.record(&token_ids, &logits); } - Ok(token_ids) } @@ -711,6 +804,8 @@ impl OwnedQuantizedModelCuda { state.positions.push(seq_len); state.last_tokens.push(last_token); state.done.push(false); + // #4971: the joined slot records nothing until its caller says so + state.clear_recording(state.m); state.m = new_m; state.embed_buf.resize(new_m * state.hidden_dim, 0.0); state.pos_buf.resize(new_m, 0); @@ -814,6 +909,8 @@ impl OwnedQuantizedModelCuda { state.positions[slot_idx] = seq_len; state.last_tokens[slot_idx] = last_token; state.done[slot_idx] = false; + // #4971: the new request records nothing until its caller says so + state.clear_recording(slot_idx); // Extend max_tokens_max to cover gen_idx + new request's max_tokens state.max_tokens_max = state @@ -981,6 +1078,8 @@ impl OwnedQuantizedModelCuda { state.positions[slot_idx] = seq_len; state.last_tokens[slot_idx] = last_token; state.done[slot_idx] = false; + // #4971: the new request records nothing until its caller says so + state.clear_recording(slot_idx); state.max_tokens_max = state .max_tokens_max .max(state.gen_idx + state.configs[slot_idx].max_tokens); diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index a39b316119..e8c1e69708 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -55,7 +55,7 @@ pub use forward_qwen35_cuda::{ // #3714: the Qwen3-MoE GPU model, its device state, and the MoE shape it is built for. pub use forward_qwen3_moe_resident::{Qwen3MoeCudaModel, Qwen3MoeCudaState, Qwen3MoeShape}; // PMAT-072: Step-wise batched decode state for lock-releasing scheduler -pub use generation::BatchedDecodeState; +pub use generation::{BatchedDecodeState, SlotRecorder}; // #4971: what a prefill extracts for the session's first token. pub(crate) use generation::FirstToken; diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index 64108eb871..af6f41ad88 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -88,7 +88,7 @@ pub use config::*; #[cfg(feature = "cuda")] pub use cuda::{ BatchedDecodeState, CudaBackend, CudaInitError, Qwen35CudaModel, Qwen35CudaState, - Qwen3MoeCudaModel, Qwen3MoeCudaState, Qwen3MoeShape, + Qwen3MoeCudaModel, Qwen3MoeCudaState, Qwen3MoeShape, SlotRecorder, }; #[cfg(feature = "cuda")] pub use cuda_model::*; From 494847416c9c6e9f47f98a92c75534b3e5c2a8de Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:09:55 +0200 Subject: [PATCH 10/35] refactor(serve): split process_iteration_batch into named steps (PMAT-4971) No behaviour change. The iteration scheduler's batch function (cognitive 101) becomes setup, a completion test, one iteration step, the slot scheduling it does under the lock (recycle done slots, else join a waiting request) and the closing of done slots, each under the hook's limits, so the next commit can give a batched slot its logprobs recorder. Its callbacks use the batch scheduler's token_callback, now pub(crate) with its type aliases. "!joined && !has_done" is "!has_done", since only a recycle joins. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 414cb14c8ad2243176f4260588dcfd427ec2b67c) (cherry picked from commit d43adca71ee29f5c5eab12ff03e9a874ae3bba0c) (cherry picked from commit 698b2c8ec7340fcb163079585e6b641b1232097d) --- .../src/api/cuda_batch_scheduler.rs | 6 +- .../src/api/iteration_scheduler.rs | 470 ++++++++++-------- 2 files changed, 268 insertions(+), 208 deletions(-) diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs index 0c53e5447b..fa90d23991 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs @@ -423,15 +423,15 @@ fn log_batch_done(batch_size: usize, elapsed: std::time::Duration) { /// The channel one request's tokens (or its error) go back on. #[cfg(feature = "cuda")] -type TokenSender = tokio::sync::mpsc::Sender>; +pub(crate) type TokenSender = tokio::sync::mpsc::Sender>; /// A slot's per-token callback; `false` means its caller has gone. #[cfg(feature = "cuda")] -type TokenCallback = Box bool + Send>; +pub(crate) type TokenCallback = Box bool + Send>; /// The callback that streams a slot's tokens to `token_tx`. #[cfg(feature = "cuda")] -fn token_callback(token_tx: TokenSender) -> TokenCallback { +pub(crate) fn token_callback(token_tx: TokenSender) -> TokenCallback { Box::new(move |token_id: u32| -> bool { token_tx.try_send(Ok(token_id)).is_ok() }) } diff --git a/crates/aprender-serve/src/api/iteration_scheduler.rs b/crates/aprender-serve/src/api/iteration_scheduler.rs index d2bade7ebe..5a637554a2 100644 --- a/crates/aprender-serve/src/api/iteration_scheduler.rs +++ b/crates/aprender-serve/src/api/iteration_scheduler.rs @@ -13,7 +13,9 @@ //! Enabled via `ITERATION_SCHEDULER=1` env var (opt-in during development). #[cfg(feature = "cuda")] -use crate::api::cuda_batch_scheduler::CudaBatchRequest; +use crate::api::cuda_batch_scheduler::{ + token_callback, CudaBatchRequest, TokenCallback, TokenSender, +}; #[cfg(feature = "cuda")] use crate::gguf::OwnedQuantizedModelCuda; use renacer_core::PhaseTimer; @@ -243,6 +245,18 @@ async fn iteration_scheduler_loop( } } +/// A running batch: its decode state, each slot's error sender, and the +/// sub-phase timer of its iterations. +#[cfg(feature = "cuda")] +struct IterationBatch { + state: crate::gguf::BatchedDecodeState, + /// PMAT-088c: Option so we can drop individual senders when slots finish. + /// Both the callback's sender AND this sender must be dropped for the channel to close. + error_senders: Vec>, + /// PMAT-284: Uniform sub-phase timing via renacer-core PhaseTimer + phase_timer: PhaseTimer, +} + /// Process a batch using iteration-level scheduling. /// /// Uses the existing PMAT-072/073/074 infrastructure but adds: @@ -258,17 +272,13 @@ fn process_iteration_batch( max_slots: usize, total_iterations: &mut u64, ) { - use crate::gguf::QuantizedGenerateConfig; - - let m = batch.len(); - // Single request — fast M=1 path (CUDA graph replay) // realizr#212: non_streaming accumulates via Vec::push then bulk-sends // PERF-041: same predicate, via the one ungated statement of it, so the // F-BATCH-004 knob applies here too. Off, this is `m == 1 && waiting.is_empty()` // unchanged. if crate::api::batch_admission::fast_path_eligible( - m, + batch.len(), waiting.is_empty(), crate::api::batch_admission::force_batched_path(), ) { @@ -280,230 +290,280 @@ fn process_iteration_batch( } // Multi-request batch — PMAT-072/073/074 step-wise decode + // Phase 1: Setup + Prefill + let Some(mut running) = iteration_setup(model, batch, max_slots) else { + return; + }; + + // Phase 2: Iteration-level decode loop + // + // PMAT-088c: Continuous batch — don't restart when all slots finish if there + // are pending requests. Recycle done slots instead of exiting → restarting. + // Without this, every batch restart costs ~20ms setup + 3×30ms slot adds = ~110ms, + // causing TTFT P50 = 116ms. With continuous recycling, TTFT ≈ 27ms (recycle + decode). + while !iteration_batch_complete(&running.state, rx, waiting) { + *total_iterations += 1; + if !iteration_step(model, &mut running, rx, waiting, *total_iterations) { + return; + } + } + + // Phase 3: Cleanup + { + let mut cuda_model = model.write().expect("PMAT-088: model lock poisoned"); + cuda_model.batched_cleanup(&running.state); + } +} + +/// Phase 1: prefill the batch's prompts into a new decode state. If that +/// fails, every request gets the error and there is no batch. +#[cfg(feature = "cuda")] +fn iteration_setup( + model: &Arc>, + batch: Vec, + max_slots: usize, +) -> Option { + use crate::gguf::QuantizedGenerateConfig; + + let m = batch.len(); let prompts: Vec> = batch.iter().map(|r| r.prompt_ids.clone()).collect(); let configs: Vec = batch.iter().map(|r| r.config.clone()).collect(); - // PMAT-088c: Option so we can drop individual senders when slots finish. - // Both the callback's sender AND this sender must be dropped for the channel to close. - let mut error_senders: Vec>>> = + let error_senders: Vec> = batch.iter().map(|r| Some(r.token_tx.clone())).collect(); - - let callbacks: Vec bool + Send>> = batch + let callbacks: Vec = batch .into_iter() - .map(|req| { - Box::new(move |token_id: u32| -> bool { req.token_tx.try_send(Ok(token_id)).is_ok() }) - as Box bool + Send> - }) + .map(|req| token_callback(req.token_tx)) .collect(); - // Phase 1: Setup + Prefill - let mut state = { + let setup = { + let mut cuda_model = model.write().expect("PMAT-088: model lock poisoned"); + cuda_model.batched_setup_and_prefill(&prompts, &configs, callbacks, max_slots) + }; + match setup { + Ok(state) => Some(IterationBatch { + state, + error_senders, + phase_timer: PhaseTimer::from_env("PMAT_283_TIMING", "PMAT-283"), + }), + Err(e) => { + eprintln!("[PMAT-088] Setup+prefill ERROR (m={m}): {e}"); + for tx in error_senders.iter().flatten() { + let _ = tx.try_send(Err(e.to_string())); + } + None + }, + } +} + +/// Whether the batch is over. Exit conditions: +/// 1. All slots done AND no pending requests (batch truly complete) +/// 2. gen_idx exceeded AND all slots done (safety limit) +/// +/// With every slot done, the channel is drained into the waiting queue first: +/// a pending request keeps the batch going, to recycle a done slot. +#[cfg(feature = "cuda")] +fn iteration_batch_complete( + state: &crate::gguf::BatchedDecodeState, + rx: &mut tokio::sync::mpsc::Receiver, + waiting: &mut VecDeque, +) -> bool { + if !state.all_done() { + return false; + } + while let Ok(req) = rx.try_recv() { + waiting.push_back(req); + } + waiting.is_empty() || state.gen_idx >= state.max_tokens_max +} + +/// One iteration: under the model lock, give waiting requests slots and +/// decode a step; then hand out its tokens without the lock and close the +/// channels of the slots it finished. False when the decode failed: every +/// slot has the error, and the batch is cleaned up. +#[cfg(feature = "cuda")] +fn iteration_step( + model: &Arc>, + running: &mut IterationBatch, + rx: &mut tokio::sync::mpsc::Receiver, + waiting: &mut VecDeque, + iteration: u64, +) -> bool { + let iter_start = std::time::Instant::now(); + + running.phase_timer.start(); + + let token_ids = { let mut cuda_model = model.write().expect("PMAT-088: model lock poisoned"); - match cuda_model.batched_setup_and_prefill(&prompts, &configs, callbacks, max_slots) { - Ok(s) => s, + + running.phase_timer.mark("lock"); + + schedule_waiting(&mut cuda_model, running, rx, waiting); + + running.phase_timer.mark("sched"); + + // Decode step + match cuda_model.batched_decode_step(&mut running.state) { + Ok(ids) => ids, Err(e) => { - eprintln!("[PMAT-088] Setup+prefill ERROR (m={m}): {e}"); - for tx in error_senders.iter().flatten() { + eprintln!( + "[PMAT-088] Decode step ERROR (m={}, step={}): {e}", + running.state.m, running.state.gen_idx + ); + for tx in running.error_senders.iter().flatten() { let _ = tx.try_send(Err(e.to_string())); } - return; + cuda_model.batched_cleanup(&running.state); + return false; }, } }; - // Phase 2: Iteration-level decode loop - // - // PMAT-088c: Continuous batch — don't restart when all slots finish if there - // are pending requests. Recycle done slots instead of exiting → restarting. - // Without this, every batch restart costs ~20ms setup + 3×30ms slot adds = ~110ms, - // causing TTFT P50 = 116ms. With continuous recycling, TTFT ≈ 27ms (recycle + decode). + running.phase_timer.mark("decode"); - // PMAT-284: Uniform sub-phase timing via renacer-core PhaseTimer - let mut phase_timer = PhaseTimer::from_env("PMAT_283_TIMING", "PMAT-283"); + // Token distribution WITHOUT lock + running.state.distribute_tokens(&token_ids); - loop { - // Exit conditions: - // 1. All slots done AND no pending requests (batch truly complete) - // 2. gen_idx exceeded AND all slots done (safety limit) - if state.all_done() { - // Drain channel into waiting queue before deciding to exit - while let Ok(req) = rx.try_recv() { - waiting.push_back(req); - } - if waiting.is_empty() { - break; // No pending requests — batch complete - } - // Pending requests exist — continue loop to recycle done slots - } - if state.gen_idx >= state.max_tokens_max && state.all_done() { - break; // Safety limit reached - } + running.phase_timer.mark("dist"); - *total_iterations += 1; - let iter_start = std::time::Instant::now(); - - phase_timer.start(); - - let token_ids = { - let mut cuda_model = model.write().expect("PMAT-088: model lock poisoned"); - - phase_timer.mark("lock"); - - // PMAT-088: Check waiting queue FIRST, then rx channel. - // This ensures requests that arrived during previous iteration's - // token distribution get scheduled before new channel arrivals. - - // PMAT-088d: Batch recycle — collect ALL done slots with waiting - // requests and recycle them in one prefill_multi_prompt call (~14ms - // total regardless of count, vs N×17ms sequential). This eliminates - // recycle serialization when multiple slots finish simultaneously. - // - // Priority: RECYCLE done slots first, then ADD new slots. - let mut joined_this_step = false; - - // 1. BATCH RECYCLE done slots (multi-prompt prefill, one weight read) - let has_done_slots = state.done.iter().any(|&d| d); - if has_done_slots { - let mut recycle_pairs: Vec<( - usize, - Vec, - crate::gguf::QuantizedGenerateConfig, - Box bool + Send>, - )> = Vec::new(); - let mut recycle_error_txs: Vec<( - usize, - tokio::sync::mpsc::Sender>, - )> = Vec::new(); - - for slot_idx in 0..state.m { - if !state.done[slot_idx] { - continue; - } - let next_req = waiting.pop_front().or_else(|| rx.try_recv().ok()); - if let Some(req) = next_req { - let error_tx = req.token_tx.clone(); - let on_token: Box bool + Send> = { - let token_tx = req.token_tx; - Box::new(move |token_id: u32| -> bool { - token_tx.try_send(Ok(token_id)).is_ok() - }) - }; - recycle_error_txs.push((slot_idx, error_tx)); - recycle_pairs.push((slot_idx, req.prompt_ids, req.config, on_token)); - } else { - break; // No more waiting requests - } - } + close_done_slots(running); - if !recycle_pairs.is_empty() { - match cuda_model.recycle_slots_batch(&mut state, recycle_pairs) { - Ok(()) => { - for (slot_idx, error_tx) in recycle_error_txs { - error_senders[slot_idx] = Some(error_tx); - } - joined_this_step = true; - }, - Err(e) => { - eprintln!("[PMAT-088d] Batch recycle FAILED: {e}"); - for (_, error_tx) in &recycle_error_txs { - let _ = error_tx.try_send(Err(e.to_string())); - } - }, - } - } - } + // PMAT-284: Uniform sub-phase timing via renacer-core PhaseTimer + running.phase_timer.emit(iteration, running.state.m); + + // Per-iteration metrics (first 3 only to avoid log spam) + if iteration <= 3 || running.state.gen_idx % 50 == 0 { + let iter_ms = iter_start.elapsed().as_secs_f64() * 1000.0; + let active_slots = running.state.done.iter().filter(|&&d| !d).count(); + eprintln!( + "[PMAT-088] iter={}, m={}, active={}, step_ms={:.1}", + iteration, running.state.m, active_slots, iter_ms, + ); + } + true +} - // 2. ADD new slots only when no done slots to recycle - if !joined_this_step && !has_done_slots && state.m < state.max_kv_slots { - let next_req = waiting.pop_front().or_else(|| rx.try_recv().ok()); - if let Some(req) = next_req { - let error_tx = req.token_tx.clone(); - let on_token: Box bool + Send> = { - let token_tx = req.token_tx; - Box::new(move |token_id: u32| -> bool { - token_tx.try_send(Ok(token_id)).is_ok() - }) - }; - match cuda_model.add_slot_to_batch( - &mut state, - req.prompt_ids, - req.config, - on_token, - ) { - Ok(()) => { - error_senders.push(Some(error_tx)); - joined_this_step = true; - }, - Err(e) => { - eprintln!("[PMAT-088c] Mid-batch join FAILED: {e}"); - let _ = error_tx.try_send(Err(e.to_string())); - }, - } - } - } +/// PMAT-088: Check waiting queue FIRST, then rx channel. +/// This ensures requests that arrived during previous iteration's +/// token distribution get scheduled before new channel arrivals. +/// +/// Priority: RECYCLE done slots first, then ADD new slots. +#[cfg(feature = "cuda")] +fn schedule_waiting( + cuda_model: &mut OwnedQuantizedModelCuda, + running: &mut IterationBatch, + rx: &mut tokio::sync::mpsc::Receiver, + waiting: &mut VecDeque, +) { + if running.state.done.iter().any(|&d| d) { + // 1. BATCH RECYCLE done slots (multi-prompt prefill, one weight read) + recycle_done_slots(cuda_model, running, rx, waiting); + } else if running.state.m < running.state.max_kv_slots { + // 2. ADD new slots only when no done slots to recycle + join_waiting_slot(cuda_model, running, rx, waiting); + } +} - let _ = joined_this_step; // suppress unused warning - - phase_timer.mark("sched"); - - // Decode step - match cuda_model.batched_decode_step(&mut state) { - Ok(ids) => ids, - Err(e) => { - eprintln!( - "[PMAT-088] Decode step ERROR (m={}, step={}): {e}", - state.m, state.gen_idx - ); - for tx in error_senders.iter().flatten() { - let _ = tx.try_send(Err(e.to_string())); - } - cuda_model.batched_cleanup(&state); - return; - }, - } +/// PMAT-088d: Batch recycle — collect ALL done slots with waiting +/// requests and recycle them in one prefill_multi_prompt call (~14ms +/// total regardless of count, vs N×17ms sequential). This eliminates +/// recycle serialization when multiple slots finish simultaneously. +#[cfg(feature = "cuda")] +fn recycle_done_slots( + cuda_model: &mut OwnedQuantizedModelCuda, + running: &mut IterationBatch, + rx: &mut tokio::sync::mpsc::Receiver, + waiting: &mut VecDeque, +) { + let mut recycle_pairs: Vec<( + usize, + Vec, + crate::gguf::QuantizedGenerateConfig, + TokenCallback, + )> = Vec::new(); + let mut recycle_error_txs: Vec<(usize, TokenSender)> = Vec::new(); + + for slot_idx in 0..running.state.m { + if !running.state.done[slot_idx] { + continue; + } + let Some(req) = waiting.pop_front().or_else(|| rx.try_recv().ok()) else { + break; // No more waiting requests }; + recycle_error_txs.push((slot_idx, req.token_tx.clone())); + recycle_pairs.push(( + slot_idx, + req.prompt_ids, + req.config, + token_callback(req.token_tx), + )); + } - phase_timer.mark("decode"); - - // Token distribution WITHOUT lock - state.distribute_tokens(&token_ids); - - phase_timer.mark("dist"); - - // PMAT-088c: Drop callbacks AND error senders for done slots to close channels. - // The SSE handler waits for channel closure (ALL senders dropped) to send [DONE]. - // Without this, continuous batching keeps senders alive → SSE never ends - // → probador never sends new requests → recycling never gets requests. - for slot_idx in 0..state.m { - if state.done[slot_idx] - && error_senders - .get(slot_idx) - .and_then(|o| o.as_ref()) - .is_some() - { - // Replace callback → drops old closure → drops one sender - state.on_tokens[slot_idx] = Box::new(|_| false); - // Drop error sender → drops second sender → channel closes - error_senders[slot_idx] = None; + if recycle_pairs.is_empty() { + return; + } + match cuda_model.recycle_slots_batch(&mut running.state, recycle_pairs) { + Ok(()) => { + for (slot_idx, error_tx) in recycle_error_txs { + running.error_senders[slot_idx] = Some(error_tx); } - } - - // PMAT-284: Uniform sub-phase timing via renacer-core PhaseTimer - phase_timer.emit(*total_iterations as u64, state.m); + }, + Err(e) => { + eprintln!("[PMAT-088d] Batch recycle FAILED: {e}"); + for (_, error_tx) in &recycle_error_txs { + let _ = error_tx.try_send(Err(e.to_string())); + } + }, + } +} - // Per-iteration metrics (first 3 only to avoid log spam) - if *total_iterations <= 3 || state.gen_idx % 50 == 0 { - let iter_ms = iter_start.elapsed().as_secs_f64() * 1000.0; - let active_slots = state.done.iter().filter(|&&d| !d).count(); - eprintln!( - "[PMAT-088] iter={}, m={}, active={}, step_ms={:.1}", - total_iterations, state.m, active_slots, iter_ms, - ); - } +/// PMAT-088c: the next waiting request joins the batch in a new slot. +#[cfg(feature = "cuda")] +fn join_waiting_slot( + cuda_model: &mut OwnedQuantizedModelCuda, + running: &mut IterationBatch, + rx: &mut tokio::sync::mpsc::Receiver, + waiting: &mut VecDeque, +) { + let Some(req) = waiting.pop_front().or_else(|| rx.try_recv().ok()) else { + return; + }; + let error_tx = req.token_tx.clone(); + match cuda_model.add_slot_to_batch( + &mut running.state, + req.prompt_ids, + req.config, + token_callback(req.token_tx), + ) { + Ok(()) => running.error_senders.push(Some(error_tx)), + Err(e) => { + eprintln!("[PMAT-088c] Mid-batch join FAILED: {e}"); + let _ = error_tx.try_send(Err(e.to_string())); + }, } +} - // Phase 3: Cleanup - { - let mut cuda_model = model.write().expect("PMAT-088: model lock poisoned"); - cuda_model.batched_cleanup(&state); +/// PMAT-088c: Drop callbacks AND error senders for done slots to close channels. +/// The SSE handler waits for channel closure (ALL senders dropped) to send [DONE]. +/// Without this, continuous batching keeps senders alive → SSE never ends +/// → probador never sends new requests → recycling never gets requests. +#[cfg(feature = "cuda")] +fn close_done_slots(running: &mut IterationBatch) { + let IterationBatch { + state, + error_senders, + .. + } = running; + for slot_idx in 0..state.m { + if state.done[slot_idx] + && error_senders + .get(slot_idx) + .and_then(|o| o.as_ref()) + .is_some() + { + // Replace callback → drops old closure → drops one sender + state.on_tokens[slot_idx] = Box::new(|_| false); + // Drop error sender → drops second sender → channel closes + error_senders[slot_idx] = None; + } } } From f4f7e029dcd7c4d122f0dcc7012899d06d769add Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:16:56 +0200 Subject: [PATCH 11/35] feat(serve): chat logprobs on the iteration scheduler's batched path (PMAT-4971) A request that asks for logprobs now records them in the iteration scheduler too. A slot gets its recorder whether its prompt was prefilled at setup, joined a free slot mid-batch, or was recycled into a finished slot; each is set only after the engine call succeeds, as the batch scheduler does. FALSIFY-LOGPROBS-CUDA-4971 drives process_iteration_batch directly, so the batched path is not a race: two prompts set up (one asking, one not) and two waiting, with slots for 3. Every turn runs to its max_tokens. Each turn that asked gets one record per token, steps from 0, naming the token sent, best 3 first, every logprob <= 0, the chosen one as likely as the best. Before this change those turns got no records, so the count check fails. The handler still fails closed on a token without its record; the docs no longer name the iteration scheduler as a turn that records none. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d83020fe5ed707a5e4d714c4229a875019086b39) (cherry picked from commit 76fa08c90171b0e63514b9ac1587c6c768121dbb) (cherry picked from commit e5aa87b25700fb7beba7fc6434d94f1c4433f440) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- .../aprender-serve/src/api/chat_logprobs.rs | 4 +- .../src/api/cuda_batch_scheduler.rs | 2 +- .../src/api/cuda_chat_backend.rs | 2 +- .../src/api/iteration_scheduler.rs | 48 ++++++-- .../src/api/tests/logprobs_cuda_4971.rs | 111 +++++++++++++++++- crates/aprender-serve/src/api/tests/mod.rs | 2 +- 7 files changed, 151 insertions(+), 20 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 34f3e376a0..36cc9b44f4 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -672,7 +672,7 @@ falsification_tests: if_fails: 'The V3 witness model is served by the Qwen3.5 backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the model it is specified against.' - id: FALSIFY-LOGPROBS-CUDA-4971 name: the_dense_cuda_backend_serves_chat_logprobs - prediction: 'With --features cuda, tinyllama Q4_K_M on CUDA device 0 behind the router, first on the direct path, then behind a running batch scheduler one request at a time (its m=1 path), then two at once in one batch: `logprobs: true, top_logprobs: 3` at temperature 0 answers 200 with usage.completion_tokens entries, each with 3 top_logprobs, best first, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and completion_tokens and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob, and no chunk is an error. On the direct path a traced request (X-Trace-Level) for logprobs answers 501 naming `CUDA (traced)`. Behind a scheduler that holds a request until a second fills a batch of two, the scheduler''s in-flight peak is 2, so the pair decoded as one m=2 batch. Of a pair where one asks for logprobs and one does not, the one asking gets the same well-formed entries, and both get the same text and completion_tokens, with no `logprobs` key on the one not asking. Of a pair that both ask, one streamed, the streamed entries equal the unstreamed ones and the first pair''s, token and logprob. A turn that records none (the iteration scheduler) never answers 200 without them: a token without its record ends the stream with an error chunk and fails an unstreamed reply with 500 (chat_logprobs tests). No device or no model file prints a SKIP line, which is not_measured and never a pass.' + prediction: 'With --features cuda, tinyllama Q4_K_M on CUDA device 0 behind the router, first on the direct path, then behind a running batch scheduler one request at a time (its m=1 path), then two at once in one batch: `logprobs: true, top_logprobs: 3` at temperature 0 answers 200 with usage.completion_tokens entries, each with 3 top_logprobs, best first, every logprob <= 0, and the entry logprob equal to top_logprobs[0].logprob. The same request without `logprobs` gives the same text and completion_tokens and no `logprobs` key. Streamed, the entries over the chunks number the terminal chunk''s usage.completion_tokens and equal the unstreamed reply''s, token and logprob, and no chunk is an error. On the direct path a traced request (X-Trace-Level) for logprobs answers 501 naming `CUDA (traced)`. Behind a scheduler that holds a request until a second fills a batch of two, the scheduler''s in-flight peak is 2, so the pair decoded as one m=2 batch. Of a pair where one asks for logprobs and one does not, the one asking gets the same well-formed entries, and both get the same text and completion_tokens, with no `logprobs` key on the one not asking. Of a pair that both ask, one streamed, the streamed entries equal the unstreamed ones and the first pair''s, token and logprob. On the iteration scheduler''s batched path, driven directly with slots for 3, two prompts set up together (one asking for 3 top logprobs, one not) and two waiting: every turn runs to its max_tokens, the waiting queue ends empty, and each turn that asked, whether set up, joined into a free slot or recycled into a finished one, gets one record per token, its steps counted from 0, each naming the token sent, 3 top logprobs best first, every logprob <= 0, and the chosen logprob equal to the best one. A token that arrives without its record still never answers 200: it ends the stream with an error chunk and fails an unstreamed reply with 500 (chat_logprobs tests). No device or no model file prints a SKIP line, which is not_measured and never a pass.' test_harness: 'cargo test -p aprender-serve --features cuda --lib api::tests::logprobs_cuda_4971 -- --nocapture' expected_output: "test result: ok" if_fails: 'apr serve on a CUDA host routes dense chat through this backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the GPU leg.' diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs index 1fa257642e..d6e61acc1a 100644 --- a/crates/aprender-serve/src/api/chat_logprobs.rs +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -181,8 +181,8 @@ impl StreamLogprobs { /// Holds the entry of `token`, which the stream has just received. /// /// It does not wait for the record: one that is not here when its token is - /// was never sent, and waiting would hang a stream whose engine keeps the - /// sender (a turn of the iteration scheduler, which records none). + /// was never sent, and waiting would hang the stream if an engine kept the + /// sender and recorded nothing. /// /// # Errors /// The engine sent no record for `token`, or the record of another token. diff --git a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs index fa90d23991..106a0dde84 100644 --- a/crates/aprender-serve/src/api/cuda_batch_scheduler.rs +++ b/crates/aprender-serve/src/api/cuda_batch_scheduler.rs @@ -438,7 +438,7 @@ pub(crate) fn token_callback(token_tx: TokenSender) -> TokenCallback { /// #4971: a batched slot's recorder, which sends each record to the handler /// ahead of the token `token_callback` sends. #[cfg(feature = "cuda")] -fn slot_recorder(sink: RecordSink) -> crate::gguf::SlotRecorder { +pub(crate) fn slot_recorder(sink: RecordSink) -> crate::gguf::SlotRecorder { let RecordSink { top_n, records } = sink; crate::gguf::SlotRecorder { top_n, diff --git a/crates/aprender-serve/src/api/cuda_chat_backend.rs b/crates/aprender-serve/src/api/cuda_chat_backend.rs index 2e17791b38..6f67db30f6 100644 --- a/crates/aprender-serve/src/api/cuda_chat_backend.rs +++ b/crates/aprender-serve/src/api/cuda_chat_backend.rs @@ -189,7 +189,7 @@ async fn try_cuda_backend( Err(r) => return Some(r), }; // #4971: every record was sent ahead of its token, so all of them are here. - // A token without its own (the iteration scheduler records none) fails the request. + // A token without its own fails the request. let logprobs = match logprobs .map(|l| l.collect(&tokenizer, &token_ids)) .transpose() diff --git a/crates/aprender-serve/src/api/iteration_scheduler.rs b/crates/aprender-serve/src/api/iteration_scheduler.rs index 5a637554a2..0618dd04c0 100644 --- a/crates/aprender-serve/src/api/iteration_scheduler.rs +++ b/crates/aprender-serve/src/api/iteration_scheduler.rs @@ -14,7 +14,7 @@ #[cfg(feature = "cuda")] use crate::api::cuda_batch_scheduler::{ - token_callback, CudaBatchRequest, TokenCallback, TokenSender, + slot_recorder, token_callback, CudaBatchRequest, TokenCallback, TokenSender, }; #[cfg(feature = "cuda")] use crate::gguf::OwnedQuantizedModelCuda; @@ -263,8 +263,11 @@ struct IterationBatch { /// - Waiting queue integration (new requests from waiting queue, not just rx) /// - Per-iteration metrics /// - Prefill interleaving preparation +/// +/// #4971: a slot whose request asked for logprobs records them, whether it +/// was prefilled at setup, joined mid-batch or recycled. #[cfg(feature = "cuda")] -fn process_iteration_batch( +pub(crate) fn process_iteration_batch( model: &Arc>, batch: Vec, rx: &mut tokio::sync::mpsc::Receiver, @@ -320,7 +323,7 @@ fn process_iteration_batch( #[cfg(feature = "cuda")] fn iteration_setup( model: &Arc>, - batch: Vec, + mut batch: Vec, max_slots: usize, ) -> Option { use crate::gguf::QuantizedGenerateConfig; @@ -330,6 +333,10 @@ fn iteration_setup( let configs: Vec = batch.iter().map(|r| r.config.clone()).collect(); let error_senders: Vec> = batch.iter().map(|r| Some(r.token_tx.clone())).collect(); + let recorders: Vec> = batch + .iter_mut() + .map(|r| r.logprobs.take().map(slot_recorder)) + .collect(); let callbacks: Vec = batch .into_iter() .map(|req| token_callback(req.token_tx)) @@ -340,11 +347,15 @@ fn iteration_setup( cuda_model.batched_setup_and_prefill(&prompts, &configs, callbacks, max_slots) }; match setup { - Ok(state) => Some(IterationBatch { - state, - error_senders, - phase_timer: PhaseTimer::from_env("PMAT_283_TIMING", "PMAT-283"), - }), + Ok(mut state) => { + // #4971: one per prompt prefilled, in its slot + state.recorders = recorders; + Some(IterationBatch { + state, + error_senders, + phase_timer: PhaseTimer::from_env("PMAT_283_TIMING", "PMAT-283"), + }) + }, Err(e) => { eprintln!("[PMAT-088] Setup+prefill ERROR (m={m}): {e}"); for tx in error_senders.iter().flatten() { @@ -480,7 +491,9 @@ fn recycle_done_slots( crate::gguf::QuantizedGenerateConfig, TokenCallback, )> = Vec::new(); - let mut recycle_error_txs: Vec<(usize, TokenSender)> = Vec::new(); + // #4971: each recycled slot's error sender, and its recorder + let mut recycle_error_txs: Vec<(usize, TokenSender, Option)> = + Vec::new(); for slot_idx in 0..running.state.m { if !running.state.done[slot_idx] { @@ -489,7 +502,11 @@ fn recycle_done_slots( let Some(req) = waiting.pop_front().or_else(|| rx.try_recv().ok()) else { break; // No more waiting requests }; - recycle_error_txs.push((slot_idx, req.token_tx.clone())); + recycle_error_txs.push(( + slot_idx, + req.token_tx.clone(), + req.logprobs.map(slot_recorder), + )); recycle_pairs.push(( slot_idx, req.prompt_ids, @@ -503,13 +520,14 @@ fn recycle_done_slots( } match cuda_model.recycle_slots_batch(&mut running.state, recycle_pairs) { Ok(()) => { - for (slot_idx, error_tx) in recycle_error_txs { + for (slot_idx, error_tx, recorder) in recycle_error_txs { running.error_senders[slot_idx] = Some(error_tx); + running.state.recorders[slot_idx] = recorder; } }, Err(e) => { eprintln!("[PMAT-088d] Batch recycle FAILED: {e}"); - for (_, error_tx) in &recycle_error_txs { + for (_, error_tx, _) in &recycle_error_txs { let _ = error_tx.try_send(Err(e.to_string())); } }, @@ -528,13 +546,17 @@ fn join_waiting_slot( return; }; let error_tx = req.token_tx.clone(); + let recorder = req.logprobs.map(slot_recorder); match cuda_model.add_slot_to_batch( &mut running.state, req.prompt_ids, req.config, token_callback(req.token_tx), ) { - Ok(()) => running.error_senders.push(Some(error_tx)), + Ok(()) => { + running.state.recorders[running.state.m - 1] = recorder; + running.error_senders.push(Some(error_tx)); + }, Err(e) => { eprintln!("[PMAT-088c] Mid-batch join FAILED: {e}"); let _ = error_tx.try_send(Err(e.to_string())); diff --git a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs index be2fb8ff20..5a4578988c 100644 --- a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs +++ b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs @@ -1,7 +1,8 @@ //! FALSIFY-LOGPROBS-CUDA-4971: chat `logprobs` on the dense CUDA backend //! (ASOC-INV-021), through the real router, on the direct path and through the //! batch scheduler: one request at a time (its m=1 path), and two at once in -//! one m=2 batch. +//! one m=2 batch; and on the iteration scheduler's batched path, driven +//! directly, where requests join and recycle into slots mid-batch. //! //! Before this the CUDA backend answered a request for logprobs with 501. It //! needs a CUDA device and tinyllama; without either the test prints SKIP, @@ -232,6 +233,113 @@ async fn batched_logprobs_are_served(state: &AppState) { same_entries(again, content, "batched again"); } +/// One request to the iteration scheduler, and the ends its tokens and (when +/// it asks) its records arrive on. +struct Turn { + max_tokens: usize, + tokens: tokio::sync::mpsc::Receiver>, + records: Option>, +} + +fn turn( + prompt_ids: &[u32], + max_tokens: usize, + asks: bool, +) -> (crate::api::cuda_batch_scheduler::CudaBatchRequest, Turn) { + use crate::api::cuda_batch_scheduler::{CudaBatchRequest, RecordSink}; + let (token_tx, tokens) = tokio::sync::mpsc::channel(64); + let (logprobs, records) = if asks { + let (records, rx) = tokio::sync::mpsc::unbounded_channel(); + (Some(RecordSink { top_n: 3, records }), Some(rx)) + } else { + (None, None) + }; + let request = CudaBatchRequest { + prompt_ids: prompt_ids.to_vec(), + // temperature 0 and no stop tokens: each turn runs to its max_tokens + config: crate::gguf::QuantizedGenerateConfig::deterministic(max_tokens), + token_tx, + non_streaming: false, + enqueue_time: std::time::Instant::now(), + timing_tx: None, + logprobs, + }; + let turn = Turn { + max_tokens, + tokens, + records, + }; + (request, turn) +} + +/// The turn ran to the end, and if it asked, one record per token: steps from +/// 0, each naming its token, the best 3 first, every logprob <= 0, and at +/// temperature 0 the chosen one as likely as the best. +fn turn_is_served(mut turn: Turn, path: &str) { + let mut tokens = Vec::new(); + while let Ok(token) = turn.tokens.try_recv() { + tokens.push(token.unwrap_or_else(|e| panic!("{path}: {e}"))); + } + assert_eq!(tokens.len(), turn.max_tokens, "{path}: tokens {tokens:?}"); + let Some(mut rx) = turn.records else { + return; + }; + let mut records = Vec::new(); + while let Ok(record) = rx.try_recv() { + records.push(record); + } + assert_eq!(records.len(), tokens.len(), "{path}: one record per token"); + for (i, (record, &token)) in records.iter().zip(&tokens).enumerate() { + assert_eq!(record.step, i, "{path}"); + assert_eq!(record.chosen, token, "{path}: step {i}"); + assert_eq!(record.top.len(), 3, "{path}: step {i}"); + assert!( + record.top.windows(2).all(|w| w[0].logprob >= w[1].logprob), + "{path}: step {i} not best first" + ); + assert!( + record.chosen_logprob <= 0.0 && record.top.iter().all(|t| t.logprob <= 0.0), + "{path}: step {i}" + ); + assert!( + (record.chosen_logprob - record.top[0].logprob).abs() < 1e-4, + "{path}: step {i} chose below the best at temperature 0" + ); + } +} + +/// The iteration scheduler's batched path, driven directly so the path is not +/// a race: two prompts set up together (one asking, one not), a third that +/// joins a free slot (or recycles one, if only two fit) and a fourth that +/// recycles the slot of whichever finishes first. Each that asked gets its own +/// records, and the one that did not gets its tokens. +fn iteration_scheduler_records(state: &AppState) { + use std::collections::VecDeque; + let model = state.cuda_model().expect("a CUDA state"); + // "The capital of France is", "Hello,", "What is", "Once upon a time" + let (first, one) = turn(&[1, 450, 7483, 310, 3444, 338], 6, true); + let (second, two) = turn(&[1, 15043, 29892], 10, false); + let (third, three) = turn(&[1, 1724, 338], 8, true); + let (fourth, four) = turn(&[1, 9038, 2501, 263, 931], 8, true); + let (_open, mut rx) = tokio::sync::mpsc::channel(4); + let mut waiting = VecDeque::from([third, fourth]); + let mut iterations = 0; + crate::api::iteration_scheduler::process_iteration_batch( + model, + vec![first, second], + &mut rx, + &mut waiting, + 3, + &mut iterations, + ); + assert!(waiting.is_empty(), "iteration: {} never ran", waiting.len()); + assert!(iterations > 1, "iteration: the batched path did not run"); + turn_is_served(one, "iteration set up"); + turn_is_served(two, "iteration set up, not asking"); + turn_is_served(three, "iteration joined"); + turn_is_served(four, "iteration recycled"); +} + async fn traced_logprobs_are_refused(state: &AppState) { let request = Request::builder() .method("POST") @@ -260,4 +368,5 @@ async fn the_dense_cuda_backend_serves_chat_logprobs_direct_and_scheduled() { traced_logprobs_are_refused(&direct).await; logprobs_are_served(&scheduled(&direct), "scheduled").await; batched_logprobs_are_served(&direct).await; + iteration_scheduler_records(&direct); } diff --git a/crates/aprender-serve/src/api/tests/mod.rs b/crates/aprender-serve/src/api/tests/mod.rs index 57427d9c10..caaa1494a4 100644 --- a/crates/aprender-serve/src/api/tests/mod.rs +++ b/crates/aprender-serve/src/api/tests/mod.rs @@ -81,4 +81,4 @@ mod stream_tool_calls_4918; // aprender#4918: streamed tool calls arrive as delt #[cfg(feature = "gpu")] // create_test_quantized_model is gpu-gated mod logprobs_4971; // aprender#4971: chat logprobs/top_logprobs honoured on the quantized CPU path, refused 4xx/501 elsewhere (ASOC-INV-021) #[cfg(feature = "cuda")] -mod logprobs_cuda_4971; // aprender#4971: chat logprobs on the dense CUDA backend, direct and through the batch scheduler (needs a device + tinyllama; SKIP = not_measured) +mod logprobs_cuda_4971; // aprender#4971: chat logprobs on the dense CUDA backend, direct, through the batch scheduler and on the iteration scheduler (needs a device + tinyllama; SKIP = not_measured) From 7416d814ef69b9c58f04c17681cc27ee2b31dfb4 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:34:24 +0200 Subject: [PATCH 12/35] feat(serve): chat routes outside realizar's router refuse logprobs by name (PMAT-4971) ASOC-INV-021 forbids a 200 without logprobs to a request that asked for them. Four apr serve chat routes are not mounted by realizar's router and parse the body as untyped JSON (or, in banco, a type without the fields), so they dropped `logprobs` and answered 200: - APR CPU fallback (AprTransformer): 501 naming `APR CPU (AprTransformer)` - SafeTensors F32 CPU: 501 naming `SafeTensors F32 CPU`, before the no-transformer 503 - WGPU: 501 naming `WGPU` - banco: the request type gains `logprobs` and `top_logprobs` (typed, so 21 is refused when parsed); 501 naming banco All four answer as the router does: `top_logprobs` without `logprobs: true` is 400, an out-of-range `top_logprobs` or a non-bool `logprobs` is 422 naming the field. One check in realizar decides it for every route (api::json_logprobs_refusal for JSON bodies, logprobs_fields_refusal for typed ones); the router's own reasons now come from the same two text functions. Falsifiers: a case table on the check (api::chat_logprobs tests), FALSIFY-LOGPROBS-SERVERS-4971 (apr-cli tests_logprobs_4971, the real APR CPU and SafeTensors routers with no model) and FALSIFY-LOGPROBS-BANCO-4971 (banco's default router). WGPU has no router test because its state needs a GPU device; it is covered by the case table and a --features wgpu build. Not changed: build_gpu_router (handlers_include_01.rs) has no caller, and the server `apr compile` generates is out of scope. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 570d6862db9d7a9e7e05c7f0e1fbcfc82e6f30be) (cherry picked from commit 6e60f3e9fa7cb8479cbc565824bcf9cedfdf2cd9) (cherry picked from commit 5b1eda6dc0345fe584fd7ee95eb77e0218c9f5b5) --- contracts/apr-serve-openai-compat-v1.yaml | 12 ++ crates/apr-cli/src/commands/serve/chat.rs | 6 + .../serve/handler_apr_cpu_completion.rs | 21 +++ crates/apr-cli/src/commands/serve/handlers.rs | 3 + crates/apr-cli/src/commands/serve/mod.rs | 3 + .../src/commands/serve/tests_logprobs_4971.rs | 93 ++++++++++++ .../src/serve/banco/handlers.rs | 20 +++ .../src/serve/banco/handlers_completions.rs | 2 + .../src/serve/banco/handlers_tests.rs | 46 ++++++ .../src/serve/banco/handlers_ui.rs | 2 + .../src/serve/banco/types_chat.rs | 6 + .../aprender-serve/src/api/chat_logprobs.rs | 143 ++++++++++++++++-- crates/aprender-serve/src/api/mod.rs | 3 +- 13 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 crates/apr-cli/src/commands/serve/tests_logprobs_4971.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 36cc9b44f4..89949abee5 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -676,3 +676,15 @@ falsification_tests: test_harness: 'cargo test -p aprender-serve --features cuda --lib api::tests::logprobs_cuda_4971 -- --nocapture' expected_output: "test result: ok" if_fails: 'apr serve on a CUDA host routes dense chat through this backend, so a 501 or a 200 without entries there leaves the perf041 margin unmeasured on the GPU leg.' + - id: FALSIFY-LOGPROBS-SERVERS-4971 + name: chat_routes_outside_the_router_refuse_logprobs_by_name + prediction: 'The apr serve chat routes that realizar''s router does not mount compute no logprobs, so they refuse them the way the router does, with no model loaded. On the APR CPU fallback router and the SafeTensors F32 inference router, `logprobs: true` (alone, or with `top_logprobs: 3`) is 501 and names `APR CPU (AprTransformer)` or `SafeTensors F32 CPU`; `top_logprobs: 3` without `logprobs: true` is 400; `top_logprobs: 21` is 422 and names top_logprobs; `logprobs: "yes"` is 422; and `logprobs: false` is not refused for logprobs. The WGPU route calls the same check (realizar::api::json_logprobs_refusal, named `WGPU`); it has no router test here because its state needs a GPU device, so its falsifier is that check''s case table (api::chat_logprobs tests) and a build with --features wgpu. banco is FALSIFY-LOGPROBS-BANCO-4971.' + test_harness: 'cargo test -p apr-cli --lib tests_logprobs_4971' + expected_output: "test result: ok" + if_fails: 'A client that asks one of these routes for logprobs gets a 200 without them (each parses the body as untyped JSON and ignores keys it does not read), which ASOC-INV-021 forbids.' + - id: FALSIFY-LOGPROBS-BANCO-4971 + name: banco_chat_refuses_logprobs_by_name + prediction: 'On banco''s default router with no model loaded, POST /api/v1/chat/completions with `logprobs: true` (alone, or with `top_logprobs: 2`) is 501 and names the banco chat backend; `top_logprobs: 2` alone is 400; `top_logprobs: 21` is 422 and names top_logprobs; `logprobs: false` is served (200, chat.completion).' + test_harness: 'cargo test -p aprender-orchestrate --features banco --lib refuse_logprobs' + expected_output: "test result: ok" + if_fails: 'banco''s request type dropped the two fields as unknown keys, so a client asking for logprobs got a dry-run or model reply without them, a 200 that ASOC-INV-021 forbids.' diff --git a/crates/apr-cli/src/commands/serve/chat.rs b/crates/apr-cli/src/commands/serve/chat.rs index ebd2809400..457f9921f8 100644 --- a/crates/apr-cli/src/commands/serve/chat.rs +++ b/crates/apr-cli/src/commands/serve/chat.rs @@ -224,6 +224,12 @@ pub(crate) async fn safetensors_chat_completions_handler( use axum::response::{sse::Event, IntoResponse, Sse}; use futures_util::stream; + if let Some(refused) = + super::handlers::logprobs_refusal_response(&request, "SafeTensors F32 CPU") + { + return refused; + } + // Parse request - try structured first, fallback to raw JSON (GH-160) let parsed_request = match parse_chat_completion_request(&request) { Ok(req) => req, diff --git a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs index 27d26ae0db..bb082188f5 100644 --- a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs @@ -55,6 +55,24 @@ async fn handle_apr_cpu_completion( } } +/// #4971 (ASOC-INV-021): `backend` does not compute logprobs, so a request for +/// them is refused rather than served without them: 501 naming `backend`, or +/// 400/422 for a bad `logprobs`/`top_logprobs`, as realizar's router answers. +#[cfg(feature = "inference")] +#[allow(clippy::disallowed_methods)] // serde_json::json!() macro uses infallible unwrap +pub(crate) fn logprobs_refusal_response( + req: &serde_json::Value, + backend: &str, +) -> Option { + use axum::{response::IntoResponse, Json}; + + let (status, reason) = realizar::api::json_logprobs_refusal(req, backend)?; + let error = serde_json::json!({ + "error": {"message": reason, "type": "invalid_request_error", "param": "logprobs"} + }); + Some((status, Json(error)).into_response()) +} + /// GH-283: Validate the "model" field in the request matches the loaded model. /// /// Returns an error response if the model name doesn't match. Accepts "apr" as @@ -340,6 +358,9 @@ async fn handle_apr_cpu_chat_completion( if let Some(err_response) = validate_request_model(req, &s.model_name) { return err_response; } + if let Some(refused) = logprobs_refusal_response(req, "APR CPU (AprTransformer)") { + return refused; + } let messages = req.get("messages").and_then(|m| m.as_array()); let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false); diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index 8120dd763b..5b30a584bb 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -357,6 +357,9 @@ async fn wgpu_chat_completion( state: Arc, axum::Json(body): axum::Json, ) -> axum::response::Response { + if let Some(refused) = logprobs_refusal_response(&body, "WGPU") { + return refused; + } // GH-665: Cap max_tokens to prevent hangs on large values let max_tokens = body["max_tokens"].as_u64().unwrap_or(64).min(4096) as usize; let stream = body["stream"].as_bool().unwrap_or(false); diff --git a/crates/apr-cli/src/commands/serve/mod.rs b/crates/apr-cli/src/commands/serve/mod.rs index 232ee7bb6f..4ec18077fa 100644 --- a/crates/apr-cli/src/commands/serve/mod.rs +++ b/crates/apr-cli/src/commands/serve/mod.rs @@ -28,6 +28,9 @@ mod tests; mod tests_contract_enforcement; #[cfg(all(test, feature = "inference"))] mod tests_route_index_3979; +// #4971: the chat routes outside realizar's router refuse logprobs (ASOC-INV-021). +#[cfg(all(test, feature = "inference"))] +mod tests_logprobs_4971; // PP-LLAMA-001 PP-14/PP-15/§9 #8: the offload report the served process // publishes. `inference`-gated because the report type comes from realizar. #[cfg(test)] diff --git a/crates/apr-cli/src/commands/serve/tests_logprobs_4971.rs b/crates/apr-cli/src/commands/serve/tests_logprobs_4971.rs new file mode 100644 index 0000000000..2a8655505b --- /dev/null +++ b/crates/apr-cli/src/commands/serve/tests_logprobs_4971.rs @@ -0,0 +1,93 @@ +//! #4971 (ASOC-INV-021): the `apr serve` chat routes realizar's router does not +//! mount compute no logprobs, so a request for them is refused (501 naming the +//! backend, 400/422 for a bad field) and never answered 200 without them. +//! Driven through the REAL router builders, with no model loaded. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use tower::ServiceExt; + +async fn chat(router: &axum::Router, body: serde_json::Value) -> (StatusCode, String) { + let req = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + let resp = router.clone().oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (status, String::from_utf8_lossy(&bytes).into_owned()) +} + +/// Every refusal `router` must answer, naming `backend` on the 501; and a +/// request that does not ask is not refused for logprobs. +async fn refuses_logprobs(router: &axum::Router, backend: &str) { + let messages = serde_json::json!([{"role": "user", "content": "hi"}]); + // (logprobs, top_logprobs, status, words the reply must carry) + let table: &[(serde_json::Value, serde_json::Value, StatusCode, &str)] = &[ + ( + true.into(), + serde_json::Value::Null, + StatusCode::NOT_IMPLEMENTED, + backend, + ), + (true.into(), 3.into(), StatusCode::NOT_IMPLEMENTED, backend), + ( + serde_json::Value::Null, + 3.into(), + StatusCode::BAD_REQUEST, + "requires logprobs", + ), + ( + true.into(), + 21.into(), + StatusCode::UNPROCESSABLE_ENTITY, + "top_logprobs must be", + ), + ( + "yes".into(), + serde_json::Value::Null, + StatusCode::UNPROCESSABLE_ENTITY, + "logprobs must be", + ), + ]; + for (logprobs, top, want, words) in table { + let body = serde_json::json!({ + "messages": messages, "max_tokens": 2, "logprobs": logprobs, "top_logprobs": top + }); + let (status, reply) = chat(router, body.clone()).await; + assert_eq!(status, *want, "{backend} {body}: {reply}"); + assert!(reply.contains(words), "{backend} {body}: {reply}"); + } + let plain = serde_json::json!({"messages": messages, "max_tokens": 2, "logprobs": false}); + let (status, reply) = chat(router, plain).await; + assert_ne!(status, StatusCode::NOT_IMPLEMENTED, "{backend}: {reply}"); + assert!( + !reply.contains("`logprobs` is not supported"), + "{backend}: {reply}" + ); +} + +#[tokio::test] +async fn the_apr_cpu_chat_route_refuses_logprobs() { + let r = super::handlers::build_demo_apr_cpu_router_for_test(); + refuses_logprobs(&r, "APR CPU (AprTransformer)").await; +} + +#[tokio::test] +async fn the_safetensors_f32_chat_route_refuses_logprobs() { + let state = super::safetensors::SafeTensorsState { + transformer: None, + tokenizer_info: None, + model_path: "m.safetensors".into(), + }; + let r = super::safetensors::safetensors_app( + serde_json::json!({"count": 0}), + true, + state, + "m".into(), + ); + refuses_logprobs(&r, "SafeTensors F32 CPU").await; +} diff --git a/crates/aprender-orchestrate/src/serve/banco/handlers.rs b/crates/aprender-orchestrate/src/serve/banco/handlers.rs index 442319d818..293a2aba7b 100644 --- a/crates/aprender-orchestrate/src/serve/banco/handlers.rs +++ b/crates/aprender-orchestrate/src/serve/banco/handlers.rs @@ -74,10 +74,30 @@ pub async fn system_handler(State(state): State) -> Json Result<(), (StatusCode, Json)> { + let refusal = + realizar::api::logprobs_fields_refusal(request.logprobs, request.top_logprobs, "banco"); + match refusal { + None => Ok(()), + Some((status, reason)) => { + let kind = if status == StatusCode::NOT_IMPLEMENTED { + "not_implemented" + } else { + "invalid_request" + }; + Err((status, Json(ErrorResponse::new(reason, kind, status.as_u16())))) + } + } +} + pub async fn chat_completions_handler( State(state): State, Json(request): Json, ) -> Result)> { + refuse_logprobs(&request)?; // Expand @preset: references in message content let mut request = request; for msg in &mut request.messages { diff --git a/crates/aprender-orchestrate/src/serve/banco/handlers_completions.rs b/crates/aprender-orchestrate/src/serve/banco/handlers_completions.rs index 5d2d68bcce..02e396f11b 100644 --- a/crates/aprender-orchestrate/src/serve/banco/handlers_completions.rs +++ b/crates/aprender-orchestrate/src/serve/banco/handlers_completions.rs @@ -102,6 +102,8 @@ fn generate_completion(state: &BancoState, prompt: &str, request: &CompletionReq attachments: vec![], tools: None, tool_choice: None, + logprobs: None, + top_logprobs: None, }; super::handlers_inference::try_inference(state, &chat_req) diff --git a/crates/aprender-orchestrate/src/serve/banco/handlers_tests.rs b/crates/aprender-orchestrate/src/serve/banco/handlers_tests.rs index 834e9f9f75..1212b18438 100644 --- a/crates/aprender-orchestrate/src/serve/banco/handlers_tests.rs +++ b/crates/aprender-orchestrate/src/serve/banco/handlers_tests.rs @@ -335,3 +335,49 @@ async fn test_BANCO_HDL_008_privacy_header_on_health() { .expect("str"); assert_eq!(tier, "standard"); } + +// ============================================================================ +// #4971 (ASOC-INV-021): banco computes no logprobs, so a request for them is +// refused rather than answered 200 without them. +// ============================================================================ + +#[tokio::test] +#[allow(non_snake_case)] +async fn test_BANCO_HDL_004_chat_completions_refuse_logprobs() { + // (fields added to a valid request, status, words the reply must carry) + let table: &[(serde_json::Value, StatusCode, &str)] = &[ + (serde_json::json!({"logprobs": true}), StatusCode::NOT_IMPLEMENTED, "banco chat backend"), + ( + serde_json::json!({"logprobs": true, "top_logprobs": 2}), + StatusCode::NOT_IMPLEMENTED, + "banco chat backend", + ), + (serde_json::json!({"top_logprobs": 2}), StatusCode::BAD_REQUEST, "requires logprobs"), + ( + serde_json::json!({"logprobs": true, "top_logprobs": 21}), + StatusCode::UNPROCESSABLE_ENTITY, + "top_logprobs must be", + ), + (serde_json::json!({"logprobs": false}), StatusCode::OK, "chat.completion"), + ]; + for (fields, want, words) in table { + let mut body = serde_json::json!({"messages": [{"role": "user", "content": "Hello!"}]}); + for (k, v) in fields.as_object().expect("object") { + body[k] = v.clone(); + } + let response = test_app() + .oneshot( + Request::post("/api/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).expect("json"))) + .expect("req"), + ) + .await + .expect("resp"); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1_048_576).await.expect("read body"); + let reply = String::from_utf8_lossy(&bytes); + assert_eq!(status, *want, "{fields}: {reply}"); + assert!(reply.contains(words), "{fields}: {reply}"); + } +} diff --git a/crates/aprender-orchestrate/src/serve/banco/handlers_ui.rs b/crates/aprender-orchestrate/src/serve/banco/handlers_ui.rs index 22d66c4928..3c014e43b6 100644 --- a/crates/aprender-orchestrate/src/serve/banco/handlers_ui.rs +++ b/crates/aprender-orchestrate/src/serve/banco/handlers_ui.rs @@ -50,6 +50,8 @@ pub async fn chat_form_handler( attachments: vec![], tools: None, tool_choice: None, + logprobs: None, + top_logprobs: None, }; let response = super::handlers_inference::try_inference(&state, &request) diff --git a/crates/aprender-orchestrate/src/serve/banco/types_chat.rs b/crates/aprender-orchestrate/src/serve/banco/types_chat.rs index 4137df53c5..c184cb238d 100644 --- a/crates/aprender-orchestrate/src/serve/banco/types_chat.rs +++ b/crates/aprender-orchestrate/src/serve/banco/types_chat.rs @@ -37,6 +37,12 @@ pub struct BancoChatRequest { /// Tool choice: "auto", "none", or specific tool name. #[serde(default, skip_serializing_if = "Option::is_none")] pub tool_choice: Option, + /// OpenAI `logprobs`. Banco computes none, so `true` is refused (#4971). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + /// OpenAI `top_logprobs`, 0 to 20; anything else is refused when parsed (#4971). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, } /// File attachment in a chat request — text extracted and injected as context. diff --git a/crates/aprender-serve/src/api/chat_logprobs.rs b/crates/aprender-serve/src/api/chat_logprobs.rs index d6e61acc1a..80cfb244ae 100644 --- a/crates/aprender-serve/src/api/chat_logprobs.rs +++ b/crates/aprender-serve/src/api/chat_logprobs.rs @@ -76,9 +76,7 @@ impl ChatCompletionRequest { /// OpenAI refuses it; serving it would return none of what was asked for. #[must_use] pub fn logprobs_conflict(&self) -> Option { - (self.top_logprobs.is_some() && self.logprobs != Some(true)).then(|| { - "top_logprobs requires logprobs: true; send both or neither (#4971)".to_string() - }) + (self.top_logprobs.is_some() && self.logprobs != Some(true)).then(conflict_reason) } /// Every cross-field conflict the chat handler refuses with 400 before it @@ -285,12 +283,68 @@ pub(crate) fn with_logprobs( /// this, never a 200 without them. #[must_use] pub(crate) fn logprobs_refusal(request: &ChatCompletionRequest, backend: &str) -> Option { - request.logprobs_top_n().map(|_| { - format!( - "`logprobs` is not supported by the {backend} chat backend; the reply would \ - carry none. Refused rather than served (#4971, ASOC-INV-021)." - ) - }) + request.logprobs_top_n().map(|_| refusal_reason(backend)) +} + +/// The 400 reason for `top_logprobs` sent without `logprobs: true`. +fn conflict_reason() -> String { + "top_logprobs requires logprobs: true; send both or neither (#4971)".to_string() +} + +/// The 501 reason of a backend that does not compute logprobs. +fn refusal_reason(backend: &str) -> String { + format!( + "`logprobs` is not supported by the {backend} chat backend; the reply would \ + carry none. Refused rather than served (#4971, ASOC-INV-021)." + ) +} + +/// What a chat handler outside this crate's router answers for the two fields +/// when its backend does not compute logprobs (#4971, ASOC-INV-021), the same +/// as the router: `top_logprobs` without `logprobs: true` is 400, `logprobs: +/// true` is 501 naming `backend`, and anything else is served (`None`). +#[must_use] +pub fn logprobs_fields_refusal( + logprobs: Option, + top_logprobs: Option, + backend: &str, +) -> Option<(StatusCode, String)> { + if top_logprobs.is_some() && logprobs != Some(true) { + return Some((StatusCode::BAD_REQUEST, conflict_reason())); + } + (logprobs == Some(true)).then(|| (StatusCode::NOT_IMPLEMENTED, refusal_reason(backend))) +} + +/// [`logprobs_fields_refusal`] for a handler that takes the body as JSON: a +/// `logprobs` that is not true or false, or a `top_logprobs` outside 0 to +/// [`MAX_TOP_LOGPROBS`], is 422 naming the field, as the router refuses it at +/// deserialization. An absent or null field was not sent. +#[must_use] +pub fn json_logprobs_refusal( + body: &serde_json::Value, + backend: &str, +) -> Option<(StatusCode, String)> { + let sent = |name: &str| body.get(name).filter(|v| !v.is_null()); + let logprobs = match sent("logprobs").map(serde_json::Value::as_bool) { + None => None, + Some(Some(asks)) => Some(asks), + Some(None) => { + let reason = "logprobs must be true or false (#4971)".to_string(); + return Some((StatusCode::UNPROCESSABLE_ENTITY, reason)); + }, + }; + let top_logprobs = match sent("top_logprobs").map(TopLogprobs::deserialize) { + None => None, + Some(Ok(top)) => Some(top), + Some(Err(e)) => { + let reason = e.to_string(); + let reason = reason + .strip_prefix(crate::api::CLIENT_VISIBLE_MARKER) + .unwrap_or(&reason); + return Some((StatusCode::UNPROCESSABLE_ENTITY, reason.to_string())); + }, + }; + logprobs_fields_refusal(logprobs, top_logprobs, backend) } /// [`logprobs_refusal`] for a backend that computes them on its dense turn, @@ -491,6 +545,77 @@ mod tests { assert_eq!(logprobs_refusal(&plain, "CUDA"), None); } + #[test] + fn handlers_outside_the_router_refuse_the_fields_as_the_router_does() { + // (body, Some(status, words the reason must carry) / None served) + let table: &[(serde_json::Value, Option<(StatusCode, &str)>)] = &[ + (serde_json::json!({"messages": []}), None), + (serde_json::json!({"logprobs": false}), None), + ( + serde_json::json!({"logprobs": null, "top_logprobs": null}), + None, + ), + (serde_json::json!([1, 2]), None), + ( + serde_json::json!({"logprobs": true}), + Some((StatusCode::NOT_IMPLEMENTED, "WGPU chat backend")), + ), + ( + serde_json::json!({"logprobs": true, "top_logprobs": 20}), + Some((StatusCode::NOT_IMPLEMENTED, "WGPU chat backend")), + ), + ( + serde_json::json!({"top_logprobs": 3}), + Some((StatusCode::BAD_REQUEST, "requires logprobs: true")), + ), + ( + serde_json::json!({"logprobs": false, "top_logprobs": 3}), + Some((StatusCode::BAD_REQUEST, "requires logprobs: true")), + ), + ( + serde_json::json!({"logprobs": true, "top_logprobs": 21}), + Some((StatusCode::UNPROCESSABLE_ENTITY, "top_logprobs must be")), + ), + ( + serde_json::json!({"logprobs": true, "top_logprobs": "3"}), + Some((StatusCode::UNPROCESSABLE_ENTITY, "top_logprobs must be")), + ), + ( + serde_json::json!({"logprobs": "yes"}), + Some(( + StatusCode::UNPROCESSABLE_ENTITY, + "logprobs must be true or false", + )), + ), + ]; + for (body, want) in table { + let got = json_logprobs_refusal(body, "WGPU"); + match want { + None => assert_eq!(got, None, "{body}"), + Some((status, words)) => { + let (got_status, reason) = got.unwrap_or_else(|| panic!("{body} is served")); + assert_eq!(got_status, *status, "{body}: {reason}"); + assert!(reason.contains(words), "{body}: {reason}"); + assert!( + !reason.contains(crate::api::CLIENT_VISIBLE_MARKER), + "{reason}" + ); + }, + } + } + let typed = |json: &str| { + let r = request(json).expect("valid"); + logprobs_fields_refusal(r.logprobs, r.top_logprobs, "banco").map(|(s, _)| s) + }; + assert_eq!( + typed(r#","logprobs":true"#), + Some(StatusCode::NOT_IMPLEMENTED) + ); + assert_eq!(typed(r#","top_logprobs":1"#), Some(StatusCode::BAD_REQUEST)); + assert_eq!(typed(r#","logprobs":false"#), None); + assert_eq!(typed(""), None); + } + #[test] fn a_dense_backend_refuses_only_its_traced_loop() { let mut asks = request(r#","logprobs":true"#).expect("valid"); diff --git a/crates/aprender-serve/src/api/mod.rs b/crates/aprender-serve/src/api/mod.rs index c1346ed68b..45b5f22ec0 100644 --- a/crates/aprender-serve/src/api/mod.rs +++ b/crates/aprender-serve/src/api/mod.rs @@ -77,7 +77,8 @@ pub mod iteration_scheduler; // #4971: logprobs and top_logprobs on the chat route (ASOC-INV-021). mod chat_logprobs; pub use chat_logprobs::{ - ChatLogprobs, ChatTokenLogprob, ChatTopLogprob, TopLogprobs, MAX_TOP_LOGPROBS, + json_logprobs_refusal, logprobs_fields_refusal, ChatLogprobs, ChatTokenLogprob, ChatTopLogprob, + TopLogprobs, MAX_TOP_LOGPROBS, }; mod openai_handlers; mod serve_trace; From a2ec4bbd66312b8c4245a3363aa3f2ba949f5505 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:47:19 +0200 Subject: [PATCH 13/35] feat(perf-gate): V3 shape check in Rust, and the top-2 margin on the PP-26 witness (PMAT-4971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V3-d. perf_gate::v3_shape ports the hand-run v3_shape_check.py from the V3 spec branch: a perf041 witness is a serving-shape parity receipt only for the blessed Qwen3.5-4B (prefix, sha256), with identity fields, bands c in {1,4,8,16} each once with m_formed == c and PASS, and agreement with m=1 to declared_min unless a recorded top-2 margin below 0.05 explains a near-tie. Exit 0/1/2 as in the reference. It closes the draft contract's findings: unreadable is RED (F1), typed fields (F2), no declared_min is not admissible (F3), the reason names the margin and eps (F4), a band with no divergence measurement reports its result only (F5), a duplicate c is not admissible and every copy is checked (F6). The case table also plants the two mutants the Python table let survive (prefix vs substring, a defaulted declared_min). BatchInvarianceWitness gains top2_margin_at_divergence (master §12 row 22), None from token comparison and off the wire when None, so older receipts are byte-identical; it drops Eq for the f64. apr test llm bench reads it from the perf041 band and refuses a non-number by name. Not a gate: nothing on a merge, queue or release path runs it. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit f0156ba1b98ef0079e3f74dd1fc878b755a02090) (cherry picked from commit 875a16ee0e845693f7e266cf46169febad38c068) (cherry picked from commit a6d0260d05ced67cae4fe145bf08fb46126569ab) --- contracts/apr-serve-openai-compat-v1.yaml | 12 + crates/apr-cli/src/commands/test_llm_band.rs | 42 ++ crates/aprender-test-lib/src/perf_gate/mod.rs | 4 + .../src/perf_gate/v3_shape.rs | 529 ++++++++++++++++++ .../src/perf_gate/witness.rs | 32 +- 5 files changed, 617 insertions(+), 2 deletions(-) create mode 100644 crates/aprender-test-lib/src/perf_gate/v3_shape.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 89949abee5..28803f7045 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -688,3 +688,15 @@ falsification_tests: test_harness: 'cargo test -p aprender-orchestrate --features banco --lib refuse_logprobs' expected_output: "test result: ok" if_fails: 'banco''s request type dropped the two fields as unknown keys, so a client asking for logprobs got a dry-run or model reply without them, a 200 that ASOC-INV-021 forbids.' + - id: FALSIFY-WITNESS-MARGIN-4971 + name: the_pp26_witness_carries_the_top2_margin_it_was_given + prediction: 'BatchInvarianceWitness has an optional top2_margin_at_divergence (master §12 row 22). A recorded margin survives the receipt round trip; a witness without one serialises byte-identical to before (the field stays off the wire); a non-number there does not deserialise; and apr test llm bench --witness-json reads it from the perf041 band, refusing a value that is not a number or null by name.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::witness && cargo test -p apr-cli --lib test_llm_band' + expected_output: "test result: ok" + if_fails: 'The margin the logprobs routes now put on the wire is dropped between the probe and the receipt, so a near-tie flip at c > 1 still cannot be told from a wrong-KV divergence.' + - id: FALSIFY-V3-SHAPE-CHECK-4971 + name: a_v3_receipt_check_refuses_every_planted_shape_defect + prediction: 'perf_gate::v3_shape::check_v3_shape is ADMISSIBLE (exit 0) only for a perf041 witness of the blessed Qwen3.5-4B (path prefix, sha256) with identity fields, bands c in {1,4,8,16} each once with m_formed == c and perf041 PASS, and agreement with m=1 to declared_min unless a recorded top-2 margin m with 0 <= m < 0.05 explains the divergence. Its in-code case table (the 19 object rows of the spec branch cases.json plus the F2, F3, F6, prefix-not-substring, boundary and negative-margin rows) gets each exit and exact reason; an empty, blank, malformed, non-object or missing witness is RED (exit 2), never NOT ADMISSIBLE (F1). This is not a gate: no merge, queue or release job runs it.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A witness that served a shape other than the one advertised (c=16 forming m=14), measured another model, or diverged from m=1 with no near-tie to explain it is read as a V3 receipt; or an unreadable witness reads as a verdict about the server.' diff --git a/crates/apr-cli/src/commands/test_llm_band.rs b/crates/apr-cli/src/commands/test_llm_band.rs index 750a75bbd8..a20ac51827 100644 --- a/crates/apr-cli/src/commands/test_llm_band.rs +++ b/crates/apr-cli/src/commands/test_llm_band.rs @@ -795,12 +795,28 @@ fn witness_of(band: &Value, source: &str, path: &Path) -> Result Result> { + match band.get("top2_margin_at_divergence") { + None | Some(Value::Null) => Ok(None), + Some(v) => v.as_f64().map(Some).ok_or_else(|| { + CliError::InvalidFormat(format!( + "--witness-json {}: top2_margin_at_divergence {v} is not a number (#4971)", + path.display() + )) + }), + } +} + /// §4.2.2 — who measured, what served, and what the pinned comparator was. /// /// `binary_*` stay the CLIENT's, unchanged from v2.2 so existing readers keep @@ -2674,6 +2690,32 @@ mod tests { assert!(err.to_string().contains("SKIP"), "{err}"); } + #[test] + fn a_recorded_top2_margin_is_read_and_a_non_number_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = witness_file( + dir.path(), + r#"{"bands":[ + {"c":1,"result":"PASS","divergence_at":null,"declared_min":64,"m_formed":1}, + {"c":4,"result":"PASS","divergence_at":3,"declared_min":64,"m_formed":4, + "top2_margin_at_divergence":0.01} + ]}"#, + ); + let w = load_witness(Some(&path)).expect("loads"); + assert_eq!(w[&1].top2_margin_at_divergence, None); + assert_eq!(w[&4].top2_margin_at_divergence, Some(0.01)); + let path = witness_file( + dir.path(), + r#"{"bands":[{"c":4,"result":"PASS","divergence_at":3,"declared_min":64, + "m_formed":4,"top2_margin_at_divergence":"0.01"}]}"#, + ); + let err = load_witness(Some(&path)).expect_err("a string is not a margin"); + assert!( + err.to_string().contains("top2_margin_at_divergence"), + "{err}" + ); + } + #[test] fn a_document_that_is_not_a_perf041_witness_is_refused() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/aprender-test-lib/src/perf_gate/mod.rs b/crates/aprender-test-lib/src/perf_gate/mod.rs index 0341e49ea0..e84aa61da0 100644 --- a/crates/aprender-test-lib/src/perf_gate/mod.rs +++ b/crates/aprender-test-lib/src/perf_gate/mod.rs @@ -172,6 +172,7 @@ pub mod receipt; pub mod replicate; pub mod samples; pub mod ttft_verdict; +pub mod v3_shape; pub mod window; pub mod witness; @@ -203,6 +204,9 @@ pub use receipt::{ }; pub use replicate::{log_ratio_lcb, t_lower_one_sided_95, ArmOrder, ReplicatePair, MIN_REPLICATES}; pub use samples::{read_samples_gz, write_samples_gz, SamplesFile}; +pub use v3_shape::{ + check_v3_shape, check_v3_shape_file, ShapeVerdict, BLESSED_MODEL_PREFIX, NEAR_TIE_EPS, V3_BANDS, +}; pub use window::{WindowController, WindowReport}; pub use witness::{BatchInvariance, BatchInvarianceWitness}; diff --git a/crates/aprender-test-lib/src/perf_gate/v3_shape.rs b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs new file mode 100644 index 0000000000..cdf77c612c --- /dev/null +++ b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs @@ -0,0 +1,529 @@ +//! 0.71 V3 — is a perf041 witness a serving-shape parity receipt? (#4971 V3-d) +//! +//! perf041 (PP-26 v3.1) answers one question: is the batch consistent with +//! itself? A V3 receipt asks more of the same witness, and this module is the +//! check. It ports the hand-run `v3_shape_check.py` from the spec branch +//! (`docs/lookahead/0.71-v3-serving-shape-parity.md`, draft contract +//! `serving-shape-parity-v1`, branch `la-71/v3-serving-shape-parity`) and +//! carries its case table. No Python ships with it. +//! +//! | Rule | The witness holds | +//! |---|---| +//! | S1 | the blessed Qwen3.5-4B, with its sha256 | +//! | S2 | every band `c` in {1, 4, 8, 16} | +//! | S3 | `m_formed == c` in every band: a band that formed fewer served another shape | +//! | S4 | identity: `binary_sha256`, `commit`, `host`, `prompt_sha256` | +//! | S5 | agreement with the `m = 1` reference to `declared_min`, unless a recorded top-2 margin below ε explains the divergence as a near-tie | +//! | S6 | perf041 PASS in every band | +//! +//! S7 (FALSIFY-CB-004, per-request decode at c=4 at least half of c=1) needs +//! timing the witness does not carry. The receipt cites it beside the witness, +//! and this check does not read it. +//! +//! The port closes the draft contract's findings: +//! * **F1** an unreadable witness is RED (exit 2), never NOT ADMISSIBLE; +//! * **F2** fields are typed, so `true` is not a batch width and `false` is not a margin; +//! * **F3** a band with no `declared_min` is not admissible; +//! * **F4** when a margin was recorded, the S5 reason names it and ε; +//! * **F5** a band with no divergence measurement reports only its result; +//! * **F6** a band `c` recorded twice is not admissible, and every copy is checked. +//! +//! This is not a gate. No merge, queue or release job runs it; putting it +//! beside `check_perf041_marker.sh` is a gate change with its own ruling. + +use std::collections::BTreeMap; +use std::path::Path; + +use serde::Deserialize; + +/// S1: the blessed model's file name starts with this, compared without case. +pub const BLESSED_MODEL_PREFIX: &str = "qwen3.5-4b"; + +/// S2: the bands `apr serve` advertises. +pub const V3_BANDS: [u32; 4] = [1, 4, 8, 16]; + +/// S5: ε in logits, `[A]` until measured near-ties calibrate it. A recorded +/// top-2 margin explains a divergence only when `0 <= margin < ε`. +pub const NEAR_TIE_EPS: f64 = 0.05; + +/// The check's answer. Exit codes follow the Python reference: 0, 1, 2. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShapeVerdict { + /// Every rule S1–S6 holds. + Admissible, + /// The witness was read, and these rules failed. + NotAdmissible(Vec), + /// The witness could not be read, or it measured nothing. + Red(String), +} + +impl ShapeVerdict { + /// 0 ADMISSIBLE, 1 NOT ADMISSIBLE, 2 RED. + #[must_use] + pub fn exit_code(&self) -> i32 { + match self { + Self::Admissible => 0, + Self::NotAdmissible(_) => 1, + Self::Red(_) => 2, + } + } + + /// The verdict word printed on the last line. + #[must_use] + pub fn label(&self) -> &'static str { + match self { + Self::Admissible => "ADMISSIBLE", + Self::NotAdmissible(_) => "NOT ADMISSIBLE", + Self::Red(_) => "RED", + } + } + + /// Every reason, in rule order. Empty only for [`Self::Admissible`]. + #[must_use] + pub fn reasons(&self) -> Vec { + match self { + Self::Admissible => Vec::new(), + Self::NotAdmissible(reasons) => reasons.clone(), + Self::Red(reason) => vec![reason.clone()], + } + } +} + +/// The fields of a perf041 witness that V3 reads. Every other field is +/// allowed and ignored: the witness belongs to the probe, not to this check. +#[derive(Debug, Deserialize)] +struct ShapeWitness { + model: Option, + binary_sha256: Option, + commit: Option, + host: Option, + prompt_sha256: Option, + bands: Option>, +} + +#[derive(Debug, Deserialize)] +struct ShapeModel { + path: Option, + sha256: Option, +} + +#[derive(Debug, Deserialize)] +struct ShapeBand { + c: u32, + result: Option, + m_formed: Option, + divergence_at: Option, + declared_min: Option, + top2_margin_at_divergence: Option, +} + +/// Check the witness file at `path`. A file that cannot be read is RED (F1). +#[must_use] +pub fn check_v3_shape_file(path: &Path) -> ShapeVerdict { + match std::fs::read_to_string(path) { + Ok(text) => check_v3_shape(&text), + Err(e) => ShapeVerdict::Red(format!("RED: cannot read witness {}: {e}", path.display())), + } +} + +/// Check one witness, given as JSON text. +#[must_use] +pub fn check_v3_shape(text: &str) -> ShapeVerdict { + if text.trim().is_empty() { + return ShapeVerdict::Red("RED: empty witness".to_string()); + } + let witness: ShapeWitness = match serde_json::from_str(text) { + Ok(witness) => witness, + Err(e) => return ShapeVerdict::Red(format!("RED: unreadable witness: {e}")), + }; + let Some(bands) = witness.bands.as_deref().filter(|bands| !bands.is_empty()) else { + return ShapeVerdict::Red("RED: the witness measured no bands".to_string()); + }; + let reasons: Vec = model_reasons(witness.model.as_ref()) + .into_iter() + .chain(identity_reasons(&witness)) + .chain(bands_reasons(bands)) + .collect(); + if reasons.is_empty() { + ShapeVerdict::Admissible + } else { + ShapeVerdict::NotAdmissible(reasons) + } +} + +/// S1: the blessed model, and its sha256. +fn model_reasons(model: Option<&ShapeModel>) -> Vec { + let path = model.and_then(|m| m.path.as_deref()).unwrap_or(""); + let mut out = Vec::new(); + if !path.to_ascii_lowercase().starts_with(BLESSED_MODEL_PREFIX) { + out.push(format!("model {path:?} is not the blessed Qwen3.5-4B")); + } + if model.and_then(|m| m.sha256.as_deref()).is_none_or(blank) { + out.push("model sha256 missing".to_string()); + } + out +} + +/// S4: every identity field present and not blank. +fn identity_reasons(witness: &ShapeWitness) -> Vec { + [ + ("binary_sha256", &witness.binary_sha256), + ("commit", &witness.commit), + ("host", &witness.host), + ("prompt_sha256", &witness.prompt_sha256), + ] + .into_iter() + .filter(|(_, value)| value.as_deref().is_none_or(blank)) + .map(|(name, _)| format!("identity field {name} missing")) + .collect() +} + +fn blank(value: &str) -> bool { + value.trim().is_empty() +} + +/// S2, F6 and the per-band rules, in band order. +fn bands_reasons(bands: &[ShapeBand]) -> Vec { + let mut by_c: BTreeMap> = BTreeMap::new(); + for band in bands { + by_c.entry(band.c).or_default().push(band); + } + let mut out = Vec::new(); + for c in V3_BANDS { + let copies = by_c.get(&c).map_or(&[][..], Vec::as_slice); + if copies.is_empty() { + out.push(format!("band c={c} not measured")); + } + if copies.len() > 1 { + out.push(format!( + "band c={c} recorded {} times; a witness has one band per c", + copies.len() + )); + } + for band in copies { + out.extend(band_reasons(band)); + } + } + out +} + +/// S6, S3 and S5 for one band. +fn band_reasons(band: &ShapeBand) -> Vec { + let c = band.c; + let result = band.result.as_deref(); + let mut out = Vec::new(); + if result != Some("PASS") { + out.push(format!( + "c={c}: perf041 result {}", + result.unwrap_or("not recorded") + )); + } + match band.m_formed { + Some(m) if m == c => {} + Some(m) => out.push(format!("c={c}: m_formed {m} != c (shape not served)")), + None => out.push(format!("c={c}: m_formed not recorded (shape not shown)")), + } + out.extend(divergence_reason(band, result == Some("PASS"))); + out +} + +/// S5, with F3 (no `declared_min`), F4 (name the recorded margin) and F5 (no +/// divergence measurement on a band that did not pass: its result says it). +fn divergence_reason(band: &ShapeBand, passed: bool) -> Option { + let c = band.c; + let Some(min) = band.declared_min else { + return Some(format!( + "c={c}: declared_min not recorded, so agreement with m=1 cannot be checked" + )); + }; + let Some(at) = band.divergence_at else { + return passed.then(|| { + format!("c={c}: divergence_at not recorded on a PASS band, so agreement with m=1 is unmeasured") + }); + }; + if at >= min { + return None; + } + match band.top2_margin_at_divergence { + None => Some(format!( + "c={c}: diverges from m=1 at {at} < {min} with no near-tie margin recorded" + )), + Some(margin) if (0.0..NEAR_TIE_EPS).contains(&margin) => None, + Some(margin) => Some(format!( + "c={c}: diverges from m=1 at {at} < {min}; the recorded top-2 margin {margin} \ + is not a near-tie (needs 0 <= margin < {NEAR_TIE_EPS})" + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + /// The admissible control, as `fixtures/v3/admissible.json` on the spec + /// branch has it. Every case below changes one thing in it. + fn admissible() -> Value { + let band = |c: u32| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127}); + json!({ + "_planted": "FIXTURE: not a measurement", + "binary_sha256": "a".repeat(64), + "commit": "f".repeat(40), + "host": "lambda", + "prompt_sha256": "b".repeat(64), + "model": {"path": "Qwen3.5-4B-Q4_K_M.gguf", "sha256": "c".repeat(64)}, + "bands": [band(1), band(4), band(8), band(16)], + }) + } + + /// The band at `index` of the control (0..4 is c = 1, 4, 8, 16). + fn band_mut(w: &mut Value, index: usize) -> &mut serde_json::Map { + w["bands"][index] + .as_object_mut() + .expect("a band is an object") + } + + fn set(index: usize, key: &str, value: Value) -> impl Fn(&mut Value) { + let key = key.to_string(); + move |w: &mut Value| { + band_mut(w, index).insert(key.clone(), value.clone()); + } + } + + fn remove_top(key: &'static str) -> impl Fn(&mut Value) { + move |w: &mut Value| { + w.as_object_mut().expect("object").remove(key); + } + } + + fn drop_band(index: usize) -> impl Fn(&mut Value) { + move |w: &mut Value| { + w["bands"].as_array_mut().expect("bands").remove(index); + } + } + + fn diverge(index: usize, at: u32, margin: Option) -> impl Fn(&mut Value) { + move |w: &mut Value| { + let band = band_mut(w, index); + band.insert("divergence_at".into(), json!(at)); + if let Some(margin) = &margin { + band.insert("top2_margin_at_divergence".into(), margin.clone()); + } + } + } + + type Mutation = Box; + + /// One planted case: a change to the control, the exit code it must give, + /// and its exact reasons. The first 19 rows are the spec branch's + /// `cases.json` (its `empty` and `blank` rows are in the RED test below); + /// rows marked F2–F6 are the findings, and their wording is this port's. + fn cases() -> Vec<(&'static str, Mutation, i32, Vec<&'static str>)> { + vec![ + ("admissible", Box::new(|_: &mut Value| {}), 0, vec![]), + ("near-tie", Box::new(diverge(1, 3, Some(json!(0.01)))), 0, vec![]), + ("short-band", Box::new(set(3, "m_formed", json!(14))), 1, vec!["c=16: m_formed 14 != c (shape not served)"]), + ( + "wrong-model", + Box::new(|w: &mut Value| w["model"]["path"] = json!("qwen2.5-coder-1.5b-instruct-q4_k_m.gguf")), + 1, + vec!["model \"qwen2.5-coder-1.5b-instruct-q4_k_m.gguf\" is not the blessed Qwen3.5-4B"], + ), + ("unexplained-divergence", Box::new(diverge(1, 3, None)), 1, vec!["c=4: diverges from m=1 at 3 < 64 with no near-tie margin recorded"]), + ("no-identity-binary-sha256", Box::new(remove_top("binary_sha256")), 1, vec!["identity field binary_sha256 missing"]), + ("no-identity-commit", Box::new(remove_top("commit")), 1, vec!["identity field commit missing"]), + ("no-identity-host", Box::new(remove_top("host")), 1, vec!["identity field host missing"]), + ("no-identity-prompt-sha256", Box::new(remove_top("prompt_sha256")), 1, vec!["identity field prompt_sha256 missing"]), + ( + "no-model-sha256", + Box::new(|w: &mut Value| { + w["model"].as_object_mut().expect("model").remove("sha256"); + }), + 1, + vec!["model sha256 missing"], + ), + ( + "margin-at-epsilon (F4 wording)", + Box::new(diverge(1, 3, Some(json!(0.05)))), + 1, + vec!["c=4: diverges from m=1 at 3 < 64; the recorded top-2 margin 0.05 is not a near-tie (needs 0 <= margin < 0.05)"], + ), + ("failed-band", Box::new(set(2, "result", json!("FAIL"))), 1, vec!["c=8: perf041 result FAIL"]), + ("missing-band-c1", Box::new(drop_band(0)), 1, vec!["band c=1 not measured"]), + ("missing-band-c4", Box::new(drop_band(1)), 1, vec!["band c=4 not measured"]), + ("missing-band-c8", Box::new(drop_band(2)), 1, vec!["band c=8 not measured"]), + ("missing-band-c16", Box::new(drop_band(3)), 1, vec!["band c=16 not measured"]), + ("divergence-at-min", Box::new(diverge(1, 64, None)), 0, vec![]), + ("no-bands", Box::new(|w: &mut Value| w["bands"] = json!([])), 2, vec!["RED: the witness measured no bands"]), + ( + "unmeasurable-band (F5: its result only)", + Box::new(|w: &mut Value| { + let band = band_mut(w, 2); + band.insert("result".into(), json!("UNMEASURABLE")); + band.insert("divergence_at".into(), Value::Null); + }), + 1, + vec!["c=8: perf041 result UNMEASURABLE"], + ), + // F2: typed fields. A boolean is not a width and not a margin. + ("F2 m_formed true", Box::new(set(0, "m_formed", json!(true))), 2, vec![]), + ("F2 margin false", Box::new(diverge(1, 3, Some(json!(false)))), 2, vec![]), + ("F2 c as a string", Box::new(set(1, "c", json!("4"))), 2, vec![]), + // F3: no declared_min, so S5 cannot hold. + ( + "F3 no declared_min", + Box::new(|w: &mut Value| { + band_mut(w, 1).remove("declared_min"); + }), + 1, + vec!["c=4: declared_min not recorded, so agreement with m=1 cannot be checked"], + ), + // F6: a FAIL first and a PASS second; the FAIL is not masked. + ( + "F6 duplicate c", + Box::new(|w: &mut Value| { + let mut failed = w["bands"][2].clone(); + failed["result"] = json!("FAIL"); + w["bands"].as_array_mut().expect("bands").insert(2, failed); + }), + 1, + vec!["band c=8 recorded 2 times; a witness has one band per c", "c=8: perf041 result FAIL"], + ), + // The mutants the Python table let survive, and the other boundaries. + ( + "prefix, not substring", + Box::new(|w: &mut Value| w["model"]["path"] = json!("not-qwen3.5-4b.gguf")), + 1, + vec!["model \"not-qwen3.5-4b.gguf\" is not the blessed Qwen3.5-4B"], + ), + ("divergence one below min", Box::new(diverge(1, 63, None)), 1, vec!["c=4: diverges from m=1 at 63 < 64 with no near-tie margin recorded"]), + ("margin just below epsilon", Box::new(diverge(1, 3, Some(json!(0.0499)))), 0, vec![]), + ("margin zero (an exact tie)", Box::new(diverge(1, 3, Some(json!(0)))), 0, vec![]), + ( + "negative margin", + Box::new(diverge(1, 3, Some(json!(-0.01)))), + 1, + vec!["c=4: diverges from m=1 at 3 < 64; the recorded top-2 margin -0.01 is not a near-tie (needs 0 <= margin < 0.05)"], + ), + ( + "PASS band without divergence_at", + Box::new(set(3, "divergence_at", Value::Null)), + 1, + vec!["c=16: divergence_at not recorded on a PASS band, so agreement with m=1 is unmeasured"], + ), + ("m_formed null", Box::new(set(1, "m_formed", Value::Null)), 1, vec!["c=4: m_formed not recorded (shape not shown)"]), + ( + "result missing", + Box::new(|w: &mut Value| { + band_mut(w, 0).remove("result"); + }), + 1, + vec!["c=1: perf041 result not recorded"], + ), + ( + "blank identity", + Box::new(|w: &mut Value| w["host"] = json!(" ")), + 1, + vec!["identity field host missing"], + ), + ( + "model missing", + Box::new(remove_top("model")), + 1, + vec!["model \"\" is not the blessed Qwen3.5-4B", "model sha256 missing"], + ), + ( + "a band outside V3 is ignored", + Box::new(|w: &mut Value| { + let mut extra = w["bands"][0].clone(); + extra["c"] = json!(2); + extra["result"] = json!("FAIL"); + w["bands"].as_array_mut().expect("bands").push(extra); + }), + 0, + vec![], + ), + ] + } + + /// FALSIFY-SSP-001..006, 008..016 and F1–F6: every planted case exits as + /// its row says and prints exactly its row's reasons. A RED row with no + /// reasons listed checks the exit code and the `RED:` prefix. + #[test] + fn every_planted_case_exits_as_its_row_says_with_exactly_its_reasons() { + let cases = cases(); + assert!( + cases.len() >= 19, + "the case table holds at least the spec's 19 object rows" + ); + let mut ids: Vec<_> = cases.iter().map(|(id, ..)| *id).collect(); + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), cases.len(), "case ids are unique"); + for (id, mutate, want_exit, want_reasons) in cases { + let mut w = admissible(); + mutate(&mut w); + let verdict = check_v3_shape(&w.to_string()); + assert_eq!(verdict.exit_code(), want_exit, "{id}: {verdict:?}"); + if want_exit == 2 { + let reasons = verdict.reasons(); + assert!( + reasons.len() == 1 && reasons[0].starts_with("RED: "), + "{id}: {reasons:?}" + ); + } + if want_exit < 2 || !want_reasons.is_empty() { + assert_eq!(verdict.reasons(), want_reasons, "{id}"); + } + } + } + + /// F1 and FALSIFY-SSP-006/016: empty, blank, malformed, a JSON value that + /// is not an object, and a file that does not exist are all RED. + #[test] + fn an_unreadable_witness_is_red_never_not_admissible() { + for text in ["", "\n", " \t\n", "{", "[]", "3", "null", "{\"bands\": 4}"] { + let verdict = check_v3_shape(text); + assert_eq!(verdict.exit_code(), 2, "{text:?}: {verdict:?}"); + assert_eq!(verdict.label(), "RED"); + } + assert_eq!( + check_v3_shape("{}"), + ShapeVerdict::Red("RED: the witness measured no bands".to_string()) + ); + let missing = Path::new("/nonexistent/la-0.71.1a/v3-shape/witness.json"); + let verdict = check_v3_shape_file(missing); + assert_eq!(verdict.exit_code(), 2, "{verdict:?}"); + assert!(verdict.reasons()[0].starts_with("RED: cannot read witness ")); + } + + /// The file entry point reads what the text entry point checks. + #[test] + fn the_file_entry_point_checks_the_file() { + let dir = std::env::temp_dir().join(format!("v3-shape-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("witness.json"); + std::fs::write(&path, admissible().to_string()).expect("write"); + assert_eq!(check_v3_shape_file(&path), ShapeVerdict::Admissible); + std::fs::remove_file(&path).expect("remove"); + let _ = std::fs::remove_dir(&dir); + } + + #[test] + fn exit_codes_and_labels_follow_the_python_reference() { + let rows = [ + (ShapeVerdict::Admissible, 0, "ADMISSIBLE"), + ( + ShapeVerdict::NotAdmissible(vec!["x".into()]), + 1, + "NOT ADMISSIBLE", + ), + (ShapeVerdict::Red("RED: x".into()), 2, "RED"), + ]; + for (verdict, code, label) in rows { + assert_eq!(verdict.exit_code(), code); + assert_eq!(verdict.label(), label); + } + assert!(ShapeVerdict::Admissible.reasons().is_empty()); + } +} diff --git a/crates/aprender-test-lib/src/perf_gate/witness.rs b/crates/aprender-test-lib/src/perf_gate/witness.rs index 8ca253c941..b466f3a50f 100644 --- a/crates/aprender-test-lib/src/perf_gate/witness.rs +++ b/crates/aprender-test-lib/src/perf_gate/witness.rs @@ -29,7 +29,9 @@ //! kernel family is batch-size invariant to the end while the families differ //! from each other (`evidence/perf041/lambda/`). Without a top-2 margin on the //! wire a near-tie flip and a wrong-KV divergence are indistinguishable, so that -//! comparison stays a report until master §12 row 22 lands. +//! comparison stays a report here. Master §12 row 22 adds the margin as +//! `top2_margin_at_divergence`; the V3 shape check ([`super::v3_shape`], #4971) +//! reads it to tell a near-tie flip from a defect, and is not a gate either. //! //! Three verdicts, not two. "The slots agree but every stream stopped at 32 //! tokens" is **not** a pass — it is [`BatchInvariance::Unmeasurable`], because @@ -69,7 +71,7 @@ impl BatchInvariance { } /// PP-26 — one band's correctness witness, as it appears in the receipt. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BatchInvarianceWitness { /// The verdict. @@ -86,6 +88,13 @@ pub struct BatchInvarianceWitness { /// on a witness written before v3.1. #[serde(default, skip_serializing_if = "Option::is_none")] pub max_constant_run: Option, + /// Master §12 row 22: the `m = 1` stream's top-2 logit margin at + /// `divergence_at`, when the probe recorded one (#4971). Below + /// [`super::v3_shape::NEAR_TIE_EPS`] it marks the divergence a near-tie + /// flip, not a defect (V3 S5). Token comparison cannot measure it, so + /// [`Self::compare_batch`] leaves it `None`, and `None` stays off the wire. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top2_margin_at_divergence: Option, /// Tokens that had to agree, from `perf-matrix.yaml`. pub declared_min: u32, /// The batch size the batched arm actually formed. `0` when the caller has @@ -134,6 +143,7 @@ impl BatchInvarianceWitness { divergence_at: None, intra_agree_to: None, max_constant_run: None, + top2_margin_at_divergence: None, declared_min, m_formed: 0, source: source.to_string(), @@ -168,6 +178,7 @@ impl BatchInvarianceWitness { divergence_at, intra_agree_to: Some(intra), max_constant_run: Some(run), + top2_margin_at_divergence: None, declared_min, m_formed: 0, source: source.to_string(), @@ -330,6 +341,23 @@ mod tests { BatchInvarianceWitness::compare_batch(&[1, 2, 3], &[&[1, 2, 3], &[1, 2, 3]], 2, 16); let j = serde_json::to_string(&new).expect("serialises"); assert!(j.contains("\"intra_agree_to\":3"), "{j}"); + assert!(!j.contains("top2_margin_at_divergence"), "{j}"); + } + + /// #4971: a recorded margin survives the receipt round trip, and a witness + /// with a margin that is not a number does not read. + #[test] + fn a_recorded_top2_margin_round_trips_and_must_be_a_number() { + let mut w = BatchInvarianceWitness::compare(&[1, 2, 3], &[1, 9, 3], 1); + assert_eq!(w.divergence_at, Some(1)); + assert_eq!(w.top2_margin_at_divergence, None); + w.top2_margin_at_divergence = Some(0.0125); + let j = serde_json::to_string(&w).expect("serialises"); + assert!(j.contains("\"top2_margin_at_divergence\":0.0125"), "{j}"); + let back: BatchInvarianceWitness = serde_json::from_str(&j).expect("reads back"); + assert_eq!(back, w); + let wrong = j.replace("0.0125", "true"); + assert!(serde_json::from_str::(&wrong).is_err()); } /// Two identical but SHORT streams prove nothing. Reading that as a pass is From 2f6bc5fe09a8ef0f09899b69c498a2d18a6262a9 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 07:55:55 +0200 Subject: [PATCH 14/35] feat(apr-cli): `apr test llm shape-check`, the CLI over the V3 shape check (PMAT-4971) defect. perf_gate::v3_shape (a50cc1d81a) is that check as a library; this is its front door: apr test llm shape-check It prints the verdict and every reason, and exits by apr's convention: ADMISSIBLE 0; NOT ADMISSIBLE 5 (ValidationFailed); a missing path or a directory 3; an empty, malformed or bandless witness 4, never 5, so the draft contract's F1 holds across the mapping. It reads a file and needs no runtime, so it routes before the tokio runtime and works in a build without `inference`; the two bench modes take the command by let-else. Tests (commands::test_llm_shape): the control witness exits 0; #4971's falsifier rows (a band at m_formed = c - 1, a divergence before declared_min with no margin) exit 5 naming the band; the same divergence at margin 0.01 exits 0; five unreadable shapes exit 4; missing and directory paths exit 3. Falsifier FALSIFY-V3-SHAPE-CLI-4971. Registry prose for `test` names the new route. validate.rs lists both llm subcommands explicitly (neither names a model file). Not a gate: no merge, queue or release job runs it. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 98cafc22316975e01681a2672791af2283dda5e7) (cherry picked from commit 6e40a4dd2d28bb0381a8af7ca7d534a5d76dd328) (cherry picked from commit 788865367ec1b509502924df7af92e0379bb8747) --- contracts/apr-cli-commands-v1.yaml | 6 +- contracts/apr-serve-openai-compat-v1.yaml | 6 + crates/apr-cli/src/commands/mod.rs | 2 + crates/apr-cli/src/commands/test_llm.rs | 38 ++-- crates/apr-cli/src/commands/test_llm_shape.rs | 174 ++++++++++++++++++ crates/apr-cli/src/extended_commands.rs | 17 ++ crates/apr-cli/src/validate.rs | 11 +- 7 files changed, 233 insertions(+), 21 deletions(-) create mode 100644 crates/apr-cli/src/commands/test_llm_shape.rs diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index dc6445bc22..6d656e5a7b 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -486,8 +486,10 @@ commands: replay the runner itself (record, playbook, coverage, report) Routed today: `tensor` (PMAT-481 visual regression), `llm bench` - (GH-876 Milestone 2) and `llm ttft-verdict` (#4954). The rest land as they - are delegated to the probador library. + (GH-876 Milestone 2), `llm ttft-verdict` (#4954) and `llm shape-check` + (#4971: checks a perf041 witness against the V3 serving-shape rule; reads + a file, no network). The rest land as they are delegated to the probador + library. requires_model: false side_effects: [filesystem, network] # tensor — export tensor activations for visual regression testing (PMAT-481) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 28803f7045..fd33626381 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -700,3 +700,9 @@ falsification_tests: test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' expected_output: "test result: ok" if_fails: 'A witness that served a shape other than the one advertised (c=16 forming m=14), measured another model, or diverged from m=1 with no near-tie to explain it is read as a V3 receipt; or an unreadable witness reads as a verdict about the server.' + - id: FALSIFY-V3-SHAPE-CLI-4971 + name: apr_test_llm_shape_check_exits_by_the_verdict + prediction: 'apr test llm shape-check runs perf_gate::v3_shape on the file and maps its verdict onto apr''s exit codes: ADMISSIBLE exits 0; NOT ADMISSIBLE prints every reason and exits 5 (ValidationFailed); a missing path or a directory exits 3; an empty, blank, malformed or bandless witness exits 4 and never 5 (F1 across the mapping). #4971''s falsifier rows hold through the CLI: one band at m_formed = c - 1, or one divergence before declared_min with no margin, exits 5 and names the band; the same divergence with a recorded margin of 0.01 exits 0. This is not a gate: no merge, queue or release job runs it.' + test_harness: 'cargo test -p apr-cli --lib test_llm_shape' + expected_output: "test result: ok" + if_fails: 'The Rust check exists but #4971''s "exits non-zero" has no front door: an operator runs the Python reference by hand, or a NOT ADMISSIBLE witness and an unreadable one leave the same exit code.' diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index a8882c9a5f..fa5327a510 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -157,6 +157,8 @@ pub(crate) mod test_llm; pub(crate) mod test_llm_band; // #4954: `apr test llm ttft-verdict`, the TTFT ratio row as an exit code. pub(crate) mod test_llm_ttft; +// #4971 V3-d: `apr test llm shape-check`, the CLI over perf_gate::v3_shape. +pub(crate) mod test_llm_shape; // #2399: gated on the crate it actually needs (aprender-explain, aliased // `trueno-explain`) rather than on `full`, so `--features ptx` is enough and a // user does not have to pull CUDA + training to analyze a .ptx file. diff --git a/crates/apr-cli/src/commands/test_llm.rs b/crates/apr-cli/src/commands/test_llm.rs index 085caf7941..0cded9aa3c 100644 --- a/crates/apr-cli/src/commands/test_llm.rs +++ b/crates/apr-cli/src/commands/test_llm.rs @@ -467,15 +467,23 @@ mod tests { // =========================================================================== use crate::LlmSubcommand; -/// Route `apr test llm ` (GH-876 Milestone 2; PERF-025 band mode). +/// Route `apr test llm ` (GH-876 Milestone 2; PERF-025 band mode; +/// #4954 ttft-verdict and #4971 shape-check, which read files and need no +/// runtime). /// /// # Errors /// Propagates whichever mode ran. -#[cfg(feature = "inference")] pub fn dispatch(command: &LlmSubcommand) -> Result<()> { - if let LlmSubcommand::TtftVerdict { runs } = command { - return super::test_llm_ttft::run(runs); + match command { + LlmSubcommand::TtftVerdict { runs } => super::test_llm_ttft::run(runs), + LlmSubcommand::ShapeCheck { witness } => super::test_llm_shape::run(witness), + LlmSubcommand::Bench { .. } => dispatch_bench(command), } +} + +/// `apr test llm bench`, in either of its two modes. +#[cfg(feature = "inference")] +fn dispatch_bench(command: &LlmSubcommand) -> Result<()> { let rt = tokio::runtime::Runtime::new() .map_err(|e| CliError::InferenceFailed(format!("tokio runtime: {e}")))?; match command { @@ -486,21 +494,25 @@ pub fn dispatch(command: &LlmSubcommand) -> Result<()> { /// #4041: the benchmark drives its server through tokio, which only the `inference` feature brings. A minimal /// build (`--no-default-features`) names the missing feature instead of failing to compile. -/// `ttft-verdict` reads files only, so it runs in every build (#4954). +/// `ttft-verdict` and `shape-check` read files only and never reach here, so +/// they run in every build (#4954, #4971). /// /// # Errors /// `bench`, always: this build cannot run the benchmark. #[cfg(not(feature = "inference"))] -pub fn dispatch(command: &LlmSubcommand) -> Result<()> { - if let LlmSubcommand::TtftVerdict { runs } = command { - return super::test_llm_ttft::run(runs); - } +fn dispatch_bench(_command: &LlmSubcommand) -> Result<()> { Err(CliError::InferenceFailed( "`apr test llm bench` needs the `inference` feature (this apr was built without it)" .to_string(), )) } +/// The refusal of a `bench` mode handed another subcommand. `dispatch` routes +/// only `bench` to them, so this names a routing defect if it ever prints. +fn not_bench() -> CliError { + CliError::InvalidInput("`apr test llm`: only `bench` routes to a benchmark mode".to_string()) +} + /// TWO MODES, ONE ENTRYPOINT — the §4.4-conformant one. /// /// `--band` selects PERF-024's `run_band`, which until PERF-025 was called by @@ -544,9 +556,7 @@ async fn dispatch_band(command: &LlmSubcommand) -> Result<()> { .. } = command else { - return Err(CliError::InvalidInput( - "the band mode runs `bench` only".to_string(), - )); + return Err(not_bench()); }; // Unreachable: clap's `requires = "receipt"` enforces it. Stated rather // than unwrapped, because a receipt-less band run would measure for @@ -622,9 +632,7 @@ async fn dispatch_legacy(command: &LlmSubcommand) -> Result<()> { .. } = command else { - return Err(CliError::InvalidInput( - "the legacy mode runs `bench` only".to_string(), - )); + return Err(not_bench()); }; run_bench(BenchArgs { url, diff --git a/crates/apr-cli/src/commands/test_llm_shape.rs b/crates/apr-cli/src/commands/test_llm_shape.rs new file mode 100644 index 0000000000..c78c4e2d97 --- /dev/null +++ b/crates/apr-cli/src/commands/test_llm_shape.rs @@ -0,0 +1,174 @@ +//! `apr test llm shape-check `, the front door of +//! `apr_test::perf_gate::v3_shape` (#4971 V3-d). +//! +//! The library keeps the reference script's exit codes (0 ADMISSIBLE, +//! 1 NOT ADMISSIBLE, 2 RED). `apr` exits through `CliError`, so here they map +//! onto apr's own convention: +//! +//! | Verdict | apr exit | +//! |---|---| +//! | ADMISSIBLE | 0 | +//! | NOT ADMISSIBLE | 5 (`ValidationFailed`), after every reason is printed | +//! | RED: no such file, or not a file | 3 (`FileNotFound` / `NotAFile`) | +//! | RED: unreadable, empty, or no bands | 4 (`InvalidFormat`) | +//! +//! A witness that is not admissible and a witness that cannot be read never +//! share a code. That is the draft contract's F1, kept across the mapping. +//! +//! This is not a gate. No merge, queue or release job runs it. + +use std::path::Path; + +use apr_test::perf_gate::{check_v3_shape_file, ShapeVerdict}; + +use crate::error::{CliError, Result}; + +/// Check `witness`, print the verdict with every reason, and exit by it. +/// +/// # Errors +/// NOT ADMISSIBLE is `ValidationFailed`. A missing path is `FileNotFound`, +/// a directory `NotAFile`, and an unreadable or empty witness `InvalidFormat`. +pub(crate) fn run(witness: &Path) -> Result<()> { + if !witness.exists() { + return Err(CliError::FileNotFound(witness.to_path_buf())); + } + if !witness.is_file() { + return Err(CliError::NotAFile(witness.to_path_buf())); + } + let verdict = check_v3_shape_file(witness); + if !matches!(verdict, ShapeVerdict::Red(_)) { + println!("{}", report(witness, &verdict)); + } + outcome(witness, verdict) +} + +/// The printed verdict: one line naming it, then one line per reason. +fn report(witness: &Path, verdict: &ShapeVerdict) -> String { + std::iter::once(format!( + "V3 shape check {}: {}", + witness.display(), + verdict.label() + )) + .chain(verdict.reasons().iter().map(|r| format!(" - {r}"))) + .collect::>() + .join("\n") +} + +/// The verdict as apr's result. Only ADMISSIBLE is `Ok`. +fn outcome(witness: &Path, verdict: ShapeVerdict) -> Result<()> { + match verdict { + ShapeVerdict::Admissible => Ok(()), + ShapeVerdict::NotAdmissible(reasons) => Err(CliError::ValidationFailed(format!( + "V3 shape witness {}: NOT ADMISSIBLE, {} reason(s) printed above (#4971)", + witness.display(), + reasons.len() + ))), + ShapeVerdict::Red(why) => Err(CliError::InvalidFormat(format!( + "V3 shape witness {}: {why} (#4971)", + witness.display() + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + /// The library's control witness: every rule S1-S6 holds. + fn admissible() -> Value { + let band = |c: u32| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127}); + json!({ + "_planted": "FIXTURE: not a measurement", + "binary_sha256": "a".repeat(64), + "commit": "f".repeat(40), + "host": "lambda", + "prompt_sha256": "b".repeat(64), + "model": {"path": "Qwen3.5-4B-Q4_K_M.gguf", "sha256": "c".repeat(64)}, + "bands": [band(1), band(4), band(8), band(16)], + }) + } + + /// Run the command on `text` written to a fresh file; the exit code apr + /// would leave (0 on `Ok`) and the printed report. + fn exit_of(text: &str) -> (u8, String) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("witness.json"); + std::fs::write(&path, text).expect("write witness"); + let verdict = check_v3_shape_file(&path); + let printed = report(&path, &verdict); + let code = run(&path).map_or_else(|e| e.exit_code_value(), |()| 0); + (code, printed) + } + + #[test] + fn the_control_witness_is_admissible_and_exits_0() { + let (code, printed) = exit_of(&admissible().to_string()); + assert_eq!(code, 0, "{printed}"); + assert!(printed.ends_with(": ADMISSIBLE"), "{printed}"); + } + + /// #4971's falsifier, row 1: one band at `m_formed = c - 1`. + #[test] + fn a_band_that_formed_c_minus_1_exits_5_and_names_the_band() { + let mut w = admissible(); + w["bands"][1]["m_formed"] = json!(3); + let (code, printed) = exit_of(&w.to_string()); + assert_eq!(code, 5, "{printed}"); + assert!(printed.contains("NOT ADMISSIBLE"), "{printed}"); + assert!( + printed.contains(" - c=4: m_formed 3 != c (shape not served)"), + "{printed}" + ); + } + + /// #4971's falsifier, row 2: one divergence with no margin recorded. + #[test] + fn an_early_divergence_without_a_margin_exits_5_and_says_so() { + let mut w = admissible(); + w["bands"][2]["divergence_at"] = json!(3); + let (code, printed) = exit_of(&w.to_string()); + assert_eq!(code, 5, "{printed}"); + assert!( + printed + .contains(" - c=8: diverges from m=1 at 3 < 64 with no near-tie margin recorded"), + "{printed}" + ); + } + + /// The same divergence carried by a near-tie margin is admissible. + #[test] + fn an_early_divergence_at_a_near_tie_exits_0() { + let mut w = admissible(); + w["bands"][2]["divergence_at"] = json!(3); + w["bands"][2]["top2_margin_at_divergence"] = json!(0.01); + let (code, printed) = exit_of(&w.to_string()); + assert_eq!(code, 0, "{printed}"); + } + + /// F1 across the mapping: RED never shares NOT ADMISSIBLE's code. + #[test] + fn an_unreadable_or_empty_witness_exits_4_never_5() { + for text in ["", " \n", "{not json", "[]", r#"{"bands": []}"#] { + let (code, _) = exit_of(text); + assert_eq!( + code, 4, + "witness {text:?} must be RED (4), never NOT ADMISSIBLE (5)" + ); + } + } + + #[test] + fn a_missing_path_exits_3_and_a_directory_exits_3() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = run(&dir.path().join("absent.json")).expect_err("missing is refused"); + assert!(matches!(missing, CliError::FileNotFound(_)), "{missing:?}"); + assert_eq!(missing.exit_code_value(), 3); + let not_a_file = run(dir.path()).expect_err("a directory is refused"); + assert!( + matches!(not_a_file, CliError::NotAFile(_)), + "{not_a_file:?}" + ); + assert_eq!(not_a_file.exit_code_value(), 3); + } +} diff --git a/crates/apr-cli/src/extended_commands.rs b/crates/apr-cli/src/extended_commands.rs index febb8b2c8e..8504a0dc6d 100644 --- a/crates/apr-cli/src/extended_commands.rs +++ b/crates/apr-cli/src/extended_commands.rs @@ -2021,6 +2021,23 @@ pub enum LlmSubcommand { #[arg(long = "run", required = true)] runs: Vec, }, + + /// #4971 V3: check a perf041 witness against the serving-shape rule. + /// + /// Reads the `witness.json` that `scripts/perf041_batched_parity_probe.py` + /// writes and decides whether it can stand as the V3 Qwen3.5-4B + /// serving-parity witness: the blessed model with its sha256, the + /// identity fields, one band per c in {1, 4, 8, 16}, perf041 PASS and + /// `m_formed == c` in every band, and every divergence from m=1 before + /// `declared_min` explained by a recorded top-2 margin below 0.05. + /// + /// Exits 0 when admissible. A witness that is not admissible exits 5 + /// after printing every reason; one that cannot be read exits 3 (no such + /// file) or 4 (unreadable or empty). Not a gate: no release job runs it. + ShapeCheck { + /// The perf041 `witness.json` to check. + witness: InputFile, + }, } /// Parse `apr cbtop --iterations`, rejecting 0. diff --git a/crates/apr-cli/src/validate.rs b/crates/apr-cli/src/validate.rs index 586a17c28a..8da2e9a82b 100644 --- a/crates/apr-cli/src/validate.rs +++ b/crates/apr-cli/src/validate.rs @@ -13,10 +13,13 @@ fn extract_extended_model_paths(command: &ExtendedCommands) -> Vec { TestSubcommand::Tensor { file, .. } => { vec![file.to_path_buf()] }, - // `llm` measures an HTTP endpoint, not a local model file. Listed - // explicitly so a future subcommand that DOES name a file has to - // be handled here rather than absorbed by a wildcard. - TestSubcommand::Llm { .. } => vec![], + // `llm bench` measures an HTTP endpoint, not a local model file, + // and `llm shape-check` reads a perf041 witness JSON, not a model. + // Listed explicitly so a future subcommand that DOES name a model + // has to be handled here rather than absorbed by a wildcard. + TestSubcommand::Llm { command } => match command { + LlmSubcommand::Bench { .. } | LlmSubcommand::ShapeCheck { .. } => vec![], + }, }, ExtendedCommands::CompareHf { file, .. } | ExtendedCommands::Chat { file, .. } From 718ade7d0ba288a5100724d9eca2e1d0d5d314fe Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 08:08:18 +0200 Subject: [PATCH 15/35] fix(apr-cli): a RED V3 witness exits 4 as InvalidInput, not "Invalid APR format" (PMAT-4971) `apr test llm shape-check` mapped an unreadable or empty witness to `CliError::InvalidFormat`, which prints "Invalid APR format: ..." over a JSON witness. It is now `InvalidInput` ("Invalid input: V3 shape witness ..."), still exit 4, so RED and NOT ADMISSIBLE (5) never share a code. The F1 test asserts the variant, the prefix, and that no "APR" appears. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e70987c923633ed4c8b1dcc033b67cb331ffb4fb) (cherry picked from commit 0c183a990b0a2e716ec6aaf5af40e5e467e2fa62) (cherry picked from commit e3c85fab70472662c677947399f47426aa121c88) --- crates/apr-cli/src/commands/test_llm_shape.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/apr-cli/src/commands/test_llm_shape.rs b/crates/apr-cli/src/commands/test_llm_shape.rs index c78c4e2d97..8ff04ffafa 100644 --- a/crates/apr-cli/src/commands/test_llm_shape.rs +++ b/crates/apr-cli/src/commands/test_llm_shape.rs @@ -10,7 +10,7 @@ //! | ADMISSIBLE | 0 | //! | NOT ADMISSIBLE | 5 (`ValidationFailed`), after every reason is printed | //! | RED: no such file, or not a file | 3 (`FileNotFound` / `NotAFile`) | -//! | RED: unreadable, empty, or no bands | 4 (`InvalidFormat`) | +//! | RED: unreadable, empty, or no bands | 4 (`InvalidInput`: a JSON witness, not a model) | //! //! A witness that is not admissible and a witness that cannot be read never //! share a code. That is the draft contract's F1, kept across the mapping. @@ -27,7 +27,7 @@ use crate::error::{CliError, Result}; /// /// # Errors /// NOT ADMISSIBLE is `ValidationFailed`. A missing path is `FileNotFound`, -/// a directory `NotAFile`, and an unreadable or empty witness `InvalidFormat`. +/// a directory `NotAFile`, and an unreadable or empty witness `InvalidInput`. pub(crate) fn run(witness: &Path) -> Result<()> { if !witness.exists() { return Err(CliError::FileNotFound(witness.to_path_buf())); @@ -63,7 +63,7 @@ fn outcome(witness: &Path, verdict: ShapeVerdict) -> Result<()> { witness.display(), reasons.len() ))), - ShapeVerdict::Red(why) => Err(CliError::InvalidFormat(format!( + ShapeVerdict::Red(why) => Err(CliError::InvalidInput(format!( "V3 shape witness {}: {why} (#4971)", witness.display() ))), @@ -146,7 +146,9 @@ mod tests { assert_eq!(code, 0, "{printed}"); } - /// F1 across the mapping: RED never shares NOT ADMISSIBLE's code. + /// F1 across the mapping: RED never shares NOT ADMISSIBLE's code. And it + /// names the input, not a model: `InvalidFormat` would print "Invalid APR + /// format" over a JSON witness. #[test] fn an_unreadable_or_empty_witness_exits_4_never_5() { for text in ["", " \n", "{not json", "[]", r#"{"bands": []}"#] { @@ -156,6 +158,17 @@ mod tests { "witness {text:?} must be RED (4), never NOT ADMISSIBLE (5)" ); } + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("witness.json"); + std::fs::write(&path, "{not json").expect("write witness"); + let err = run(&path).expect_err("RED is refused"); + assert!(matches!(err, CliError::InvalidInput(_)), "{err:?}"); + let shown = err.to_string(); + assert!( + shown.starts_with("Invalid input: V3 shape witness "), + "{shown}" + ); + assert!(!shown.contains("APR"), "{shown}"); } #[test] From 2ff74d833d9a41eb5e017cca8a0e39760df43232 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 08:11:41 +0200 Subject: [PATCH 16/35] feat(perf-gate): the PP-26 witness and its top-2 margin from OpenAI logprob entries (PMAT-4971) V3's S5 admits an early divergence only on a recorded near-tie margin, and until now nothing in Rust could compute one: the margin was a field a witness file carried in. This is the producer's core. - perf_gate::margin: TokenLogprob / TopLogprob read the OpenAI entry (realizar, OpenAI with bytes null, a null logprob, llama.cpp's `id`). token_ids interns entries by text and bytes across all streams, so BatchInvarianceWitness::compare_logprobs hands compare_batch the same ids it would get from token ids. The margin is the gap between the two largest finite logprobs of the m=1 entry at divergence_at, which is the logit gap at temperature 0 (shared log-sum-exp). It is never read from a batch slot and never 0.0 for a step with no runner-up. - llm client: ChatRequest gains `logprobs` / `top_logprobs`, sent only when set (wire_request carries them by struct update). StreamedChatResponse collects every `logprobs.content[]` entry in order, including an entry held to the terminal chunk. The chunk fold moves out of chat_completion_stream into StreamTally (cognitive complexity 69 -> 22), and the loopback fixture's per-script fields into SseScript methods (26 -> 24). Loopback tests cover the request on the socket, the order, null chunks and the margins, plus a MUST-NOT-FIRE where an unset request sends neither key. - Contract: FALSIFY-WITNESS-LOGPROBS-MARGIN-4971, and the shape-check CLI row now names InvalidInput for exit 4. Not a gate. The perf041 port that fires the bands with these fields and writes witness.json is the next slice. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 271d700834c35ddbd820b3cfc096bb8afd6c4b34) (cherry picked from commit a4a064d89803c5b7c4ae83e97212d0eceda3e989) (cherry picked from commit 1c9622607966199446b105e8ea26be98bf6becc9) --- contracts/apr-serve-openai-compat-v1.yaml | 8 +- crates/aprender-test-cli/src/handlers/llm.rs | 2 + crates/aprender-test-lib/src/llm/band.rs | 2 + crates/aprender-test-lib/src/llm/client.rs | 197 ++++++++-- crates/aprender-test-lib/src/llm/loadtest.rs | 2 + crates/aprender-test-lib/src/llm/prompts.rs | 10 + .../aprender-test-lib/src/perf_gate/margin.rs | 357 ++++++++++++++++++ crates/aprender-test-lib/src/perf_gate/mod.rs | 2 + 8 files changed, 557 insertions(+), 23 deletions(-) create mode 100644 crates/aprender-test-lib/src/perf_gate/margin.rs diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index fd33626381..b5733f66ee 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -702,7 +702,13 @@ falsification_tests: if_fails: 'A witness that served a shape other than the one advertised (c=16 forming m=14), measured another model, or diverged from m=1 with no near-tie to explain it is read as a V3 receipt; or an unreadable witness reads as a verdict about the server.' - id: FALSIFY-V3-SHAPE-CLI-4971 name: apr_test_llm_shape_check_exits_by_the_verdict - prediction: 'apr test llm shape-check runs perf_gate::v3_shape on the file and maps its verdict onto apr''s exit codes: ADMISSIBLE exits 0; NOT ADMISSIBLE prints every reason and exits 5 (ValidationFailed); a missing path or a directory exits 3; an empty, blank, malformed or bandless witness exits 4 and never 5 (F1 across the mapping). #4971''s falsifier rows hold through the CLI: one band at m_formed = c - 1, or one divergence before declared_min with no margin, exits 5 and names the band; the same divergence with a recorded margin of 0.01 exits 0. This is not a gate: no merge, queue or release job runs it.' + prediction: 'apr test llm shape-check runs perf_gate::v3_shape on the file and maps its verdict onto apr''s exit codes: ADMISSIBLE exits 0; NOT ADMISSIBLE prints every reason and exits 5 (ValidationFailed); a missing path or a directory exits 3; an empty, blank, malformed or bandless witness exits 4 (InvalidInput, naming the witness and never an APR format) and never 5 (F1 across the mapping). #4971''s falsifier rows hold through the CLI: one band at m_formed = c - 1, or one divergence before declared_min with no margin, exits 5 and names the band; the same divergence with a recorded margin of 0.01 exits 0. This is not a gate: no merge, queue or release job runs it.' test_harness: 'cargo test -p apr-cli --lib test_llm_shape' expected_output: "test result: ok" if_fails: 'The Rust check exists but #4971''s "exits non-zero" has no front door: an operator runs the Python reference by hand, or a NOT ADMISSIBLE witness and an unreadable one leave the same exit code.' + - id: FALSIFY-WITNESS-LOGPROBS-MARGIN-4971 + name: the_witness_margin_is_computed_from_the_m1_logprob_entries + prediction: 'perf_gate::margin turns OpenAI logprob entries into the PP-26 witness. Entries are the same token when text and bytes agree, interned across every stream before compare_batch, so divergence_at is the one the id path gives. The top-2 margin is the gap between the two largest finite logprobs of the m=1 entry at divergence_at, which equals the logit gap at temperature 0 (shared log-sum-exp); it is read from m=1, never from a batch slot, and does not depend on the listed order. A step with fewer than two finite logprobs (null, -inf, NaN, truncated or empty top_logprobs) records no margin, never 0. Identical streams record no divergence and no margin. The served wire shapes (realizar, OpenAI with bytes null, a null logprob, llama.cpp with id) parse. The llm client puts logprobs and top_logprobs on the socket only when the request sets them and collects every streamed entry in order, including a null on a chunk and an entry held to the terminal chunk.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::margin && cargo test -p aprender-test-lib --features llm --lib -- llm::client::tests::a_logprobs_request llm::client::tests::a_request_without_logprobs' + expected_output: "test result: ok" + if_fails: 'A V3 witness can only carry a margin someone typed in: a near-tie flip at c > 1 reads as NOT ADMISSIBLE, or a margin from the wrong stream or step (or a 0.0 for a step with no runner-up) makes a wrong-KV divergence read as a near-tie.' diff --git a/crates/aprender-test-cli/src/handlers/llm.rs b/crates/aprender-test-cli/src/handlers/llm.rs index b1821baab9..23a208d816 100644 --- a/crates/aprender-test-cli/src/handlers/llm.rs +++ b/crates/aprender-test-cli/src/handlers/llm.rs @@ -1545,6 +1545,8 @@ fn parse_dataset_line( // rejects it outright. `None` is the only correct value; the field was // simply never added because nothing builds this crate with `llm`. stream_options: None, + logprobs: None, + top_logprobs: None, }; Ok((prompt, estimated_tokens, max_tokens)) } diff --git a/crates/aprender-test-lib/src/llm/band.rs b/crates/aprender-test-lib/src/llm/band.rs index 13d34ad082..95ac81f213 100644 --- a/crates/aprender-test-lib/src/llm/band.rs +++ b/crates/aprender-test-lib/src/llm/band.rs @@ -657,6 +657,8 @@ mod tests { seed: Some(0), ignore_eos: Some(true), stream_options: None, + logprobs: None, + top_logprobs: None, }] } diff --git a/crates/aprender-test-lib/src/llm/client.rs b/crates/aprender-test-lib/src/llm/client.rs index 389ca7b3a4..b45e84b3c7 100644 --- a/crates/aprender-test-lib/src/llm/client.rs +++ b/crates/aprender-test-lib/src/llm/client.rs @@ -9,6 +9,7 @@ use std::time::Duration; use std::time::Instant; use crate::perf_gate::drain::StreamMode; +use crate::perf_gate::margin::TokenLogprob; /// SSE streaming chunk from an OpenAI-compatible chat completion endpoint. #[derive(Debug, Clone, Deserialize)] @@ -47,6 +48,9 @@ struct StreamAccum { finish_reason: Option, stream_mode: Option, timings: Option, + /// #4971: every `logprobs.content[]` entry, in order, whichever chunk + /// carried it, the terminal one included. + logprobs: Vec, } impl StreamAccum { @@ -72,6 +76,9 @@ impl StreamAccum { if choice.finish_reason.is_some() { self.finish_reason.clone_from(&choice.finish_reason); } + if let Some(entries) = choice.logprobs.as_ref().and_then(|lp| lp.content.as_ref()) { + self.logprobs.extend(entries.iter().cloned()); + } } // PP-27: the server declares the mechanism on the FIRST chunk. Later // chunks do not carry it, and a later one that did must not overwrite @@ -96,6 +103,18 @@ pub struct StreamChoice { pub delta: StreamDelta, /// Finish reason (present on final chunk). pub finish_reason: Option, + /// OpenAI `logprobs`, present only when the request asked for them (#4971). + #[serde(default)] + pub logprobs: Option, +} + +/// `choices[].logprobs` of a streamed chunk: the entries of the tokens this +/// chunk delivers, which need not be one per chunk (#4971). +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ChoiceLogprobs { + /// One entry per token, in order. `null` reads as none. + #[serde(default)] + pub content: Option>, } /// PP-LLAMA-001 v3.0 §3 — the phase timings the SERVER measured. @@ -189,6 +208,10 @@ pub struct StreamedChatResponse { pub timings: Option, /// Why generation stopped (e.g., "stop", "length"). pub finish_reason: Option, + /// #4971: every `logprobs.content[]` entry of the stream, in order, + /// whichever chunk carried it. Empty when the server sent none, as it + /// does when the request did not set `logprobs`. + pub logprobs: Vec, } /// Chat message role. @@ -270,6 +293,14 @@ pub struct ChatRequest { /// set on the blocking path would turn a working lane into a 400. #[serde(skip_serializing_if = "Option::is_none")] pub stream_options: Option, + /// OpenAI `logprobs`: return each generated token's logprob (#4971). + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + /// OpenAI `top_logprobs`: also return each step's best N tokens, 0 to 20. + /// Servers refuse it without `logprobs: true`. The V3 witness asks for 2, + /// the fewest that carry a top-2 margin. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, } /// OpenAI `stream_options` — the ask that makes a server emit terminal usage. @@ -314,6 +345,8 @@ impl ChatRequest { seed: None, ignore_eos: None, stream_options: None, + logprobs: None, + top_logprobs: None, } } } @@ -829,6 +862,7 @@ impl LlmClient { stream_mode: acc.stream_mode, timings: acc.timings, finish_reason: acc.finish_reason, + logprobs: acc.logprobs, }) } @@ -898,6 +932,8 @@ mod tests { seed: None, ignore_eos: None, stream_options: None, + logprobs: None, + top_logprobs: None, }; let json = serde_json::to_string(&req).unwrap(); assert!(json.contains("\"temperature\":0.0")); @@ -917,6 +953,8 @@ mod tests { seed: None, ignore_eos: None, stream_options: None, + logprobs: None, + top_logprobs: None, }; let json = serde_json::to_string(&req).unwrap(); assert!(!json.contains("temperature")); @@ -1166,6 +1204,8 @@ mod tests { seed: None, ignore_eos: None, stream_options: None, + logprobs: None, + top_logprobs: None, }; let json = serde_json::to_string(&req).unwrap(); assert!(json.contains("\"stream\":true")); @@ -1262,7 +1302,8 @@ mod tests { /// What one SSE probe should emit. Each field is a separate defect shape: /// a stream with no `usage`, a stream with no content, a stream that - /// declares nothing. + /// declares nothing. `logprobs` adds one entry per content chunk, a + /// `null` on the role chunk and one held entry on the terminal chunk. #[cfg(feature = "llm")] #[derive(Clone, Copy)] struct SseScript { @@ -1270,6 +1311,7 @@ mod tests { declare_mode: Option<&'static str>, terminal_usage: bool, timings: bool, + logprobs: bool, } #[cfg(feature = "llm")] @@ -1282,8 +1324,80 @@ mod tests { declare_mode: Some("live"), terminal_usage: true, timings: true, + logprobs: false, + } + } + + /// The terminal chunk's `timings` field, or nothing. + fn timings_field(self) -> &'static str { + if self.timings { + ",\"timings\":{\"prompt_n\":512,\"prompt_ms\":40.0,\"prompt_per_second\":12800.0,\ + \"predicted_n\":4,\"predicted_ms\":20.0,\"predicted_per_second\":200.0,\ + \"clock\":\"server std::time::Instant (CLOCK_MONOTONIC)\"}" + } else { + "" + } + } + + /// The role chunk's `logprobs` field: a `null`, as OpenAI sends on a + /// chunk with no token, or nothing. + fn role_logprobs(self) -> &'static str { + if self.logprobs { + ",\"logprobs\":null" + } else { + "" } } + + /// Content chunk `i`'s `logprobs` field (margin `0.5 * (i + 1)`), or + /// nothing. + fn chunk_logprobs(self, i: usize) -> String { + if self.logprobs { + scripted_logprobs(&format!("t{i} "), 0.5 * (i + 1) as f64) + } else { + String::new() + } + } + + /// The terminal chunk: `finish_reason`, then (when scripted) the + /// `usage` block with `timings`, and the held `logprobs` entry. + fn terminal_chunk(self) -> String { + let timings = self.timings_field(); + let held = self.held_logprobs(); + if self.terminal_usage { + format!( + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"length\"{held}}}],\ + \"usage\":{{\"prompt_tokens\":512,\"completion_tokens\":128,\"total_tokens\":640}}\ + {timings}}}\n\n" + ) + } else { + "data: {\"choices\":[{\"index\":0,\"delta\":{},\ + \"finish_reason\":\"length\"}]}\n\n" + .to_string() + } + } + + /// The terminal chunk's `logprobs` field: one entry the server held + /// back to the end (margin 0.01), or nothing. + fn held_logprobs(self) -> String { + if self.logprobs { + scripted_logprobs("held", 0.01) + } else { + String::new() + } + } + } + + /// One scripted `logprobs` field: `token` at logprob -0.25, the runner-up + /// `gap` below it, so the entry's top-2 margin is `gap`. + #[cfg(feature = "llm")] + fn scripted_logprobs(token: &str, gap: f64) -> String { + let runner_up = -0.25 - gap; + format!( + ",\"logprobs\":{{\"content\":[{{\"token\":\"{token}\",\"logprob\":-0.25,\ + \"bytes\":null,\"top_logprobs\":[{{\"token\":\"{token}\",\"logprob\":-0.25,\ + \"bytes\":null}},{{\"token\":\"x\",\"logprob\":{runner_up},\"bytes\":null}}]}}]}}" + ) } /// Read one HTTP request off `sock` and return its body, or `None` if the @@ -1334,38 +1448,22 @@ mod tests { let mode = script .declare_mode .map_or(String::new(), |m| format!(",\"stream_mode\":\"{m}\"")); + let none = script.role_logprobs(); let first = format!( - "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"role\":\"assistant\"}}}}]{mode}}}\n\n" + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"role\":\"assistant\"}}{none}}}]{mode}}}\n\n" ); let _ = sock.write_all(first.as_bytes()).await; for i in 0..script.content_chunks { tokio::time::sleep(Duration::from_millis(5)).await; + let lp = script.chunk_logprobs(i); let chunk = format!( - "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"t{i} \"}}}}]}}\n\n" + "data: {{\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"t{i} \"}}{lp}}}]}}\n\n" ); if sock.write_all(chunk.as_bytes()).await.is_err() { return; } } - let timings = if script.timings { - ",\"timings\":{\"prompt_n\":512,\"prompt_ms\":40.0,\"prompt_per_second\":12800.0,\ - \"predicted_n\":4,\"predicted_ms\":20.0,\"predicted_per_second\":200.0,\ - \"clock\":\"server std::time::Instant (CLOCK_MONOTONIC)\"}" - } else { - "" - }; - if script.terminal_usage { - let terminal = format!( - "data: {{\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"length\"}}],\ - \"usage\":{{\"prompt_tokens\":512,\"completion_tokens\":128,\"total_tokens\":640}}\ - {timings}}}\n\n" - ); - let _ = sock.write_all(terminal.as_bytes()).await; - } else { - let terminal = "data: {\"choices\":[{\"index\":0,\"delta\":{},\ - \"finish_reason\":\"length\"}]}\n\n"; - let _ = sock.write_all(terminal.as_bytes()).await; - } + let _ = sock.write_all(script.terminal_chunk().as_bytes()).await; let _ = sock.write_all(b"data: [DONE]\n\n").await; let _ = sock.flush().await; let _ = sock.shutdown().await; @@ -1502,6 +1600,61 @@ mod tests { ); } + /// #4971: a request that sets `logprobs` asks for them on the socket, and + /// the stream's entries come back in order, whichever chunk carried them: + /// one per content chunk, none from a `null`, and the one held to the + /// terminal chunk. Each keeps its runner-up, so the top-2 margin survives. + #[cfg(feature = "llm")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_logprobs_request_asks_on_the_wire_and_collects_every_entry_in_order() { + use crate::perf_gate::margin::top2_margin; + let script = SseScript { + logprobs: true, + ..SseScript::conformant() + }; + let (url, bodies) = spawn_scripted_sse(script).await; + let request = ChatRequest { + logprobs: Some(true), + top_logprobs: Some(2), + ..pinned_request() + }; + let response = LlmClient::new(&url, "qwen-coder") + .chat_completion_stream(&request) + .await + .expect("a conformant stream is accepted"); + + let received = bodies.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let body = received.first().expect("the probe recorded one body"); + assert!(body.contains("\"logprobs\":true"), "{body}"); + assert!(body.contains("\"top_logprobs\":2"), "{body}"); + + let tokens: Vec<&str> = response.logprobs.iter().map(|e| e.token.as_str()).collect(); + assert_eq!(tokens, ["t0 ", "t1 ", "t2 ", "t3 ", "held"]); + let margins: Vec = response.logprobs.iter().filter_map(top2_margin).collect(); + let want = [0.5, 1.0, 1.5, 2.0, 0.01]; + assert_eq!(margins.len(), want.len(), "{margins:?}"); + for (got, want) in margins.iter().zip(want) { + assert!((got - want).abs() < 1e-9, "{margins:?}"); + } + } + + /// MUST-NOT-FIRE for #4971: a request that does not set `logprobs` sends + /// neither key, so a server that refuses them (a 501 naming the backend) + /// is never asked, and the response collects nothing. + #[cfg(feature = "llm")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_request_without_logprobs_sends_neither_key_and_collects_none() { + let (url, bodies) = spawn_scripted_sse(SseScript::conformant()).await; + let response = LlmClient::new(&url, "qwen-coder") + .chat_completion_stream(&pinned_request()) + .await + .expect("a conformant stream is accepted"); + let received = bodies.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let body = received.first().expect("the probe recorded one body"); + assert!(!body.contains("logprobs"), "{body}"); + assert!(response.logprobs.is_empty(), "{:?}", response.logprobs); + } + /// MUST-FIRE for PP-27: a stream whose terminal chunk carries no `usage` is /// refused. The alternative -- counting the four content frames as four /// tokens -- is a number with the shape of a measurement. diff --git a/crates/aprender-test-lib/src/llm/loadtest.rs b/crates/aprender-test-lib/src/llm/loadtest.rs index 9268321b27..a80361e12f 100644 --- a/crates/aprender-test-lib/src/llm/loadtest.rs +++ b/crates/aprender-test-lib/src/llm/loadtest.rs @@ -1494,6 +1494,8 @@ fn default_prompt() -> ChatRequest { // PP-27: set on the wire by `LlmClient::wire_request` when (and only // when) the request actually streams; a profile never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } diff --git a/crates/aprender-test-lib/src/llm/prompts.rs b/crates/aprender-test-lib/src/llm/prompts.rs index a28c3b0361..9ec191b3f6 100644 --- a/crates/aprender-test-lib/src/llm/prompts.rs +++ b/crates/aprender-test-lib/src/llm/prompts.rs @@ -572,6 +572,8 @@ impl PromptRecord { // PP-27: `LlmClient::wire_request` sets this when the request // actually streams; a corpus record never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } } @@ -601,6 +603,8 @@ fn micro_prompt() -> ChatRequest { // PP-27: set on the wire by `LlmClient::wire_request` when (and only // when) the request actually streams; a profile never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } @@ -620,6 +624,8 @@ fn short_prompt() -> ChatRequest { // PP-27: set on the wire by `LlmClient::wire_request` when (and only // when) the request actually streams; a profile never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } @@ -646,6 +652,8 @@ fn medium_prompt() -> ChatRequest { // PP-27: set on the wire by `LlmClient::wire_request` when (and only // when) the request actually streams; a profile never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } @@ -693,6 +701,8 @@ fn long_prompt() -> ChatRequest { // PP-27: set on the wire by `LlmClient::wire_request` when (and only // when) the request actually streams; a profile never asks for it. stream_options: None, + logprobs: None, + top_logprobs: None, } } diff --git a/crates/aprender-test-lib/src/perf_gate/margin.rs b/crates/aprender-test-lib/src/perf_gate/margin.rs new file mode 100644 index 0000000000..cd55c9064e --- /dev/null +++ b/crates/aprender-test-lib/src/perf_gate/margin.rs @@ -0,0 +1,357 @@ +//! #4971 V3-d: the witness from OpenAI `logprobs`, with the top-2 margin. +//! +//! [`BatchInvarianceWitness::compare_batch`] compares token ids, and the +//! OpenAI wire carries none: each generated token arrives as one +//! `logprobs.content[]` entry with its text, its bytes and the step's best +//! tokens. This module is the bridge. [`token_ids`] numbers the entries by text +//! and bytes, so the comparison runs unchanged on them, and [`top2_margin`] +//! reads the gap between a step's two best tokens. +//! +//! Two logprobs of one step share one log-sum-exp, so their difference IS the +//! logit difference, `ln p0 - ln p1 = l0 - l1`, whenever the logprobs are taken +//! over the unscaled logits. The witness decodes at temperature 0, where they +//! are. The margin needs no logit on the wire. +//! +//! Only the `m = 1` stream's margin is recorded (master §12 row 22). It says how +//! close the reference's own choice was at the token where the batch parted +//! from it; below [`super::v3_shape::NEAR_TIE_EPS`] the V3 shape check reads +//! the divergence as a near-tie flip, not a defect. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::witness::BatchInvarianceWitness; + +/// One generated token, as OpenAI's `logprobs.content[]` carries it. +/// +/// `logprob` is optional because a server serialising `-inf` through +/// `serde_json` writes `null`, and `bytes` because OpenAI allows `null` there. +/// Unknown keys (llama.cpp sends `id`) are ignored. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TokenLogprob { + /// The chosen token's text. + pub token: String, + /// `ln P(token)` at this step. + #[serde(default)] + pub logprob: Option, + /// The token's own bytes, which can be part of one character. + #[serde(default)] + pub bytes: Option>, + /// The step's best tokens, best first. + #[serde(default)] + pub top_logprobs: Vec, +} + +/// One of a step's best tokens. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TopLogprob { + /// The token's text. + pub token: String, + /// `ln P(token)` at this step. + #[serde(default)] + pub logprob: Option, + /// The token's own bytes. + #[serde(default)] + pub bytes: Option>, +} + +/// The source string of a witness built from logprob entries. +pub const LOGPROBS_SOURCE: &str = + "client-side comparison of OpenAI logprob entries across the batch's slots (m=1 top-2 margin recorded)"; + +impl TokenLogprob { + /// What makes two entries the same token: the text and the bytes. Text + /// alone is lossy for a token that is part of one character. + fn identity(&self) -> (&str, &[u8]) { + ( + self.token.as_str(), + self.bytes.as_deref().unwrap_or_default(), + ) + } +} + +/// The gap between a step's two best tokens, `ln p_best - ln p_second`. +/// +/// `None` when the step lists fewer than two finite logprobs: an absent margin +/// is never `0.0`, which would read as the closest near-tie there is. The two +/// largest are taken whatever order the server listed them in, so the margin is +/// never negative. +#[must_use] +pub fn top2_margin(entry: &TokenLogprob) -> Option { + let mut finite = entry + .top_logprobs + .iter() + .filter_map(|top| top.logprob) + .filter(|lp| lp.is_finite()); + let (mut best, mut second) = (finite.next()?, finite.next()?); + if second > best { + std::mem::swap(&mut best, &mut second); + } + for lp in finite { + if lp > best { + second = best; + best = lp; + } else if lp > second { + second = lp; + } + } + Some(best - second) +} + +/// Number every stream's tokens so they compare as ids. Two entries share an id +/// exactly when they share text and bytes, across all the streams given. +#[must_use] +pub fn token_ids(streams: &[&[TokenLogprob]]) -> Vec> { + let mut ids: HashMap<(&str, &[u8]), u32> = HashMap::new(); + streams + .iter() + .map(|stream| { + stream + .iter() + .map(|entry| { + let next = u32::try_from(ids.len()).unwrap_or(u32::MAX); + *ids.entry(entry.identity()).or_insert(next) + }) + .collect() + }) + .collect() +} + +impl BatchInvarianceWitness { + /// PP-26 v3.1 over logprob streams, as [`Self::compare_batch`] decides it, + /// plus the `m = 1` stream's top-2 margin at `divergence_at`. + /// + /// The margin stays `None` when the streams never part, or when the `m = 1` + /// entry at the divergence lists fewer than two finite logprobs. + #[must_use] + pub fn compare_logprobs( + m1: &[TokenLogprob], + slots: &[&[TokenLogprob]], + declared_min: u32, + max_constant_run: u32, + ) -> Self { + let streams: Vec<&[TokenLogprob]> = + std::iter::once(m1).chain(slots.iter().copied()).collect(); + let ids = token_ids(&streams); + let (m1_ids, slot_ids) = ids + .split_first() + .map_or((&[][..], &[][..]), |(m1, rest)| (m1.as_slice(), rest)); + let slot_refs: Vec<&[u32]> = slot_ids.iter().map(Vec::as_slice).collect(); + let mut witness = Self::compare_batch(m1_ids, &slot_refs, declared_min, max_constant_run); + witness.top2_margin_at_divergence = witness + .divergence_at + .and_then(|at| m1.get(usize::try_from(at).ok()?)) + .and_then(top2_margin); + witness.source = LOGPROBS_SOURCE.to_string(); + witness + } +} + +#[cfg(test)] +mod tests { + use super::super::v3_shape::NEAR_TIE_EPS; + use super::super::witness::{BatchInvariance, DEFAULT_MAX_CONSTANT_RUN}; + use super::*; + + /// `ln softmax(logits)`, as a server computes logprobs from the logits. + fn log_softmax(logits: &[f64]) -> Vec { + let max = logits.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let lse = max + logits.iter().map(|l| (l - max).exp()).sum::().ln(); + logits.iter().map(|l| l - lse).collect() + } + + fn top(token: &str, logprob: f64) -> TopLogprob { + TopLogprob { + token: token.to_string(), + logprob: Some(logprob), + bytes: Some(token.as_bytes().to_vec()), + } + } + + /// One step that chose `token`, whose step logits put `token` first by + /// `gap` over `runner_up` and far ahead of a third token. + fn step(token: &str, runner_up: &str, gap: f64) -> TokenLogprob { + let lps = log_softmax(&[10.0, 10.0 - gap, 0.0]); + TokenLogprob { + token: token.to_string(), + logprob: Some(lps[0]), + bytes: Some(token.as_bytes().to_vec()), + top_logprobs: vec![top(token, lps[0]), top(runner_up, lps[1]), top("~", lps[2])], + } + } + + /// A 128-token `m = 1` stream, decisive at every step except `tie_at`, + /// where its top two are `tie_gap` apart. + fn reference(tie_at: usize, tie_gap: f64) -> Vec { + (0..128) + .map(|i| { + let gap = if i == tie_at { tie_gap } else { 4.0 }; + step(&format!("t{i}"), &format!("u{i}"), gap) + }) + .collect() + } + + /// The batch's stream: the reference until `at`, the runner-up there, and + /// its own tokens after. + fn parted_at(m1: &[TokenLogprob], at: usize) -> Vec { + let mut out = m1.to_vec(); + out[at] = step(&format!("u{at}"), &format!("t{at}"), 0.5); + for (i, entry) in out.iter_mut().enumerate().skip(at + 1) { + *entry = step(&format!("b{i}"), &format!("c{i}"), 4.0); + } + out + } + + /// The claim the module rests on: a logprob gap equals the logit gap. + #[test] + fn the_logprob_gap_is_the_logit_gap() { + for gap in [0.0, 0.01, 0.049, 0.5, 3.25] { + let margin = top2_margin(&step("a", "b", gap)).expect("two finite logprobs"); + assert!((margin - gap).abs() < 1e-9, "gap {gap} read as {margin}"); + } + } + + /// The planted near-tie: the batch takes the runner-up where the `m = 1` + /// choice led by 0.01. The witness records the divergence and the margin, + /// and the margin sits inside the near-tie band. + #[test] + fn a_near_tie_flip_records_its_margin() { + let m1 = reference(3, 0.01); + let batch = parted_at(&m1, 3); + let w = BatchInvarianceWitness::compare_logprobs( + &m1, + &[&batch, &batch], + 64, + DEFAULT_MAX_CONSTANT_RUN, + ); + assert_eq!(w.divergence_at, Some(3)); + assert_eq!(w.batch_invariance, BatchInvariance::Pass, "slots agree"); + let margin = w.top2_margin_at_divergence.expect("margin recorded"); + assert!((margin - 0.01).abs() < 1e-9, "{margin}"); + assert!(margin < NEAR_TIE_EPS); + assert_eq!(w.source, LOGPROBS_SOURCE); + } + + /// The must-fire twin: the same flip where the `m = 1` choice led by 2.5 + /// records 2.5, which no near-tie band admits. + #[test] + fn a_decisive_flip_records_a_margin_outside_the_band() { + let m1 = reference(3, 2.5); + let batch = parted_at(&m1, 3); + let w = BatchInvarianceWitness::compare_logprobs(&m1, &[&batch], 64, 16); + let margin = w.top2_margin_at_divergence.expect("margin recorded"); + assert!((margin - 2.5).abs() < 1e-9, "{margin}"); + assert!(margin >= NEAR_TIE_EPS); + } + + /// The margin is the `m = 1` stream's, not the batch's: the batch's own + /// step at the divergence leads by 0.5, and 0.5 is not what is recorded. + #[test] + fn the_margin_is_read_from_the_m1_stream() { + let m1 = reference(5, 0.02); + let batch = parted_at(&m1, 5); + let w = BatchInvarianceWitness::compare_logprobs(&m1, &[&batch], 64, 16); + let margin = w.top2_margin_at_divergence.expect("margin recorded"); + assert!((margin - 0.02).abs() < 1e-9, "{margin}"); + } + + /// No divergence, no margin: identical streams record neither. + #[test] + fn identical_streams_record_no_divergence_and_no_margin() { + let m1 = reference(3, 0.01); + let w = BatchInvarianceWitness::compare_logprobs(&m1, &[&m1, &m1], 64, 16); + assert_eq!(w.divergence_at, None); + assert_eq!(w.top2_margin_at_divergence, None); + assert_eq!(w.batch_invariance, BatchInvariance::Pass); + } + + /// MUST-FIRE: an `m = 1` step with fewer than two finite logprobs has no + /// margin. Reading it as `0.0` would admit the divergence as the closest + /// near-tie there is. + #[test] + fn a_step_without_two_finite_logprobs_has_no_margin_never_zero() { + let mut lone = step("a", "b", 0.01); + lone.top_logprobs.truncate(1); + assert_eq!(top2_margin(&lone), None); + lone.top_logprobs.clear(); + assert_eq!(top2_margin(&lone), None); + let mut nulled = step("a", "b", 0.01); + nulled.top_logprobs[1].logprob = None; + nulled.top_logprobs[2].logprob = None; + assert_eq!(top2_margin(&nulled), None); + let mut nan = step("a", "b", 0.01); + nan.top_logprobs[1].logprob = Some(f64::NAN); + nan.top_logprobs[2].logprob = Some(f64::NEG_INFINITY); + assert_eq!(top2_margin(&nan), None); + + let mut m1 = reference(3, 0.01); + m1[3].top_logprobs.truncate(1); + let batch = parted_at(&m1, 3); + let w = BatchInvarianceWitness::compare_logprobs(&m1, &[&batch], 64, 16); + assert_eq!(w.divergence_at, Some(3)); + assert_eq!(w.top2_margin_at_divergence, None); + } + + /// The best two are found in any listed order, and the gap is never negative. + #[test] + fn the_margin_does_not_depend_on_the_listed_order() { + let mut entry = step("a", "b", 0.3); + entry.top_logprobs.reverse(); + let margin = top2_margin(&entry).expect("two finite logprobs"); + assert!((margin - 0.3).abs() < 1e-9, "{margin}"); + entry.top_logprobs.swap(0, 1); + assert!((top2_margin(&entry).expect("still two") - 0.3).abs() < 1e-9); + } + + /// Ids follow text AND bytes: one text with other bytes is another token, + /// and the same token gets one id across streams. + #[test] + fn token_ids_follow_text_and_bytes_across_streams() { + let a = step("a", "b", 1.0); + let mut a_other_bytes = a.clone(); + a_other_bytes.bytes = Some(vec![0xE2]); + let mut a_no_bytes = a.clone(); + a_no_bytes.bytes = None; + let first = [a.clone(), a_other_bytes]; + let second = [a, a_no_bytes]; + let ids = token_ids(&[&first, &second]); + assert_eq!(ids[0][0], ids[1][0], "same token, same id across streams"); + assert_ne!(ids[0][0], ids[0][1], "other bytes, other token"); + assert_ne!( + ids[0][0], ids[1][1], + "absent bytes differ from present ones" + ); + assert_ne!(ids[0][1], ids[1][1]); + } + + /// The wire shapes the witness reads: realizar's (bytes as numbers), OpenAI's + /// (`bytes: null`), a `-inf` serialised as `null`, and llama.cpp's extra `id`. + #[test] + fn the_served_wire_shapes_parse() { + let realizar = r#"{"token":"Hi","logprob":-0.1,"bytes":[72,105], + "top_logprobs":[{"token":"Hi","logprob":-0.1,"bytes":[72,105]}, + {"token":"Hey","logprob":-2.4,"bytes":[72,101,121]}]}"#; + let e: TokenLogprob = serde_json::from_str(realizar).expect("realizar shape"); + assert_eq!(e.bytes.as_deref(), Some(&b"Hi"[..])); + assert!((top2_margin(&e).expect("two") - 2.3).abs() < 1e-9); + + let openai = r#"{"token":"Hi","logprob":-0.1,"bytes":null,"top_logprobs":[]}"#; + let e: TokenLogprob = serde_json::from_str(openai).expect("openai shape"); + assert_eq!(e.bytes, None); + assert_eq!(top2_margin(&e), None); + + let neg_inf = r#"{"token":"Hi","logprob":null,"bytes":[72,105], + "top_logprobs":[{"token":"Hi","logprob":-0.1,"bytes":[72,105]}, + {"token":"x","logprob":null,"bytes":[120]}]}"#; + let e: TokenLogprob = serde_json::from_str(neg_inf).expect("null logprob"); + assert_eq!(e.logprob, None); + assert_eq!(top2_margin(&e), None); + + let llama = r#"{"id":13048,"token":"Hi","logprob":-0.1,"bytes":[72,105], + "top_logprobs":[{"id":13048,"token":"Hi","logprob":-0.1,"bytes":[72,105]}, + {"id":17,"token":"Hey","logprob":-0.15,"bytes":[72,101,121]}]}"#; + let e: TokenLogprob = serde_json::from_str(llama).expect("llama.cpp shape"); + assert!((top2_margin(&e).expect("two") - 0.05).abs() < 1e-9); + } +} diff --git a/crates/aprender-test-lib/src/perf_gate/mod.rs b/crates/aprender-test-lib/src/perf_gate/mod.rs index e84aa61da0..3ee1a5d15f 100644 --- a/crates/aprender-test-lib/src/perf_gate/mod.rs +++ b/crates/aprender-test-lib/src/perf_gate/mod.rs @@ -166,6 +166,7 @@ pub mod ab; pub mod bootstrap; pub mod drain; pub mod join; +pub mod margin; pub mod metrics; pub mod protocol; pub mod receipt; @@ -190,6 +191,7 @@ pub use drain::{ StreamWitness, StreamWitnessSource, DRAIN_SUSPECT_FRACTION, REQUEST_TIMEOUT_MS, SCHEMA_VERSION, }; pub use join::{BandRatios, JoinKey, Ratio, RatioBound, RatioMethod}; +pub use margin::{token_ids, top2_margin, TokenLogprob, TopLogprob, LOGPROBS_SOURCE}; pub use metrics::{agg_tok_s, aggregate_terms, BandMetrics, RequestSample}; pub use protocol::{ min_sampled_requests, warmup_requests, BandConfig, ClientModel, ProtocolParams, ProtocolSource, From 3de50ddfc5211e9652df299098cfd25a766757f1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 08:25:55 +0200 Subject: [PATCH 17/35] feat(perf-gate): the perf041 probe's decisions in Rust (PMAT-4971) V3-d, first slice: scripts/perf041_batched_parity_probe.py's verdicts, ported to perf_gate::parity with no HTTP. PP-28 refuses a slot whose completion_tokens or logprob entries differ from n_predict; a c>1 band with no m>1 batch in its own log window is UNMEASURABLE; the reference is stable only when two m=1 decodes agree to declared_min; the run passes only with a stable reference, every band decided, and a c>1 band run. A band whose streams never part records divergence_at as the shorter stream's length, not null, so a clean band is admissible to the V3 shape check; a test feeds the producer's witness to check_v3_shape to keep the two in step. The top-2 margin comes from compare_logprobs. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 673f14286da8d613aef1af124cd2ecd5291815c7) (cherry picked from commit d39949183a01cfeec65425ea1fa13cba0ced90a2) (cherry picked from commit c66f6abf8797b533f8b4267971d4e5bccbb31f31) --- crates/aprender-test-lib/src/perf_gate/mod.rs | 5 + .../aprender-test-lib/src/perf_gate/parity.rs | 809 ++++++++++++++++++ 2 files changed, 814 insertions(+) create mode 100644 crates/aprender-test-lib/src/perf_gate/parity.rs diff --git a/crates/aprender-test-lib/src/perf_gate/mod.rs b/crates/aprender-test-lib/src/perf_gate/mod.rs index 3ee1a5d15f..e28d0bacaa 100644 --- a/crates/aprender-test-lib/src/perf_gate/mod.rs +++ b/crates/aprender-test-lib/src/perf_gate/mod.rs @@ -168,6 +168,7 @@ pub mod drain; pub mod join; pub mod margin; pub mod metrics; +pub mod parity; pub mod protocol; pub mod receipt; pub mod replicate; @@ -193,6 +194,10 @@ pub use drain::{ pub use join::{BandRatios, JoinKey, Ratio, RatioBound, RatioMethod}; pub use margin::{token_ids, top2_margin, TokenLogprob, TopLogprob, LOGPROBS_SOURCE}; pub use metrics::{agg_tok_s, aggregate_terms, BandMetrics, RequestSample}; +pub use parity::{ + batch_formed_since, check_reference, evaluate_band, run_verdict, ProbeBand, ProbeModel, + ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, ProbeSlot, ProbeWitness, SampleResult, +}; pub use protocol::{ min_sampled_requests, warmup_requests, BandConfig, ClientModel, ProtocolParams, ProtocolSource, Sampler, BOOTSTRAP_RESAMPLES, BOOTSTRAP_SEED, COOLDOWN, INTERLEAVED, MIN_WALL_CLOCK, N_PREDICT, diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs new file mode 100644 index 0000000000..8857a9fc4a --- /dev/null +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -0,0 +1,809 @@ +//! #4971 V3-d: the perf041 probe's decisions, in Rust. +//! +//! `scripts/perf041_batched_parity_probe.py` decodes one prompt twice alone +//! (`m = 1`, the reference), then as `c` identical concurrent requests for each +//! `c` of the ladder, and writes a witness whose bands the V3 shape check +//! ([`super::v3_shape`]) reads. This module is that probe's judgement, ported +//! (C301): what makes a sample comparable, when the reference is stable, how a +//! band is scored, and the run's verdict. Every function here is pure in what +//! came back over the wire, so it compiles and is tested under default +//! features. The requests themselves are fired behind the `llm` feature. +//! +//! The port adds the margin. Every request asks for `logprobs` with the two +//! best tokens per step, so a band records the reference's top-2 margin at the +//! token where the batch parted from it (`top2_margin_at_divergence`), and the +//! V3 shape check can tell a near-tie flip from a defect. +//! +//! Three verdicts, as in the script: PASS, FAIL (a slot parted from its own +//! batch before `declared_min`, or froze), UNMEASURABLE (the run could not +//! decide). A run in which no band at `c > 1` was measured is UNMEASURABLE, +//! never PASS. +//! +//! This is not a gate. No merge, queue or release job runs it. + +use std::collections::BTreeMap; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::margin::{token_ids, TokenLogprob}; +use super::witness::{BatchInvariance, BatchInvarianceWitness}; + +/// The witness layout this module writes. The script's is 2; 3 adds the margin. +pub const WITNESS_VERSION: u32 = 3; + +/// The `probe` field: the script's name, with the port marked. +pub const PROBE: &str = "perf041-rs"; + +/// What the scheduler logs after each decode step, +/// `[PMAT-044] Batch m=N done` (`crates/aprender-serve/src/api/cuda_batch_scheduler.rs`). +const BATCH_MARK: &str = "Batch m="; + +/// Knobs that change which kernels decode, recorded rather than inherited. +/// A key that is unset is recorded as `null`, so the witness says so. +pub const ENV_KEYS: [&str; 9] = [ + "CUDA_BATCH_WINDOW_MS", + "CUBLAS_GEMM_THRESHOLD", + "FP8_DECODE", + "FP8_PREFILL", + "BATCHED_PREFILL", + "MULTI_PROMPT_PREFILL", + "APR_DECODE_GEMM", + "ITERATION_SCHEDULER", + "FUSED_GATE_UP", +]; + +/// The numbers a probe run compares against, from `scripts/perf-matrix.yaml`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProbePolicy { + /// `witness.min_agree_tokens`: tokens a batch's slots must agree for. + pub declared_min: u32, + /// `protocol.n_predict`: tokens every request generates (`ignore_eos`). + pub n_predict: u32, + /// `witness.max_constant_run`: a run of one token this long is a frozen slot. + pub max_constant_run: u32, +} + +/// One request, as the probe saw it come back. +#[derive(Debug, Clone, PartialEq)] +pub struct ProbeSample { + /// One logprob entry per generated token, in order. + pub logprobs: Vec, + /// `usage.completion_tokens`, as the server reported it. + pub completion_tokens: Option, + /// The terminal chunk's `finish_reason`. + pub finish_reason: Option, +} + +/// A request that came back, or why it did not. +pub type SampleResult = Result; + +/// The sample when it can be compared, else why not (PP-28): the server must +/// have generated exactly `n_predict` tokens and sent one logprob entry each. +fn accepted(sample: &SampleResult, n_predict: u32) -> Result<&ProbeSample, String> { + let sample = sample.as_ref().map_err(Clone::clone)?; + if sample.completion_tokens != Some(n_predict) { + return Err(format!( + "completion_tokens={} != n_predict={n_predict} (PP-28)", + sample + .completion_tokens + .map_or_else(|| "none".to_string(), |n| n.to_string()) + )); + } + let entries = sample.logprobs.len(); + if u32::try_from(entries).ok() != Some(n_predict) { + return Err(format!( + "{entries} logprob entries != n_predict={n_predict}; the request must ask for logprobs" + )); + } + Ok(sample) +} + +/// Tokens on which two streams agree from the start. +fn agreement(a: &[TokenLogprob], b: &[TokenLogprob]) -> u32 { + let ids = token_ids(&[a, b]); + let agree = ids[0] + .iter() + .zip(&ids[1]) + .take_while(|(x, y)| x == y) + .count(); + u32::try_from(agree).unwrap_or(u32::MAX) +} + +/// The `m = 1` reference: two solo decodes of the prompt, which must agree to +/// `declared_min` before a batched difference can be laid at batching's door. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct ProbeReference { + /// Both decodes were comparable and agreed to `declared_min`. + pub stable: bool, + /// The first decode's `usage.completion_tokens`. + pub tokens: Option, + /// Tokens on which the two decodes agree from the start. + pub self_divergence_at: Option, + /// Why the reference is not stable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// Judge the two `m = 1` decodes. +#[must_use] +pub fn check_reference( + first: &SampleResult, + second: &SampleResult, + policy: &ProbePolicy, +) -> ProbeReference { + let mut reference = ProbeReference { + tokens: first.as_ref().ok().and_then(|s| s.completion_tokens), + ..ProbeReference::default() + }; + let pair = accepted(first, policy.n_predict) + .map_err(|why| format!("ref#1: {why}")) + .and_then(|a| { + accepted(second, policy.n_predict) + .map(|b| (a, b)) + .map_err(|why| format!("ref#2: {why}")) + }); + let (a, b) = match pair { + Ok(pair) => pair, + Err(reason) => { + reference.reason = Some(reason); + return reference; + } + }; + let agree = agreement(&a.logprobs, &b.logprobs); + reference.self_divergence_at = Some(agree); + if agree < policy.declared_min { + reference.reason = Some(format!( + "the two m=1 references diverge at token {agree} < declared_min {}; a batched \ + difference could not be laid at batching's door", + policy.declared_min + )); + } else { + reference.stable = true; + } + reference +} + +/// One slot of a band, as recorded. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProbeSlot { + /// The slot's index in the band. + pub i: u32, + /// `usage.completion_tokens`. + pub completion_tokens: Option, + /// The terminal chunk's `finish_reason`. + pub finish_reason: Option, + /// Why the slot could not be compared. + pub refused: Option, +} + +impl ProbeSlot { + fn of(i: usize, sample: &SampleResult, n_predict: u32) -> Self { + let ok = sample.as_ref().ok(); + Self { + i: u32::try_from(i).unwrap_or(u32::MAX), + completion_tokens: ok.and_then(|s| s.completion_tokens), + finish_reason: ok.and_then(|s| s.finish_reason.clone()), + refused: accepted(sample, n_predict).err(), + } + } +} + +/// One band of the witness, in the shape the V3 shape check reads. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProbeBand { + /// Concurrent identical requests fired. + pub c: u32, + /// The largest batch the scheduler logged while this band ran. + pub m_formed: u32, + /// PP-26 v3.1's verdict on the band. + pub result: BatchInvariance, + /// The script's `divergence_at`: the first token at which the `m = 1` + /// reference and the batch part, or the shorter stream's length when they + /// never do. Recorded, not gated. `None` when the band was not measured. + pub divergence_at: Option, + /// The reference's top-2 margin at `divergence_at`, when they part. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub top2_margin_at_divergence: Option, + /// PP-26 (a): the shortest agreement of any slot with slot 0. + pub intra_agree_to: Option, + /// PP-26 (b): the longest run of one token in any slot. + pub max_constant_run: Option, + /// Tokens the slots had to agree for. + pub declared_min: u32, + /// The run of one token that marks a frozen slot. + pub max_constant_run_declared: u32, + /// Why the band is not PASS. + pub reason: Option, + /// Every slot fired. + pub slots: Vec, +} + +impl ProbeBand { + fn unmeasurable(c: u32, m_formed: u32, policy: &ProbePolicy, slots: Vec) -> Self { + Self { + c, + m_formed, + result: BatchInvariance::Unmeasurable, + divergence_at: None, + top2_margin_at_divergence: None, + intra_agree_to: None, + max_constant_run: None, + declared_min: policy.declared_min, + max_constant_run_declared: policy.max_constant_run, + reason: None, + slots, + } + } +} + +/// Why the band cannot be decided, before any token is compared. +fn band_refusal(c: u32, m_formed: u32, slots: &[ProbeSlot]) -> Option { + if u32::try_from(slots.len()).ok() != Some(c) { + return Some(format!("fired c={c} requests, {} came back", slots.len())); + } + let refusals: Vec = slots + .iter() + .filter_map(|s| s.refused.as_ref().map(|why| format!("slot {}: {why}", s.i))) + .collect(); + if !refusals.is_empty() { + return Some(refusals.join("; ")); + } + // A batch is two or more sequences in one step, so `m_formed < 2` is the + // word "batch", not a threshold. c = 1 needs none: its witness is the + // reference's agreement with itself. + (c > 1 && m_formed < 2).then(|| { + format!( + "no batch with m>1 formed in this band's window (max m={m_formed}); the batched \ + path was never exercised, so nothing about it was witnessed" + ) + }) +} + +/// Why a measured band FAILed, in PP-26 v3.1's two rules. +fn fail_reason(witness: &BatchInvarianceWitness, policy: &ProbePolicy) -> Option { + let mut reasons = Vec::new(); + let run = witness.max_constant_run.unwrap_or(0); + if policy.max_constant_run > 0 && run >= policy.max_constant_run { + reasons.push(format!( + "a slot repeated one token {run} times, at or above max_constant_run={} (#2753 signature)", + policy.max_constant_run + )); + } + let intra = witness.intra_agree_to.unwrap_or(0); + if intra < policy.declared_min { + reasons.push(format!( + "slots of one batch disagree at token {intra} < declared_min={}", + policy.declared_min + )); + } + (!reasons.is_empty()).then(|| reasons.join("; ")) +} + +/// Score one band: `samples` are the `c` slots, `m1` the reference stream, +/// `m_formed` the largest batch the scheduler logged while the band ran. +#[must_use] +pub fn evaluate_band( + c: u32, + m_formed: u32, + m1: &[TokenLogprob], + samples: &[SampleResult], + policy: &ProbePolicy, +) -> ProbeBand { + let slots = samples + .iter() + .enumerate() + .map(|(i, s)| ProbeSlot::of(i, s, policy.n_predict)) + .collect::>(); + let refusal = band_refusal(c, m_formed, &slots); + let mut band = ProbeBand::unmeasurable(c, m_formed, policy, slots); + if refusal.is_some() { + band.reason = refusal; + return band; + } + let streams: Vec<&[TokenLogprob]> = samples + .iter() + .filter_map(|s| s.as_ref().ok()) + .map(|s| s.logprobs.as_slice()) + .collect(); + let witness = BatchInvarianceWitness::compare_logprobs( + m1, + &streams, + policy.declared_min, + policy.max_constant_run, + ); + let shorter = m1.len().min(streams.first().map_or(0, |s| s.len())); + band.divergence_at = witness + .divergence_at + .or_else(|| u32::try_from(shorter).ok()); + band.top2_margin_at_divergence = witness.top2_margin_at_divergence; + band.intra_agree_to = witness.intra_agree_to; + band.max_constant_run = witness.max_constant_run; + band.result = witness.batch_invariance; + band.reason = match band.result { + BatchInvariance::Pass => None, + BatchInvariance::Fail => fail_reason(&witness, policy), + BatchInvariance::Unmeasurable => Some(format!( + "no slot reached declared_min={}", + policy.declared_min + )), + }; + band +} + +/// The run's verdict: FAIL on any failed band; UNMEASURABLE when the reference +/// was not stable, a band could not be decided, or no band at `c > 1` ran. +#[must_use] +pub fn run_verdict(reference: &ProbeReference, bands: &[ProbeBand]) -> BatchInvariance { + if bands.iter().any(|b| b.result == BatchInvariance::Fail) { + return BatchInvariance::Fail; + } + let undecided = bands + .iter() + .any(|b| b.result == BatchInvariance::Unmeasurable); + if !reference.stable || undecided || !bands.iter().any(|b| b.c > 1) { + return BatchInvariance::Unmeasurable; + } + BatchInvariance::Pass +} + +/// The largest `m` in the scheduler's `Batch m=N done` lines of `log`, or 0. +#[must_use] +pub fn max_batch_formed(log: &str) -> u32 { + log.match_indices(BATCH_MARK) + .filter_map(|(at, _)| { + let rest = &log[at + BATCH_MARK.len()..]; + let end = rest + .find(|ch: char| !ch.is_ascii_digit()) + .unwrap_or(rest.len()); + let (digits, tail) = rest.split_at(end); + tail.starts_with(" done") + .then_some(digits) + .and_then(|d| d.parse::().ok()) + }) + .max() + .unwrap_or(0) +} + +/// The server log's length now, so a band reads only what it caused. A log +/// that cannot be read has length 0. +#[must_use] +pub fn log_offset(server_log: &Path) -> u64 { + std::fs::metadata(server_log).map_or(0, |m| m.len()) +} + +/// The largest batch the scheduler logged after `offset` bytes, 0 when the log +/// cannot be read. Reading from the band's own offset is what keeps a batch +/// formed during c=4 from being credited to c=8. +#[must_use] +pub fn batch_formed_since(server_log: &Path, offset: u64) -> u32 { + let mut bytes = Vec::new(); + let read = std::fs::File::open(server_log).and_then(|mut file| { + file.seek(SeekFrom::Start(offset))?; + file.read_to_end(&mut bytes) + }); + read.map_or(0, |_| max_batch_formed(&String::from_utf8_lossy(&bytes))) +} + +/// The sampler every request carries. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct ProbeSampler { + /// 0 for the witness: greedy. + pub temperature: f64, + /// Pinned so two runs draw alike. + pub seed: u64, + /// So every request generates exactly `max_tokens`. + pub ignore_eos: bool, + /// `protocol.n_predict`. + pub max_tokens: u32, + /// The best tokens asked for per step; 2 gives the top-2 margin. + pub top_logprobs: u8, +} + +/// The served model: the file name, never a host path, and its bytes' sha256. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProbeModel { + /// The file name only. + pub path: Option, + /// The file's sha256. + pub sha256: Option, +} + +/// The witness a probe run writes. Identity fields the caller did not give +/// stay `null`; the V3 shape check names each one missing. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ProbeWitness { + /// [`WITNESS_VERSION`]. + pub witness_version: u32, + /// [`PROBE`]. + pub probe: String, + /// The host the server ran on. + pub host: Option, + /// The commit the server was built from. + pub commit: Option, + /// The serving binary's sha256. + pub binary_sha256: Option, + /// The served model. + pub model: ProbeModel, + /// The prompt's sha256. + pub prompt_sha256: String, + /// The sampler on every request. + pub sampler: ProbeSampler, + /// Tokens a batch's slots had to agree for. + pub declared_min: u32, + /// [`ENV_KEYS`] as this process saw them. + pub env: BTreeMap>, + /// The `m = 1` reference. + pub reference: ProbeReference, + /// One per `c` fired, in ladder order. + pub bands: Vec, + /// [`run_verdict`]. + pub result: BatchInvariance, +} + +/// The prompt's sha256, as 64 lowercase hex characters. +#[must_use] +pub fn prompt_sha256(prompt: &str) -> String { + format!("{:x}", Sha256::digest(prompt.as_bytes())) +} + +/// [`ENV_KEYS`] as this process sees them; an unset key is `None`. +#[must_use] +pub fn env_block() -> BTreeMap> { + ENV_KEYS + .iter() + .map(|key| ((*key).to_string(), std::env::var(key).ok())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::super::margin::TopLogprob; + use super::super::v3_shape::{check_v3_shape, ShapeVerdict}; + use super::*; + + const POLICY: ProbePolicy = ProbePolicy { + declared_min: 4, + n_predict: 8, + max_constant_run: 5, + }; + + /// A token whose step's runner-up trails it by `gap`. + fn tok(text: &str, gap: f64) -> TokenLogprob { + let top = |token: &str, logprob: f64| TopLogprob { + token: token.to_string(), + logprob: Some(logprob), + bytes: Some(token.as_bytes().to_vec()), + }; + TokenLogprob { + token: text.to_string(), + logprob: Some(-0.1), + bytes: Some(text.as_bytes().to_vec()), + top_logprobs: vec![top(text, -0.1), top("~", -0.1 - gap)], + } + } + + /// `n_predict` tokens named by `words`, every step decided by a margin of 1. + fn stream(words: &[&str]) -> Vec { + words.iter().map(|w| tok(w, 1.0)).collect() + } + + fn sample(tokens: Vec) -> SampleResult { + Ok(ProbeSample { + completion_tokens: u32::try_from(tokens.len()).ok(), + finish_reason: Some("length".to_string()), + logprobs: tokens, + }) + } + + const TEXT: [&str; 8] = ["a", "b", "c", "d", "e", "f", "g", "h"]; + + fn reference() -> Vec { + stream(&TEXT) + } + + fn band_of(c: u32, m_formed: u32, slots: &[SampleResult]) -> ProbeBand { + evaluate_band(c, m_formed, &reference(), slots, &POLICY) + } + + fn copies(c: u32, tokens: &[TokenLogprob]) -> Vec { + (0..c).map(|_| sample(tokens.to_vec())).collect() + } + + #[test] + fn identical_slots_pass_and_record_the_full_length_as_divergence_at() { + let band = band_of(4, 4, &copies(4, &reference())); + assert_eq!(band.result, BatchInvariance::Pass, "{band:?}"); + assert_eq!(band.divergence_at, Some(8)); + assert_eq!(band.top2_margin_at_divergence, None); + assert_eq!(band.reason, None); + } + + /// The script's `witness_constant_token_m3`: #2753, one id forever. + #[test] + fn a_frozen_batch_fails_and_names_the_signature() { + let frozen = stream(&["z"; 8]); + let band = band_of(3, 3, &copies(3, &frozen)); + assert_eq!(band.result, BatchInvariance::Fail, "{band:?}"); + assert!( + band.reason.as_deref().is_some_and(|r| r.contains("#2753")), + "{band:?}" + ); + } + + /// `witness_no_batch_formed_is_unmeasurable`: c > 1 with no batch logged. + #[test] + fn a_band_where_no_batch_formed_is_unmeasurable_never_a_pass() { + let band = band_of(4, 1, &copies(4, &reference())); + assert_eq!(band.result, BatchInvariance::Unmeasurable); + assert!(band + .reason + .as_deref() + .is_some_and(|r| r.contains("no batch with m>1"))); + assert_eq!(band.divergence_at, None); + } + + /// c = 1 needs no batch. + #[test] + fn the_c1_band_passes_with_m_formed_1() { + let band = band_of(1, 1, &copies(1, &reference())); + assert_eq!(band.result, BatchInvariance::Pass, "{band:?}"); + } + + /// `witness_short_slot_is_unmeasurable`: PP-28 per slot. + #[test] + fn a_short_slot_makes_the_band_unmeasurable_and_names_the_slot() { + let mut slots = copies(4, &reference()); + slots[2] = sample(reference()[..6].to_vec()); + let band = band_of(4, 4, &slots); + assert_eq!(band.result, BatchInvariance::Unmeasurable); + let reason = band.reason.unwrap_or_default(); + assert!( + reason.contains("slot 2: completion_tokens=6 != n_predict=8"), + "{reason}" + ); + } + + #[test] + fn a_slot_without_logprob_entries_is_refused_by_name() { + let mut slots = copies(2, &reference()); + slots[1] = Ok(ProbeSample { + logprobs: Vec::new(), + completion_tokens: Some(8), + finish_reason: None, + }); + let band = band_of(2, 2, &slots); + assert_eq!(band.result, BatchInvariance::Unmeasurable); + assert!(band + .reason + .unwrap_or_default() + .contains("must ask for logprobs")); + } + + #[test] + fn a_failed_request_or_a_missing_slot_is_unmeasurable() { + let mut slots = copies(2, &reference()); + slots[0] = Err("connection refused".to_string()); + let band = band_of(2, 2, &slots); + assert_eq!(band.result, BatchInvariance::Unmeasurable); + assert!(band + .reason + .unwrap_or_default() + .contains("slot 0: connection refused")); + let band = band_of(4, 4, &copies(3, &reference())); + assert_eq!(band.result, BatchInvariance::Unmeasurable); + assert!(band + .reason + .unwrap_or_default() + .contains("fired c=4 requests, 3 came back")); + } + + /// `witness_intra_batch_disagree_m4`: slot 3 parts from slot 0 at 2 < 4. + #[test] + fn slots_that_disagree_before_declared_min_fail() { + let mut slots = copies(4, &reference()); + let mut odd = reference(); + odd[2] = tok("X", 1.0); + slots[3] = sample(odd); + let band = band_of(4, 4, &slots); + assert_eq!(band.result, BatchInvariance::Fail, "{band:?}"); + assert_eq!(band.intra_agree_to, Some(2)); + assert!(band + .reason + .unwrap_or_default() + .contains("disagree at token 2")); + } + + /// `witness_kernel_family_flip_recorded_ok`, now with the margin: the whole + /// batch parts from m=1 at a near-tie, which is recorded and still PASS. + #[test] + fn a_batch_that_parts_from_m1_at_a_near_tie_passes_and_records_the_margin() { + let mut m1 = reference(); + m1[1] = tok("b", 0.01); + let mut flipped = reference(); + flipped[1] = tok("~", 0.01); + let slots = copies(4, &flipped); + let band = evaluate_band(4, 4, &m1, &slots, &POLICY); + assert_eq!(band.result, BatchInvariance::Pass, "{band:?}"); + assert_eq!(band.divergence_at, Some(1)); + let margin = band.top2_margin_at_divergence.expect("margin recorded"); + assert!((margin - 0.01).abs() < 1e-9, "{margin}"); + } + + #[test] + fn the_reference_is_stable_only_when_both_decodes_agree_to_declared_min() { + let ok = check_reference(&sample(reference()), &sample(reference()), &POLICY); + assert!(ok.stable, "{ok:?}"); + assert_eq!(ok.self_divergence_at, Some(8)); + let mut early = reference(); + early[2] = tok("X", 1.0); + let unstable = check_reference(&sample(reference()), &sample(early), &POLICY); + assert!(!unstable.stable); + assert_eq!(unstable.self_divergence_at, Some(2)); + let short = check_reference( + &sample(reference()[..3].to_vec()), + &sample(reference()), + &POLICY, + ); + assert!(!short.stable); + assert!(short + .reason + .unwrap_or_default() + .starts_with("ref#1: completion_tokens=3")); + let gone = check_reference(&sample(reference()), &Err("timeout".into()), &POLICY); + assert_eq!(gone.reason.as_deref(), Some("ref#2: timeout")); + } + + fn passed(c: u32) -> ProbeBand { + band_of(c, c, &copies(c, &reference())) + } + + #[test] + fn the_run_passes_only_with_a_measured_batched_band_and_a_stable_reference() { + let stable = ProbeReference { + stable: true, + ..ProbeReference::default() + }; + let full = vec![passed(1), passed(4)]; + assert_eq!(run_verdict(&stable, &full), BatchInvariance::Pass); + // `witness_c1_only_is_not_a_pass`. + assert_eq!( + run_verdict(&stable, &[passed(1)]), + BatchInvariance::Unmeasurable + ); + assert_eq!(run_verdict(&stable, &[]), BatchInvariance::Unmeasurable); + assert_eq!( + run_verdict(&ProbeReference::default(), &full), + BatchInvariance::Unmeasurable + ); + let undecided = vec![passed(4), band_of(8, 1, &copies(8, &reference()))]; + assert_eq!( + run_verdict(&stable, &undecided), + BatchInvariance::Unmeasurable + ); + let frozen = band_of(3, 3, &copies(3, &stream(&["z"; 8]))); + assert_eq!( + run_verdict(&stable, &[passed(4), frozen]), + BatchInvariance::Fail + ); + } + + /// `witness_cross_band_log_not_credited`: the scheduler's lines are read + /// from the band's own offset. + #[test] + fn batch_formed_since_reads_only_past_the_offset() { + let dir = tempfile::tempdir().expect("tempdir"); + let log = dir.path().join("server.log"); + std::fs::write( + &log, + "[PMAT-044] Batch m=4 done in 3.0ms (9.0 tok/s/slot)\n", + ) + .expect("write"); + let offset = log_offset(&log); + assert_eq!(batch_formed_since(&log, 0), 4); + assert_eq!(batch_formed_since(&log, offset), 0); + let mut text = std::fs::read_to_string(&log).expect("read"); + text.push_str("[PMAT-044] Batch m=2 done in 1.0ms (9.0 tok/s/slot)\n"); + std::fs::write(&log, text).expect("append"); + assert_eq!(batch_formed_since(&log, offset), 2); + assert_eq!(batch_formed_since(&dir.path().join("absent.log"), 0), 0); + assert_eq!(log_offset(&dir.path().join("absent.log")), 0); + } + + #[test] + fn max_batch_formed_takes_the_largest_well_formed_line() { + let log = "Batch m=3 done\nBatch m=16 done in 2ms\nBatch m=99 started\nBatch m= done\nBatch m=8 done"; + assert_eq!(max_batch_formed(log), 16); + assert_eq!(max_batch_formed(""), 0); + assert_eq!(max_batch_formed("Batch m=12"), 0); + } + + fn witness(bands: Vec) -> ProbeWitness { + let reference = ProbeReference { + stable: true, + ..ProbeReference::default() + }; + ProbeWitness { + witness_version: WITNESS_VERSION, + probe: PROBE.to_string(), + host: Some("lambda".to_string()), + commit: Some("f".repeat(40)), + binary_sha256: Some("a".repeat(64)), + model: ProbeModel { + path: Some("Qwen3.5-4B-Q4_K_M.gguf".to_string()), + sha256: Some("c".repeat(64)), + }, + prompt_sha256: prompt_sha256("Write an essay on compilers."), + sampler: ProbeSampler { + temperature: 0.0, + seed: 0, + ignore_eos: true, + max_tokens: POLICY.n_predict, + top_logprobs: 2, + }, + declared_min: POLICY.declared_min, + env: env_block(), + result: run_verdict(&reference, &bands), + reference, + bands, + } + } + + /// Producer against consumer: what this module writes for a clean run is + /// what the V3 shape check admits. A band that never parts from m=1 must + /// still record `divergence_at`, or V3 reads it as unmeasured. + #[test] + fn a_clean_run_writes_a_witness_the_v3_shape_check_admits() { + let w = witness(vec![passed(1), passed(4), passed(8), passed(16)]); + assert_eq!(w.result, BatchInvariance::Pass); + let text = serde_json::to_string_pretty(&w).expect("serialize"); + assert_eq!(check_v3_shape(&text), ShapeVerdict::Admissible, "{text}"); + let back: ProbeWitness = serde_json::from_str(&text).expect("round trip"); + assert_eq!(back, w); + } + + /// The same, with the c=8 band parting from m=1 at a near-tie: admitted on + /// the margin. Without the margin it would not be. + #[test] + fn a_near_tie_divergence_is_admitted_by_v3_only_with_its_margin() { + let mut m1 = reference(); + m1[1] = tok("b", 0.01); + let mut flipped = reference(); + flipped[1] = tok("~", 0.01); + let flip = evaluate_band(8, 8, &m1, &copies(8, &flipped), &POLICY); + let mut w = witness(vec![passed(1), passed(4), flip, passed(16)]); + let text = serde_json::to_string(&w).expect("serialize"); + assert_eq!(check_v3_shape(&text), ShapeVerdict::Admissible, "{text}"); + w.bands[2].top2_margin_at_divergence = None; + let text = serde_json::to_string(&w).expect("serialize"); + assert!( + matches!(check_v3_shape(&text), ShapeVerdict::NotAdmissible(_)), + "{text}" + ); + } + + #[test] + fn missing_identity_stays_null_and_v3_names_it() { + let mut w = witness(vec![passed(1), passed(4), passed(8), passed(16)]); + w.commit = None; + let text = serde_json::to_string(&w).expect("serialize"); + assert!(text.contains("\"commit\":null"), "{text}"); + assert_eq!( + check_v3_shape(&text), + ShapeVerdict::NotAdmissible(vec!["identity field commit missing".to_string()]) + ); + } + + #[test] + fn env_block_records_every_key_and_prompt_sha256_is_64_hex() { + let env = env_block(); + assert_eq!(env.len(), ENV_KEYS.len()); + let sha = prompt_sha256(""); + assert_eq!( + sha, + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + } +} From 4789f169e14d0ebdb7c0f58539e36310d2b985e8 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 08:26:36 +0200 Subject: [PATCH 18/35] feat(test-lib): the perf041 probe's requests over LlmClient (PMAT-4971) V3-d, second slice, under the llm feature: ParityProbe fires the two m=1 references one after the other, then each band's c requests at once, every one asking for logprobs with the sampler's top_logprobs, and reads the server log from the band's own offset for the batch it formed. An unstable reference ends the run with no bands. The verdicts are perf_gate::parity's. Tests replay canned SSE from a loopback server that appends the scheduler's Batch m=N line from inside its handler, as the script's _replay_server does: a clean ladder passes, a box that never batches is UNMEASURABLE, frozen batched slots FAIL, a near-tie flip passes with its margin recorded, an untrusted reference runs no bands. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5b3bf75303cbcb989b79447a624e32fe68916178) (cherry picked from commit 97546b38a6dbfb895135d61d797491cdeb154045) (cherry picked from commit 3c2f2cd041a8bfb5f97cc74df98906625b644f51) --- crates/aprender-test-lib/src/llm/mod.rs | 6 + crates/aprender-test-lib/src/llm/parity.rs | 361 +++++++++++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 crates/aprender-test-lib/src/llm/parity.rs diff --git a/crates/aprender-test-lib/src/llm/mod.rs b/crates/aprender-test-lib/src/llm/mod.rs index 76e5daf4e4..d966793c41 100644 --- a/crates/aprender-test-lib/src/llm/mod.rs +++ b/crates/aprender-test-lib/src/llm/mod.rs @@ -20,6 +20,10 @@ pub mod experiment; pub mod gpu_telemetry; #[cfg(feature = "llm")] pub mod loadtest; +/// #4971 V3-d: the perf041 batched-parity probe's requests; the verdicts +/// live in `crate::perf_gate::parity`. +#[cfg(feature = "llm")] +pub mod parity; pub mod prompts; #[cfg(feature = "llm")] pub mod report; @@ -52,6 +56,8 @@ pub use loadtest::{ LoadTest, LoadTestConfig, LoadTestResult, QualityFailure, QualityResult, RequestDetail, RequestRate, SweepLevel, SweepResult, TailAnalysis, TelemetryStat, ValidationMode, }; +#[cfg(feature = "llm")] +pub use parity::{ParityProbe, ParityRun}; pub use prompts::{ assert_prompt_tokens_in_band, load_corpus as load_prompt_corpus, load_from_file as load_prompts_from_file, load_profile, Corpus, PromptProfile, PromptTokenBand, diff --git a/crates/aprender-test-lib/src/llm/parity.rs b/crates/aprender-test-lib/src/llm/parity.rs new file mode 100644 index 0000000000..8eee310f47 --- /dev/null +++ b/crates/aprender-test-lib/src/llm/parity.rs @@ -0,0 +1,361 @@ +//! #4971 V3-d: the perf041 probe's requests. +//! +//! What a run means is decided in [`crate::perf_gate::parity`], under default +//! features. This module only fires the requests: two `m = 1` references one +//! after the other, then for each `c` of the ladder `c` identical requests at +//! once, reading the server log from the band's own offset to learn the batch +//! it formed. Every request asks for `logprobs` with the sampler's +//! `top_logprobs`, so each band can record the reference's top-2 margin. + +use std::path::PathBuf; + +use futures::future::join_all; + +use super::client::{ChatMessage, ChatRequest, LlmClient, Role}; +use crate::perf_gate::parity::{ + batch_formed_since, check_reference, evaluate_band, log_offset, run_verdict, ProbeBand, + ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, SampleResult, +}; +use crate::perf_gate::witness::BatchInvariance; + +/// One probe run's settings. +#[derive(Debug, Clone)] +pub struct ParityProbe { + /// The prompt every request sends. + pub prompt: String, + /// The concurrencies to fire, in order. + pub ladder: Vec, + /// What the bands are judged against. + pub policy: ProbePolicy, + /// What every request carries. + pub sampler: ProbeSampler, + /// The server's log, where the scheduler writes `Batch m=N done`. + pub server_log: PathBuf, +} + +/// What a probe run measured. +#[derive(Debug, Clone, PartialEq)] +pub struct ParityRun { + /// The two `m = 1` decodes, judged. + pub reference: ProbeReference, + /// One per `c` fired. Empty when the reference was not stable. + pub bands: Vec, + /// [`run_verdict`] over the two. + pub result: BatchInvariance, +} + +impl ParityProbe { + /// The request every slot sends. The model is left to the client. + #[must_use] + pub fn request(&self) -> ChatRequest { + let prompt = ChatMessage { + role: Role::User, + content: self.prompt.clone(), + }; + ChatRequest { + temperature: Some(self.sampler.temperature), + max_tokens: Some(self.sampler.max_tokens), + seed: Some(self.sampler.seed), + ignore_eos: Some(self.sampler.ignore_eos), + logprobs: Some(true), + top_logprobs: Some(self.sampler.top_logprobs), + ..ChatRequest::new("", vec![prompt]) + } + } + + /// The two references, then each band. A reference that is not stable + /// ends the run with no bands: no batched difference after it could be + /// laid at batching's door. + pub async fn run(&self, client: &LlmClient) -> ParityRun { + let request = self.request(); + let first = fire(client, &request).await; + let second = fire(client, &request).await; + let reference = check_reference(&first, &second, &self.policy); + let m1 = match first { + Ok(m1) if reference.stable => m1.logprobs, + _ => { + return ParityRun { + result: run_verdict(&reference, &[]), + reference, + bands: Vec::new(), + } + } + }; + let mut bands = Vec::with_capacity(self.ladder.len()); + for &c in &self.ladder { + let offset = log_offset(&self.server_log); + let samples = join_all((0..c).map(|_| fire(client, &request))).await; + let m_formed = batch_formed_since(&self.server_log, offset); + bands.push(evaluate_band(c, m_formed, &m1, &samples, &self.policy)); + } + ParityRun { + result: run_verdict(&reference, &bands), + reference, + bands, + } + } +} + +/// One streamed request, as the probe records it. +async fn fire(client: &LlmClient, request: &ChatRequest) -> SampleResult { + client + .chat_completion_stream(request) + .await + .map(|r| ProbeSample { + logprobs: r.logprobs, + completion_tokens: Some(r.usage.completion_tokens), + finish_reason: r.finish_reason, + }) + .map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use std::io::Write as _; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + use serde_json::json; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + use super::*; + + /// What the replay server sends for the request with this global index: + /// `(token, gap to the runner-up)` per step, and a scheduler line to log. + type Plan = Arc (Vec<(String, f64)>, Option) + Send + Sync>; + + const POLICY: ProbePolicy = ProbePolicy { + declared_min: 4, + n_predict: 8, + max_constant_run: 4, + }; + + fn canned(prefix: &str) -> Vec<(String, f64)> { + (0..8).map(|i| (format!("{prefix}{i} "), 1.0)).collect() + } + + /// One SSE frame carrying one token and its logprob entry. + fn token_frame(token: &str, gap: f64) -> String { + let entry = json!({ + "token": token, + "logprob": -0.1, + "bytes": token.as_bytes(), + "top_logprobs": [ + {"token": token, "logprob": -0.1}, + {"token": "~", "logprob": -0.1 - gap}, + ], + }); + let chunk = + json!({"choices": [{"delta": {"content": token}, "logprobs": {"content": [entry]}}]}); + format!("data: {chunk}\n\n") + } + + fn sse_body(tokens: &[(String, f64)]) -> String { + let n = tokens.len(); + let usage = json!({"prompt_tokens": 5, "completion_tokens": n, "total_tokens": n + 5}); + let terminal = + json!({"choices": [{"delta": {}, "finish_reason": "length"}], "usage": usage}); + std::iter::once( + "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n".to_string(), + ) + .chain(tokens.iter().map(|(t, gap)| token_frame(t, *gap))) + .chain([ + format!("data: {terminal}\n\n"), + "data: [DONE]\n\n".to_string(), + ]) + .collect() + } + + /// Read one request through its body, so the client never sees a reset. + async fn read_request(sock: &mut tokio::net::TcpStream) { + let mut buf = Vec::new(); + let mut chunk = [0_u8; 4096]; + while let Ok(n @ 1..) = sock.read(&mut chunk).await { + buf.extend_from_slice(&chunk[..n]); + let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { + continue; + }; + let head = String::from_utf8_lossy(&buf[..end]).to_ascii_lowercase(); + let body = head + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(0); + if buf.len() >= end + 4 + body { + return; + } + } + } + + /// The script's `_replay_server`: canned SSE per request index, and the + /// scheduler's line appended to the log from inside the handler, as a real + /// server does, so the band offsets are exercised for real. + async fn replay(plan: Plan, log: PathBuf) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback"); + let addr = listener.local_addr().expect("local addr"); + let counter = Arc::new(AtomicUsize::new(0)); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let (plan, log, counter) = (Arc::clone(&plan), log.clone(), Arc::clone(&counter)); + tokio::spawn(async move { + read_request(&mut sock).await; + let (tokens, line) = plan(counter.fetch_add(1, Ordering::SeqCst)); + if let Some(line) = line { + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&log) + .expect("open log"); + writeln!(file, "{line}").expect("append log"); + } + let body = sse_body(&tokens); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = sock.write_all((head + &body).as_bytes()).await; + let _ = sock.shutdown().await; + }); + } + }); + format!("http://{addr}") + } + + /// Run the probe over the ladder [1, 2] against `plan`. Requests 0 and 1 + /// are the references, 2 is c=1, and 3 and 4 are c=2. + async fn probe( + plan: impl Fn(usize) -> (Vec<(String, f64)>, Option) + Send + Sync + 'static, + ) -> ParityRun { + let dir = tempfile::tempdir().expect("tempdir"); + let log = dir.path().join("server.log"); + std::fs::write(&log, "server starting\n").expect("seed log"); + let url = replay(Arc::new(plan), log.clone()).await; + let probe = ParityProbe { + prompt: "Write an essay on compilers.".to_string(), + ladder: vec![1, 2], + policy: POLICY, + sampler: ProbeSampler { + temperature: 0.0, + seed: 0, + ignore_eos: true, + max_tokens: POLICY.n_predict, + top_logprobs: 2, + }, + server_log: log, + }; + probe.run(&LlmClient::new(url, "replay")).await + } + + /// The batch line a request logs: m=1 for the c=1 band, m=2 for c=2. + fn batch_line(index: usize) -> Option { + match index { + 2 => Some("[PMAT-044] Batch m=1 done in 1.0ms (9.0 tok/s/slot)".to_string()), + 3.. => Some("[PMAT-044] Batch m=2 done in 1.0ms (9.0 tok/s/slot)".to_string()), + _ => None, + } + } + + #[tokio::test] + async fn identical_streams_with_batches_formed_pass_every_band() { + let run = probe(|i| (canned("a"), batch_line(i))).await; + assert_eq!(run.result, BatchInvariance::Pass, "{run:?}"); + assert!(run.reference.stable); + let formed: Vec = run.bands.iter().map(|b| b.m_formed).collect(); + assert_eq!(formed, vec![1, 2], "each band reads only its own lines"); + assert_eq!(run.bands[1].divergence_at, Some(8)); + } + + /// `witness_no_batch_formed_is_unmeasurable`, end to end. + #[tokio::test] + async fn a_box_that_never_batches_is_unmeasurable_never_a_pass() { + let run = probe(|_| (canned("a"), None)).await; + assert_eq!(run.result, BatchInvariance::Unmeasurable, "{run:?}"); + assert_eq!(run.bands[1].result, BatchInvariance::Unmeasurable); + } + + /// `witness_constant_token_m3`, end to end: the batched slots freeze. + #[tokio::test] + async fn frozen_batched_slots_fail() { + let frozen = |i: usize| { + let tokens = if i >= 3 { + vec![("z".to_string(), 1.0); 8] + } else { + canned("a") + }; + (tokens, batch_line(i)) + }; + let run = probe(frozen).await; + assert_eq!(run.result, BatchInvariance::Fail, "{run:?}"); + assert_eq!(run.bands[1].result, BatchInvariance::Fail); + } + + /// The margin, end to end: the batch parts from m=1 at token 1, where the + /// reference's own choice was a near-tie; the band passes and records it. + #[tokio::test] + async fn a_near_tie_flip_is_recorded_with_its_margin() { + let flip = |i: usize| { + let mut tokens = canned("a"); + tokens[1] = if i >= 3 { + ("flip ".to_string(), 0.01) + } else { + ("a1 ".to_string(), 0.01) + }; + (tokens, batch_line(i)) + }; + let run = probe(flip).await; + assert_eq!(run.result, BatchInvariance::Pass, "{run:?}"); + assert_eq!(run.bands[1].divergence_at, Some(1)); + let margin = run.bands[1].top2_margin_at_divergence.expect("margin"); + assert!((margin - 0.01).abs() < 1e-9, "{margin}"); + } + + /// `witness_short_reference_is_unmeasurable`, and an unstable reference: + /// either ends the run before any band. + #[tokio::test] + async fn a_reference_that_cannot_be_trusted_ends_the_run_with_no_bands() { + let unstable = |i: usize| { + let mut tokens = canned("a"); + if i == 1 { + tokens[2] = ("X ".to_string(), 1.0); + } + (tokens, batch_line(i)) + }; + let run = probe(unstable).await; + assert_eq!(run.result, BatchInvariance::Unmeasurable, "{run:?}"); + assert!(run.bands.is_empty()); + assert_eq!(run.reference.self_divergence_at, Some(2)); + let short = probe(|i| (canned("a")[..5].to_vec(), batch_line(i))).await; + assert_eq!(short.result, BatchInvariance::Unmeasurable); + assert!(short.bands.is_empty()); + } + + #[test] + fn every_request_asks_for_logprobs_and_pins_the_sampler() { + let probe = ParityProbe { + prompt: "p".to_string(), + ladder: vec![1], + policy: POLICY, + sampler: ProbeSampler { + temperature: 0.0, + seed: 7, + ignore_eos: true, + max_tokens: 8, + top_logprobs: 2, + }, + server_log: PathBuf::from("server.log"), + }; + let body = serde_json::to_value(probe.request()).expect("serialize"); + for (key, want) in [ + ("logprobs", json!(true)), + ("top_logprobs", json!(2)), + ("seed", json!(7)), + ("ignore_eos", json!(true)), + ("temperature", json!(0.0)), + ("max_tokens", json!(8)), + ] { + assert_eq!(body[key], want, "{key}: {body}"); + } + } +} From 712b4f20ba95ec7b5df9dcebdc14d8aa4b7ed569 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 08:38:51 +0200 Subject: [PATCH 19/35] feat(apr-cli): `apr test llm parity`, the perf041 probe's front door (PMAT-4971) The Rust port of scripts/perf041_batched_parity_probe.py now has a command. It fires the probe at --url, reads --server-log for the batch each band formed, prints the verdict, the reference and every band, and writes the witness that `apr test llm shape-check` reads. Exits: PASS 0, FAIL 5 (ValidationFailed), UNMEASURABLE 8 (InferenceFailed). FAIL and UNMEASURABLE stay apart, as the script's 1 and 2 do. A server log, binary or model that is not a file exits 3 before any request is fired. perf_gate::parity::ProbeMatrix reads the policy, the sampler and the ladder from the compiled-in perf-matrix.yaml. The script falls back to its own literals when a key is missing. ProbeMatrix refuses instead and names the key. It also refuses an empty ladder, a 0 in the ladder, a max_constant_run of 0, and a declared_min above n_predict. --ladder replaces only the ladder. The witness records the model by file name only, beside its sha256. An unknown host stays null and is never made up. Not a gate: no merge, queue or release job runs it. Falsifier rows: FALSIFY-PERF041-PORT-4971 (library) and FALSIFY-PERF041-CLI-4971. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 445cdee51bc4190fa17305365c79a238830c6123) (cherry picked from commit e2415f34b7a104ad78e8f1f9015e5275e79468f8) (cherry picked from commit 891edf31307165b1e975b13f0663151ae53d14f6) --- contracts/apr-cli-commands-v1.yaml | 7 +- contracts/apr-serve-openai-compat-v1.yaml | 12 + crates/apr-cli/src/commands/mod.rs | 4 + crates/apr-cli/src/commands/test_llm.rs | 21 +- .../apr-cli/src/commands/test_llm_parity.rs | 523 ++++++++++++++++++ crates/apr-cli/src/extended_commands.rs | 51 ++ crates/apr-cli/src/validate.rs | 7 +- crates/aprender-test-lib/src/perf_gate/mod.rs | 5 +- .../aprender-test-lib/src/perf_gate/parity.rs | 186 +++++++ .../src/perf_gate/protocol.rs | 2 +- 10 files changed, 810 insertions(+), 8 deletions(-) create mode 100644 crates/apr-cli/src/commands/test_llm_parity.rs diff --git a/contracts/apr-cli-commands-v1.yaml b/contracts/apr-cli-commands-v1.yaml index 6d656e5a7b..0376c34a07 100644 --- a/contracts/apr-cli-commands-v1.yaml +++ b/contracts/apr-cli-commands-v1.yaml @@ -486,10 +486,11 @@ commands: replay the runner itself (record, playbook, coverage, report) Routed today: `tensor` (PMAT-481 visual regression), `llm bench` - (GH-876 Milestone 2), `llm ttft-verdict` (#4954) and `llm shape-check` + (GH-876 Milestone 2), `llm ttft-verdict` (#4954), `llm shape-check` (#4971: checks a perf041 witness against the V3 serving-shape rule; reads - a file, no network). The rest land as they are delegated to the probador - library. + a file, no network) and `llm parity` (#4971: the perf041 batched-parity + probe, ported from the script; drives an endpoint and writes the + witness). The rest land as they are delegated to the probador library. requires_model: false side_effects: [filesystem, network] # tensor — export tensor activations for visual regression testing (PMAT-481) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index b5733f66ee..1c2d3087f4 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -712,3 +712,15 @@ falsification_tests: test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::margin && cargo test -p aprender-test-lib --features llm --lib -- llm::client::tests::a_logprobs_request llm::client::tests::a_request_without_logprobs' expected_output: "test result: ok" if_fails: 'A V3 witness can only carry a margin someone typed in: a near-tie flip at c > 1 reads as NOT ADMISSIBLE, or a margin from the wrong stream or step (or a 0.0 for a step with no runner-up) makes a wrong-KV divergence read as a near-tie.' + - id: FALSIFY-PERF041-PORT-4971 + name: the_perf041_probe_decides_in_rust_as_the_script_does + prediction: 'perf_gate::parity ports the perf041 probe''s judgement (C301). The m=1 reference is stable only when its two decodes agree to declared_min; a c > 1 band whose log shows no batch of m >= 2 formed since the band''s offset is UNMEASURABLE, never PASS; a slot whose completion_tokens or logprob entries differ from n_predict makes its band UNMEASURABLE by name; a band FAILs when its slots part before declared_min or one emits a token max_constant_run times; and a run with no measured c > 1 band, or an unstable reference, is UNMEASURABLE. Each band records the m=1 reference''s top-2 margin at its divergence, so a clean run writes a witness the V3 shape check admits and a near-tie flip carries its margin. ProbeMatrix reads declared_min, max_constant_run, n_predict, the sampler and the ladder from perf-matrix.yaml and refuses, by name, a missing key, an empty ladder or a 0 in it, max_constant_run 0, and declared_min above n_predict. llm::parity fires the requests against a canned SSE server: identical streams with batches formed PASS every band, a box that never batches is UNMEASURABLE, frozen slots FAIL, a flip at a near-tie PASSes with its margin, and an untrustworthy reference ends the run with no bands. This is not a gate: no merge, queue or release job runs it.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::parity && cargo test -p aprender-test-lib --features llm --lib llm::parity' + expected_output: "test result: ok" + if_fails: 'The V3 witness can only come from the Python script (C301), or the Rust probe reads a box that never batched, or a reference that disagrees with itself, as batch-invariant.' + - id: FALSIFY-PERF041-CLI-4971 + name: apr_test_llm_parity_exits_by_the_verdict_and_writes_the_witness + prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --json before exiting, and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' + test_harness: 'cargo test -p apr-cli --lib test_llm_parity' + expected_output: "test result: ok" + if_fails: 'The Rust probe has no front door, so the V3 witness is still produced by the Python script by hand; or a FAIL and an UNMEASURABLE leave the same exit code; or a witness records a host path or a made-up host.' diff --git a/crates/apr-cli/src/commands/mod.rs b/crates/apr-cli/src/commands/mod.rs index fa5327a510..63d2db2e24 100644 --- a/crates/apr-cli/src/commands/mod.rs +++ b/crates/apr-cli/src/commands/mod.rs @@ -159,6 +159,10 @@ pub(crate) mod test_llm_band; pub(crate) mod test_llm_ttft; // #4971 V3-d: `apr test llm shape-check`, the CLI over perf_gate::v3_shape. pub(crate) mod test_llm_shape; +// #4971 V3-d: `apr test llm parity`, the CLI over the perf041 probe's Rust +// port (apr_test::llm::parity). It drives a server through tokio. +#[cfg(feature = "inference")] +pub(crate) mod test_llm_parity; // #2399: gated on the crate it actually needs (aprender-explain, aliased // `trueno-explain`) rather than on `full`, so `--features ptx` is enough and a // user does not have to pull CUDA + training to analyze a .ptx file. diff --git a/crates/apr-cli/src/commands/test_llm.rs b/crates/apr-cli/src/commands/test_llm.rs index 0cded9aa3c..db2716400f 100644 --- a/crates/apr-cli/src/commands/test_llm.rs +++ b/crates/apr-cli/src/commands/test_llm.rs @@ -469,7 +469,7 @@ use crate::LlmSubcommand; /// Route `apr test llm ` (GH-876 Milestone 2; PERF-025 band mode; /// #4954 ttft-verdict and #4971 shape-check, which read files and need no -/// runtime). +/// runtime, and #4971 parity, the perf041 probe). /// /// # Errors /// Propagates whichever mode ran. @@ -477,10 +477,29 @@ pub fn dispatch(command: &LlmSubcommand) -> Result<()> { match command { LlmSubcommand::TtftVerdict { runs } => super::test_llm_ttft::run(runs), LlmSubcommand::ShapeCheck { witness } => super::test_llm_shape::run(witness), + LlmSubcommand::Parity { .. } => dispatch_parity(command), LlmSubcommand::Bench { .. } => dispatch_bench(command), } } +/// `apr test llm parity`: the probe drives its server through tokio. +#[cfg(feature = "inference")] +fn dispatch_parity(command: &LlmSubcommand) -> Result<()> { + super::test_llm_parity::run(command) +} + +/// As [`dispatch_bench`]'s stub: a minimal build names the missing feature. +/// +/// # Errors +/// Always: this build cannot run the probe. +#[cfg(not(feature = "inference"))] +fn dispatch_parity(_command: &LlmSubcommand) -> Result<()> { + Err(CliError::InferenceFailed( + "`apr test llm parity` needs the `inference` feature (this apr was built without it)" + .to_string(), + )) +} + /// `apr test llm bench`, in either of its two modes. #[cfg(feature = "inference")] fn dispatch_bench(command: &LlmSubcommand) -> Result<()> { diff --git a/crates/apr-cli/src/commands/test_llm_parity.rs b/crates/apr-cli/src/commands/test_llm_parity.rs new file mode 100644 index 0000000000..a1e1dbb0ed --- /dev/null +++ b/crates/apr-cli/src/commands/test_llm_parity.rs @@ -0,0 +1,523 @@ +//! `apr test llm parity`, the front door of `apr_test::llm::parity` +//! (#4971 V3-d): the perf041 batched-parity probe, ported from +//! `scripts/perf041_batched_parity_probe.py` (C301). +//! +//! It decodes one prompt twice alone (`m = 1`, the reference), then as `c` +//! identical concurrent requests for each `c` of the ladder, reads the batch +//! each band formed from the server's log, and writes the witness that +//! `apr test llm shape-check` reads. Every number it judges by comes from +//! `scripts/perf-matrix.yaml`, compiled in; only the ladder can be overridden. +//! +//! The script's exit codes (0 PASS, 1 FAIL, 2 UNMEASURABLE) map onto apr's: +//! +//! | Verdict | apr exit | +//! |---|---| +//! | PASS | 0 | +//! | FAIL: a batch's slots parted before `declared_min`, or a slot froze | 5 (`ValidationFailed`) | +//! | UNMEASURABLE: the run could not decide | 8 (`InferenceFailed`) | +//! | a server log, binary or model that is not a readable file | 3 (`FileNotFound` / `NotAFile`) or 7 (`Io`) | +//! +//! FAIL and UNMEASURABLE never share a code, and the witness is written +//! before the exit, whichever verdict it carries. A file that cannot be read +//! refuses the run before any request is fired. +//! +//! This is not a gate. No merge, queue or release job runs it. + +use std::path::Path; + +use apr_test::llm::client::LlmClient; +use apr_test::llm::{ParityProbe, ParityRun}; +use apr_test::perf_gate::parity::{env_block, prompt_sha256, PROBE, WITNESS_VERSION}; +use apr_test::perf_gate::receipt::sha256_file; +use apr_test::perf_gate::{BatchInvariance, ProbeBand, ProbeMatrix, ProbeModel, ProbeWitness}; + +use crate::error::{CliError, Result}; +use crate::LlmSubcommand; + +/// Who and what was measured, as the witness records it. A field nobody gave +/// stays `None`, and the V3 shape check names it. +#[derive(Debug, Clone, PartialEq)] +struct Identity { + host: Option, + commit: Option, + binary_sha256: Option, + model: ProbeModel, +} + +/// Run the probe, print every band, write the witness when asked, and exit +/// by the verdict. +/// +/// # Errors +/// FAIL is `ValidationFailed` and UNMEASURABLE `InferenceFailed`, each after +/// the witness is written. A server log, binary or model that is not a +/// readable file refuses the run first. +pub(crate) fn run(command: &LlmSubcommand) -> Result<()> { + let LlmSubcommand::Parity { + url, + served_model, + server_log, + prompt, + ladder, + json, + host, + commit, + binary, + model, + } = command + else { + return Err(CliError::InvalidInput( + "`apr test llm`: only `parity` routes to the parity probe".to_string(), + )); + }; + let probe = configure(prompt, ladder, server_log)?; + let identity = identify( + host.as_deref(), + commit.as_deref(), + binary.as_deref(), + model.as_deref(), + )?; + let client = LlmClient::new(url.as_str(), served_model.as_str()); + let rt = tokio::runtime::Runtime::new() + .map_err(|e| CliError::InferenceFailed(format!("tokio runtime: {e}")))?; + let witness = witness(&probe, identity, rt.block_on(probe.run(&client))); + println!("{}", report(&witness)); + if let Some(path) = json { + write(path, &witness)?; + } + outcome(&witness) +} + +/// The probe as the matrix configures it. A non-empty `ladder` replaces the +/// declared one. +/// +/// # Errors +/// `FileNotFound` / `NotAFile` for a server log that is not a file, since a +/// log the probe cannot read would turn every band UNMEASURABLE. +/// `InvalidInput` for a matrix that cannot configure a run. +fn configure(prompt: &str, ladder: &[u32], server_log: &Path) -> Result { + readable_file(server_log)?; + let matrix = ProbeMatrix::from_matrix().map_err(CliError::InvalidInput)?; + Ok(ParityProbe { + prompt: prompt.to_string(), + ladder: if ladder.is_empty() { + matrix.ladder + } else { + ladder.to_vec() + }, + policy: matrix.policy, + sampler: matrix.sampler, + server_log: server_log.to_path_buf(), + }) +} + +/// The identity block. `host` falls back to `PERF041_HOST`, then this host's +/// name; `commit` to `PERF041_COMMIT`. The model is recorded by file name +/// only, never a host path, beside its sha256. +/// +/// # Errors +/// As [`hash`], for `binary` or `model`. +fn identify( + host: Option<&str>, + commit: Option<&str>, + binary: Option<&Path>, + model: Option<&Path>, +) -> Result { + Ok(Identity { + host: given_or_env(host, "PERF041_HOST").or_else(this_host), + commit: given_or_env(commit, "PERF041_COMMIT"), + binary_sha256: binary.map(hash).transpose()?, + model: ProbeModel { + path: model + .and_then(Path::file_name) + .map(|name| name.to_string_lossy().into_owned()), + sha256: model.map(hash).transpose()?, + }, + }) +} + +/// The flag's value, else the environment's; an empty value counts as unset. +fn given_or_env(given: Option<&str>, key: &str) -> Option { + given + .map(str::to_string) + .or_else(|| std::env::var(key).ok()) + .filter(|value| !value.is_empty()) +} + +/// This host's name, or `None`, so a witness never records a made-up host. +fn this_host() -> Option { + std::fs::read_to_string("/etc/hostname") + .ok() + .map(|name| name.trim().to_string()) + .or_else(|| std::env::var("HOSTNAME").ok()) + .filter(|name| !name.is_empty()) +} + +/// `path`'s sha256. +/// +/// # Errors +/// `FileNotFound` / `NotAFile` when it is not a file; `Io` when it cannot be read. +fn hash(path: &Path) -> Result { + readable_file(path)?; + Ok(sha256_file(path)?) +} + +/// Refuse a path that is not a file, naming which way. +fn readable_file(path: &Path) -> Result<()> { + if !path.exists() { + return Err(CliError::FileNotFound(path.to_path_buf())); + } + if !path.is_file() { + return Err(CliError::NotAFile(path.to_path_buf())); + } + Ok(()) +} + +/// The witness for one run. +fn witness(probe: &ParityProbe, identity: Identity, run: ParityRun) -> ProbeWitness { + ProbeWitness { + witness_version: WITNESS_VERSION, + probe: PROBE.to_string(), + host: identity.host, + commit: identity.commit, + binary_sha256: identity.binary_sha256, + model: identity.model, + prompt_sha256: prompt_sha256(&probe.prompt), + sampler: probe.sampler, + declared_min: probe.policy.declared_min, + env: env_block(), + reference: run.reference, + bands: run.bands, + result: run.result, + } +} + +/// The printed run: the verdict, the reference, then one line per band. +fn report(witness: &ProbeWitness) -> String { + let reference = &witness.reference; + let head = format!( + "perf041 batched parity: {} (declared_min {}, n_predict {})", + witness.result.wire_token(), + witness.declared_min, + witness.sampler.max_tokens + ); + let reference_line = format!( + " m=1 reference: {} tokens={} self_divergence_at={}{}", + if reference.stable { + "stable" + } else { + "NOT STABLE" + }, + or_dash(reference.tokens), + or_dash(reference.self_divergence_at), + reason(reference.reason.as_deref()) + ); + [head, reference_line] + .into_iter() + .chain(witness.bands.iter().map(band_line)) + .collect::>() + .join("\n") +} + +/// One band, as printed. +fn band_line(band: &ProbeBand) -> String { + format!( + " c={} m_formed={} {} intra_agree_to={} divergence_at={} top2_margin={}{}", + band.c, + band.m_formed, + band.result.wire_token(), + or_dash(band.intra_agree_to), + or_dash(band.divergence_at), + or_dash(band.top2_margin_at_divergence), + reason(band.reason.as_deref()) + ) +} + +/// A value, or `-` when there is none. +fn or_dash(value: Option) -> String { + value.map_or_else(|| "-".to_string(), |v| v.to_string()) +} + +/// ` (reason)`, or nothing. +fn reason(reason: Option<&str>) -> String { + reason.map_or_else(String::new, |r| format!(" ({r})")) +} + +/// Write the witness as pretty JSON. +/// +/// # Errors +/// `Io` when the file cannot be written. +fn write(path: &Path, witness: &ProbeWitness) -> Result<()> { + let json = serde_json::to_string_pretty(witness) + .map_err(|e| CliError::InvalidInput(format!("perf041 witness as JSON: {e}")))?; + std::fs::write(path, json + "\n")?; + println!("witness: {}", path.display()); + Ok(()) +} + +/// The verdict as apr's result. Only PASS is `Ok`. +fn outcome(witness: &ProbeWitness) -> Result<()> { + match witness.result { + BatchInvariance::Pass => Ok(()), + BatchInvariance::Fail => Err(CliError::ValidationFailed(format!( + "perf041 FAIL: {}", + why(witness) + ))), + BatchInvariance::Unmeasurable => Err(CliError::InferenceFailed(format!( + "perf041 UNMEASURABLE: {}", + why(witness) + ))), + } +} + +/// The first reason behind the verdict: a band that carries it, else the +/// reference, else the one case that has neither (no band at `c > 1`). +fn why(witness: &ProbeWitness) -> String { + witness + .bands + .iter() + .filter(|band| band.result == witness.result) + .find_map(|band| band.reason.as_ref().map(|r| format!("c={}: {r}", band.c))) + .or_else(|| { + witness + .reference + .reason + .as_ref() + .map(|r| format!("m=1 reference: {r}")) + }) + .unwrap_or_else(|| "no band at c > 1 was measured".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use apr_test::perf_gate::ProbeReference; + use clap::{FromArgMatches, Subcommand}; + + fn reference(stable: bool, reason: Option<&str>) -> ProbeReference { + ProbeReference { + stable, + tokens: Some(128), + self_divergence_at: None, + reason: reason.map(str::to_string), + } + } + + fn band(c: u32, result: BatchInvariance, reason: Option<&str>) -> ProbeBand { + ProbeBand { + c, + m_formed: c, + result, + divergence_at: Some(128), + top2_margin_at_divergence: None, + intra_agree_to: Some(128), + max_constant_run: Some(1), + declared_min: 64, + max_constant_run_declared: 16, + reason: reason.map(str::to_string), + slots: vec![], + } + } + + fn log() -> tempfile::NamedTempFile { + let log = tempfile::NamedTempFile::new().expect("temp log"); + std::fs::write(log.path(), "server starting\n").expect("seed log"); + log + } + + fn witness_of(run: ParityRun) -> ProbeWitness { + let log = log(); + let probe = configure("p", &[], log.path()).expect("configures"); + let identity = identify(Some("box"), Some("abc123"), None, None).expect("identity"); + witness(&probe, identity, run) + } + + #[test] + fn the_three_verdicts_exit_apart() { + let pass = witness_of(ParityRun { + reference: reference(true, None), + bands: vec![band(4, BatchInvariance::Pass, None)], + result: BatchInvariance::Pass, + }); + assert!(outcome(&pass).is_ok()); + + let fail = witness_of(ParityRun { + reference: reference(true, None), + bands: vec![ + band(4, BatchInvariance::Pass, None), + band(8, BatchInvariance::Fail, Some("slot 3 froze")), + ], + result: BatchInvariance::Fail, + }); + let err = outcome(&fail).expect_err("FAIL"); + assert_eq!(err.exit_code_value(), 5, "{err}"); + assert!(err.to_string().contains("c=8: slot 3 froze"), "{err}"); + + let unstable = witness_of(ParityRun { + reference: reference(false, Some("the two decodes parted at 2")), + bands: vec![], + result: BatchInvariance::Unmeasurable, + }); + let err = outcome(&unstable).expect_err("UNMEASURABLE"); + assert_eq!(err.exit_code_value(), 8, "{err}"); + assert!( + err.to_string() + .contains("m=1 reference: the two decodes parted at 2"), + "{err}" + ); + + let no_batch = witness_of(ParityRun { + reference: reference(true, None), + bands: vec![band(1, BatchInvariance::Pass, None)], + result: BatchInvariance::Unmeasurable, + }); + let err = outcome(&no_batch).expect_err("UNMEASURABLE"); + assert_eq!(err.exit_code_value(), 8, "{err}"); + assert!(err.to_string().contains("no band at c > 1"), "{err}"); + } + + #[test] + fn a_server_log_that_is_not_a_file_refuses_the_run() { + let dir = tempfile::tempdir().expect("tempdir"); + let missing = dir.path().join("server.log"); + assert!(matches!( + configure("p", &[], &missing), + Err(CliError::FileNotFound(p)) if p == missing + )); + assert!(matches!( + configure("p", &[], dir.path()), + Err(CliError::NotAFile(_)) + )); + } + + #[test] + fn the_matrix_configures_the_probe_and_the_ladder_flag_replaces_its_ladder() { + let log = log(); + let matrix = ProbeMatrix::from_matrix().expect("the shipped matrix"); + let declared = configure("p", &[], log.path()).expect("configures"); + assert_eq!(declared.ladder, matrix.ladder); + assert_eq!(declared.policy, matrix.policy); + assert_eq!(declared.sampler, matrix.sampler); + assert_eq!(declared.server_log, log.path()); + let overridden = configure("p", &[2, 3], log.path()).expect("configures"); + assert_eq!(overridden.ladder, vec![2, 3]); + assert_eq!(overridden.policy, matrix.policy); + } + + #[test] + fn identity_records_the_model_by_file_name_and_hashes_both_files() { + let dir = tempfile::tempdir().expect("tempdir"); + let model = dir.path().join("model.gguf"); + let binary = dir.path().join("apr"); + std::fs::write(&model, "abc").expect("model"); + std::fs::write(&binary, "").expect("binary"); + let identity = + identify(Some("box"), Some("c0ffee"), Some(&binary), Some(&model)).expect("identity"); + assert_eq!(identity.host.as_deref(), Some("box")); + assert_eq!(identity.commit.as_deref(), Some("c0ffee")); + assert_eq!(identity.model.path.as_deref(), Some("model.gguf")); + assert_eq!( + identity.model.sha256.as_deref(), + Some("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + ); + assert_eq!( + identity.binary_sha256.as_deref(), + Some("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + ); + let missing = dir.path().join("nope.gguf"); + assert!(matches!( + identify(Some("box"), None, None, Some(&missing)), + Err(CliError::FileNotFound(p)) if p == missing + )); + let bare = identify(Some("box"), Some(""), None, None).expect("identity"); + assert_eq!( + bare.model, + ProbeModel { + path: None, + sha256: None + } + ); + assert_eq!(given_or_env(Some(""), "PERF041_NO_SUCH_KEY"), None); + } + + #[test] + fn the_report_names_the_verdict_the_reference_and_every_band() { + let w = witness_of(ParityRun { + reference: reference(true, None), + bands: vec![ + band(1, BatchInvariance::Pass, None), + band(4, BatchInvariance::Unmeasurable, Some("no batch formed")), + ], + result: BatchInvariance::Unmeasurable, + }); + let text = report(&w); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 4, "{text}"); + assert!(lines[0].contains("UNMEASURABLE"), "{text}"); + assert!( + lines[1].contains("m=1 reference: stable tokens=128"), + "{text}" + ); + assert!(lines[2].starts_with(" c=1 m_formed=1 PASS"), "{text}"); + assert!(lines[3].contains("c=4 m_formed=4 UNMEASURABLE"), "{text}"); + assert!( + lines[3].ends_with("top2_margin=- (no batch formed)"), + "{text}" + ); + } + + #[test] + fn the_witness_is_written_as_json_that_reads_back_whole() { + let w = witness_of(ParityRun { + reference: reference(true, None), + bands: vec![band(4, BatchInvariance::Pass, None)], + result: BatchInvariance::Pass, + }); + assert_eq!(w.witness_version, WITNESS_VERSION); + assert_eq!(w.probe, PROBE); + assert_eq!(w.prompt_sha256, prompt_sha256("p")); + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("witness.json"); + write(&path, &w).expect("written"); + let back: ProbeWitness = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("parses"); + assert_eq!(back, w); + } + + fn parse(argv: &[&str]) -> std::result::Result { + let matches = LlmSubcommand::augment_subcommands(clap::Command::new("llm")) + .try_get_matches_from(std::iter::once("llm").chain(argv.iter().copied()))?; + LlmSubcommand::from_arg_matches(&matches) + } + + #[test] + fn the_flags_parse_with_the_scripts_defaults() { + let parsed = parse(&[ + "parity", + "--server-log", + "/tmp/server.log", + "--ladder", + "1,4", + ]) + .expect("parses"); + let LlmSubcommand::Parity { + url, + served_model, + prompt, + ladder, + json, + host, + .. + } = parsed + else { + panic!("not `apr test llm parity`"); + }; + assert_eq!(url.as_str(), "http://127.0.0.1:8080"); + assert_eq!(served_model.as_str(), "q"); + assert_eq!(prompt.as_str(), "Write an essay on compilers."); + assert_eq!(ladder, vec![1, 4]); + assert!(json.is_none()); + assert!(host.is_none()); + assert!(parse(&["parity", "--server-log", "x", "--ladder", "0"]).is_err()); + assert!(parse(&["parity"]).is_err()); + } +} diff --git a/crates/apr-cli/src/extended_commands.rs b/crates/apr-cli/src/extended_commands.rs index 8504a0dc6d..976e785ed3 100644 --- a/crates/apr-cli/src/extended_commands.rs +++ b/crates/apr-cli/src/extended_commands.rs @@ -2038,6 +2038,57 @@ pub enum LlmSubcommand { /// The perf041 `witness.json` to check. witness: InputFile, }, + + /// #4971 V3: run the perf041 batched-parity probe against a server. + /// + /// The Rust port of `scripts/perf041_batched_parity_probe.py`. It decodes + /// one prompt twice alone (m=1), then as c identical concurrent requests + /// for each c of the ladder, reads the batch each band formed from the + /// server's log, and writes the witness `apr test llm shape-check` reads, + /// with the reference's top-2 margin at every divergence. The thresholds, + /// sampler and ladder come from `scripts/perf-matrix.yaml`, compiled in. + /// + /// Exits 0 on PASS, 5 on FAIL (a batch's slots parted before + /// `declared_min`, or a slot froze), and 8 on UNMEASURABLE; the witness + /// is written first either way. A server log, binary or model that is not + /// a file exits 3 before any request. Not a gate: no release job runs it. + Parity { + /// The server's base URL; requests go to `/v1/chat/completions`. + #[arg(long, default_value = "http://127.0.0.1:8080")] + url: FreeText, + /// Model name sent in the request body, as the script sends it. + #[arg(long, default_value = "q")] + served_model: FreeText, + /// The server's log, where the scheduler writes `Batch m=N done`. + /// Read from where it ended before each band, so a band counts only + /// the batches it formed. + #[arg(long)] + server_log: InputFile, + /// The prompt every request carries. Recorded by its sha256. + #[arg(long, default_value = "Write an essay on compilers.")] + prompt: FreeText, + /// Concurrencies to fire, comma-separated, replacing the matrix's + /// `ladder.declared`. The V3 shape check admits only {1, 4, 8, 16}. + #[arg(long, value_delimiter = ',', value_parser = clap::value_parser!(u32).range(1..))] + ladder: Vec, + /// Write the witness JSON here. + #[arg(long)] + json: Option, + /// Host recorded in the witness. Defaults to `PERF041_HOST`, then + /// this host's name. + #[arg(long)] + host: Option, + /// Commit the server was built from. Defaults to `PERF041_COMMIT`. + #[arg(long)] + commit: Option, + /// The serving binary; its sha256 is recorded. + #[arg(long)] + binary: Option, + /// The served model file; its file name and sha256 are recorded. The + /// probe hashes it and never loads it. + #[arg(long)] + model: Option, + }, } /// Parse `apr cbtop --iterations`, rejecting 0. diff --git a/crates/apr-cli/src/validate.rs b/crates/apr-cli/src/validate.rs index 8da2e9a82b..eca6c7a9e1 100644 --- a/crates/apr-cli/src/validate.rs +++ b/crates/apr-cli/src/validate.rs @@ -15,10 +15,15 @@ fn extract_extended_model_paths(command: &ExtendedCommands) -> Vec { }, // `llm bench` measures an HTTP endpoint, not a local model file, // and `llm shape-check` reads a perf041 witness JSON, not a model. + // `llm parity` drives an endpoint too: its `--model` is hashed + // for the witness and never loaded, so the gate has nothing to + // refuse. // Listed explicitly so a future subcommand that DOES name a model // has to be handled here rather than absorbed by a wildcard. TestSubcommand::Llm { command } => match command { - LlmSubcommand::Bench { .. } | LlmSubcommand::ShapeCheck { .. } => vec![], + LlmSubcommand::Bench { .. } + | LlmSubcommand::ShapeCheck { .. } + | LlmSubcommand::Parity { .. } => vec![], }, }, ExtendedCommands::CompareHf { file, .. } diff --git a/crates/aprender-test-lib/src/perf_gate/mod.rs b/crates/aprender-test-lib/src/perf_gate/mod.rs index e28d0bacaa..f7e4c9cbb7 100644 --- a/crates/aprender-test-lib/src/perf_gate/mod.rs +++ b/crates/aprender-test-lib/src/perf_gate/mod.rs @@ -195,8 +195,9 @@ pub use join::{BandRatios, JoinKey, Ratio, RatioBound, RatioMethod}; pub use margin::{token_ids, top2_margin, TokenLogprob, TopLogprob, LOGPROBS_SOURCE}; pub use metrics::{agg_tok_s, aggregate_terms, BandMetrics, RequestSample}; pub use parity::{ - batch_formed_since, check_reference, evaluate_band, run_verdict, ProbeBand, ProbeModel, - ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, ProbeSlot, ProbeWitness, SampleResult, + batch_formed_since, check_reference, evaluate_band, run_verdict, ProbeBand, ProbeMatrix, + ProbeModel, ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, ProbeSlot, ProbeWitness, + SampleResult, }; pub use protocol::{ min_sampled_requests, warmup_requests, BandConfig, ClientModel, ProtocolParams, ProtocolSource, diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs index 8857a9fc4a..376a588343 100644 --- a/crates/aprender-test-lib/src/perf_gate/parity.rs +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -29,6 +29,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use super::margin::{token_ids, TokenLogprob}; +use super::protocol::{matrix_block, ProtocolParams, PERF_MATRIX_SOURCE}; use super::witness::{BatchInvariance, BatchInvarianceWitness}; /// The witness layout this module writes. The script's is 2; 3 adds the margin. @@ -402,6 +403,105 @@ pub struct ProbeSampler { pub top_logprobs: u8, } +/// The best tokens asked for per step: two, so a band can record the top-2 margin. +pub const TOP_LOGPROBS: u8 = 2; + +/// What configures a probe run, all of it from `scripts/perf-matrix.yaml` +/// (PP-33): the policy a band is judged by, the sampler every request carries, +/// and the declared ladder. +#[derive(Debug, Clone, PartialEq)] +pub struct ProbeMatrix { + /// `witness.min_agree_tokens`, `protocol.n_predict`, `witness.max_constant_run`. + pub policy: ProbePolicy, + /// `protocol.sampler`, with `max_tokens = protocol.n_predict` and [`TOP_LOGPROBS`]. + pub sampler: ProbeSampler, + /// `ladder.declared`: the concurrencies fired, in order. + pub ladder: Vec, +} + +impl ProbeMatrix { + /// Read from the compiled-in matrix. + /// + /// # Errors + /// As [`Self::from_matrix_source`]. + pub fn from_matrix() -> Result { + Self::from_matrix_source(PERF_MATRIX_SOURCE) + } + + /// Read from an explicit document. The script falls back to its own + /// literals when a key is absent and notes it; this refuses instead and + /// names the key, because a probe judged by numbers the matrix does not + /// hold is the drift PP-33 exists to prevent. + /// + /// # Errors + /// When a block or a key is missing; when the ladder is empty or holds a + /// 0; when `max_constant_run` is 0; or when `min_agree_tokens` exceeds + /// `n_predict`, so no run could pass. + pub fn from_matrix_source(source: &str) -> Result { + let protocol = ProtocolParams::from_matrix_source(source)?; + let witness: MatrixProbeWitness = + serde_yaml_ng::from_value(matrix_block(source, "witness")?) + .map_err(|e| format!("perf-matrix.yaml `witness:` block: {e}"))?; + let ladder: MatrixLadder = serde_yaml_ng::from_value(matrix_block(source, "ladder")?) + .map_err(|e| format!("perf-matrix.yaml `ladder:` block: {e}"))?; + let policy = ProbePolicy { + declared_min: witness.min_agree_tokens, + n_predict: protocol.n_predict, + max_constant_run: witness.max_constant_run, + }; + check_matrix(&policy, &ladder.declared)?; + Ok(Self { + policy, + sampler: ProbeSampler { + temperature: protocol.sampler.temperature, + seed: protocol.sampler.seed, + ignore_eos: protocol.sampler.ignore_eos, + max_tokens: protocol.n_predict, + top_logprobs: TOP_LOGPROBS, + }, + ladder: ladder.declared, + }) + } +} + +/// Refuse a matrix under which no run could be judged. +fn check_matrix(policy: &ProbePolicy, ladder: &[u32]) -> Result<(), String> { + if ladder.is_empty() || ladder.contains(&0) { + return Err(format!( + "perf-matrix.yaml `ladder.declared` must list concurrencies >= 1, got {ladder:?}" + )); + } + if policy.max_constant_run == 0 { + return Err( + "perf-matrix.yaml `witness.max_constant_run` is 0: every slot would read as frozen" + .to_string(), + ); + } + if policy.declared_min > policy.n_predict { + return Err(format!( + "perf-matrix.yaml `witness.min_agree_tokens` ({}) exceeds `protocol.n_predict` ({}): no \ + slot could agree for that long", + policy.declared_min, policy.n_predict + )); + } + Ok(()) +} + +/// The two `witness:` keys the probe reads. The rest of the block is +/// governance and is ignored, as in every `Matrix*` reader in +/// [`super::protocol`]. +#[derive(Debug, Deserialize)] +struct MatrixProbeWitness { + min_agree_tokens: u32, + max_constant_run: u32, +} + +/// The `ladder:` key the probe reads. +#[derive(Debug, Deserialize)] +struct MatrixLadder { + declared: Vec, +} + /// The served model: the file name, never a host path, and its bytes' sha256. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ProbeModel { @@ -806,4 +906,90 @@ mod tests { "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ); } + + /// A matrix with every key the probe reads, plus a governance key it ignores. + fn matrix(witness: &str, ladder: &str) -> String { + format!( + "protocol:\n window_ms: 1000\n warmup_requests_per_worker: 1\n quiesce_ms: 0\n \ + cooldown_ms: 0\n n_predict: 32\n replicates_min: 1\n interleaved: true\n \ + sampler: {{temperature: 0.0, seed: 9, ignore_eos: true}}\nwitness:\n{witness} \ + author: spec-owner\nladder:\n{ladder}" + ) + } + + const WITNESS_OK: &str = " min_agree_tokens: 24\n max_constant_run: 6\n"; + + #[test] + fn the_matrix_configures_the_policy_the_sampler_and_the_ladder() { + let m = ProbeMatrix::from_matrix_source(&matrix(WITNESS_OK, " declared: [1, 2, 3]\n")) + .expect("a complete matrix reads"); + assert_eq!( + m.policy, + ProbePolicy { + declared_min: 24, + n_predict: 32, + max_constant_run: 6 + } + ); + assert_eq!( + m.sampler, + ProbeSampler { + temperature: 0.0, + seed: 9, + ignore_eos: true, + max_tokens: 32, + top_logprobs: 2 + } + ); + assert_eq!(m.ladder, vec![1, 2, 3]); + } + + #[test] + fn the_shipped_matrix_configures_the_probe_as_the_other_readers_read_it() { + let m = ProbeMatrix::from_matrix().expect("the shipped matrix configures the probe"); + let protocol = ProtocolParams::from_matrix().expect("the shipped protocol block"); + let declared_min = + super::super::protocol::witness_min_agree_tokens_from(PERF_MATRIX_SOURCE) + .expect("the shipped witness block"); + assert_eq!(m.policy.declared_min, declared_min); + assert_eq!(m.policy.n_predict, protocol.n_predict); + assert_eq!(m.sampler.max_tokens, protocol.n_predict); + assert_eq!(m.sampler.seed, protocol.sampler.seed); + assert!( + m.ladder.iter().any(|&c| c > 1), + "a ladder with no c > 1 could never PASS: {:?}", + m.ladder + ); + } + + #[test] + fn a_matrix_that_cannot_judge_a_run_is_refused_by_name() { + for (witness, ladder, named) in [ + ( + " min_agree_tokens: 24\n", + " declared: [1, 2]\n", + "max_constant_run", + ), + (WITNESS_OK, " derive_from: []\n", "declared"), + (WITNESS_OK, " declared: []\n", "ladder.declared"), + (WITNESS_OK, " declared: [1, 0]\n", "ladder.declared"), + ( + " min_agree_tokens: 24\n max_constant_run: 0\n", + " declared: [1, 2]\n", + "max_constant_run", + ), + ( + " min_agree_tokens: 33\n max_constant_run: 6\n", + " declared: [1, 2]\n", + "min_agree_tokens", + ), + ] { + let err = ProbeMatrix::from_matrix_source(&matrix(witness, ladder)).expect_err(named); + assert!(err.contains(named), "{named}: {err}"); + } + let no_ladder = + matrix(WITNESS_OK, " declared: [1]\n").replace("ladder:\n declared: [1]\n", ""); + let err = ProbeMatrix::from_matrix_source(&no_ladder).expect_err("no ladder block"); + assert!(err.contains("`ladder:`"), "{err}"); + } } diff --git a/crates/aprender-test-lib/src/perf_gate/protocol.rs b/crates/aprender-test-lib/src/perf_gate/protocol.rs index 0e0202d0cf..71ea1952cb 100644 --- a/crates/aprender-test-lib/src/perf_gate/protocol.rs +++ b/crates/aprender-test-lib/src/perf_gate/protocol.rs @@ -323,7 +323,7 @@ pub fn witness_min_agree_tokens() -> u32 { } /// Pull one top-level block out of the matrix, naming what is missing. -fn matrix_block(source: &str, key: &str) -> Result { +pub(super) fn matrix_block(source: &str, key: &str) -> Result { let doc: serde_yaml_ng::Value = serde_yaml_ng::from_str(source) .map_err(|e| format!("perf-matrix.yaml does not parse as YAML: {e}"))?; doc.get(key).cloned().ok_or_else(|| { From a9a6409b1cace6701abac820519b0868fbd286a5 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 09:35:18 +0200 Subject: [PATCH 20/35] fix(apr-cli): `apr test llm parity` writes its witness to --witness, not --json (PMAT-4971) The parity arg `json: Option` shared its id with apr's global `--json: bool`, so clap downcast one to the other's type and panicked on every real run: gx10 g12 at 61f0fa5798 panicked at extended_commands.rs:2042 (missing log) and lib.rs:160 (refused server) before any request. The unit tests parsed `llm` alone, without the global arg, and passed. The flag is now `--witness `. A new test parses through the whole `apr` parser with the witness flag, with the global --json before the subcommand, and with it after, and checks that each lands where it belongs. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 605a9f627734ff5c3cbbf41e4acfb88e941442c9) (cherry picked from commit 530853872df788946c73c44eb4a5ca4aad1dd290) (cherry picked from commit 83c76daae81b3ddb4455599a78deb7f684ec3b72) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- .../apr-cli/src/commands/test_llm_parity.rs | 50 +++++++++++++++++-- crates/apr-cli/src/extended_commands.rs | 8 +-- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 1c2d3087f4..416d8ac3e9 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -720,7 +720,7 @@ falsification_tests: if_fails: 'The V3 witness can only come from the Python script (C301), or the Rust probe reads a box that never batched, or a reference that disagrees with itself, as batch-invariant.' - id: FALSIFY-PERF041-CLI-4971 name: apr_test_llm_parity_exits_by_the_verdict_and_writes_the_witness - prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --json before exiting, and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' + prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --witness before exiting (not --json, which is apr's global flag: a parity arg with that id panicked on every run, and a parse through the whole apr parser holds the two apart), and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' test_harness: 'cargo test -p apr-cli --lib test_llm_parity' expected_output: "test result: ok" if_fails: 'The Rust probe has no front door, so the V3 witness is still produced by the Python script by hand; or a FAIL and an UNMEASURABLE leave the same exit code; or a witness records a host path or a made-up host.' diff --git a/crates/apr-cli/src/commands/test_llm_parity.rs b/crates/apr-cli/src/commands/test_llm_parity.rs index a1e1dbb0ed..7b3c015aec 100644 --- a/crates/apr-cli/src/commands/test_llm_parity.rs +++ b/crates/apr-cli/src/commands/test_llm_parity.rs @@ -58,7 +58,7 @@ pub(crate) fn run(command: &LlmSubcommand) -> Result<()> { server_log, prompt, ladder, - json, + witness: witness_path, host, commit, binary, @@ -81,7 +81,7 @@ pub(crate) fn run(command: &LlmSubcommand) -> Result<()> { .map_err(|e| CliError::InferenceFailed(format!("tokio runtime: {e}")))?; let witness = witness(&probe, identity, rt.block_on(probe.run(&client))); println!("{}", report(&witness)); - if let Some(path) = json { + if let Some(path) = witness_path { write(path, &witness)?; } outcome(&witness) @@ -504,7 +504,7 @@ mod tests { served_model, prompt, ladder, - json, + witness, host, .. } = parsed @@ -515,9 +515,51 @@ mod tests { assert_eq!(served_model.as_str(), "q"); assert_eq!(prompt.as_str(), "Write an essay on compilers."); assert_eq!(ladder, vec![1, 4]); - assert!(json.is_none()); + assert!(witness.is_none()); assert!(host.is_none()); assert!(parse(&["parity", "--server-log", "x", "--ladder", "0"]).is_err()); assert!(parse(&["parity"]).is_err()); } + + /// Through the whole `apr` parser, not `llm` alone: that is where the + /// global `--json` lives, and a parity arg sharing its id parsed fine + /// under [`parse`] yet panicked on every real run (gx10 g12, 61f0fa5798). + #[test] + fn the_witness_flag_parses_beside_the_global_json() { + use clap::Parser; + let dir = tempfile::tempdir().expect("tempdir"); + let out = dir.path().join("w.json"); + let out = out.to_str().expect("utf-8 path"); + let parity = ["test", "llm", "parity", "--server-log", "s.log"]; + let cases: [(&[&str], &[&str], bool); 3] = [ + (&[], &["--witness", out], false), + (&["--json"], &["--witness", out], true), + (&[], &["--json"], true), + ]; + for (before, after, global_json) in cases { + let argv: Vec<&str> = std::iter::once("apr") + .chain(before.iter().copied()) + .chain(parity) + .chain(after.iter().copied()) + .collect(); + let cli = crate::Cli::try_parse_from(argv.iter().copied()) + .unwrap_or_else(|e| panic!("{argv:?} parses: {e}")); + assert_eq!(cli.json, global_json, "{argv:?}: the global --json"); + let crate::Commands::Extended(crate::ExtendedCommands::Test { + command: + crate::TestSubcommand::Llm { + command: LlmSubcommand::Parity { witness, .. }, + }, + }) = *cli.command + else { + panic!("{argv:?}: not `apr test llm parity`"); + }; + assert_eq!( + witness.as_deref().map(Path::to_path_buf), + argv.contains(&"--witness") + .then(|| Path::new(out).to_path_buf()), + "{argv:?}: --witness" + ); + } + } } diff --git a/crates/apr-cli/src/extended_commands.rs b/crates/apr-cli/src/extended_commands.rs index 976e785ed3..d1ff43ca0f 100644 --- a/crates/apr-cli/src/extended_commands.rs +++ b/crates/apr-cli/src/extended_commands.rs @@ -2071,9 +2071,11 @@ pub enum LlmSubcommand { /// `ladder.declared`. The V3 shape check admits only {1, 4, 8, 16}. #[arg(long, value_delimiter = ',', value_parser = clap::value_parser!(u32).range(1..))] ladder: Vec, - /// Write the witness JSON here. - #[arg(long)] - json: Option, + /// Write the witness JSON here. Not `--json`: that is `apr`'s global + /// flag, and a second arg with its id made clap downcast one to the + /// other's type and panic on every `apr test llm parity` run. + #[arg(long, value_name = "FILE")] + witness: Option, /// Host recorded in the witness. Defaults to `PERF041_HOST`, then /// this host's name. #[arg(long)] From 134ed00bc11c87cbdab15c1d5e1949212b965326 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 09:43:26 +0200 Subject: [PATCH 21/35] fix(apr-cli): parse the whole `apr` Cli on a 16 MB stack; quote apr's in the PERF041 row (PMAT-4971) The full-parser test added in e1f24a0645 overflowed the default 2 MiB test-thread stack in a debug build on gx10 (SIGABRT), as clap's destructuring of the full `Commands` enum does elsewhere; it now runs on a 16 MB thread, as run_tests_top_k_default_3754.rs does. The same commit's FALSIFY-PERF041-CLI-4971 wording put `apr's` inside a single-quoted YAML scalar, so `pv lint contracts/` failed PV-VAL-001 on apr-serve-openai-compat-v1.yaml; the quote is now doubled. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9bf1cebdb76fb3283c22d5d24505eaf9e84fc42e) (cherry picked from commit 9f3bb004e9641aa9b3420c687a7002eb35ee84d8) (cherry picked from commit 1b9b22387b466dc9ce06cf9808cef1fa723e2309) --- contracts/apr-serve-openai-compat-v1.yaml | 2 +- crates/apr-cli/src/commands/test_llm_parity.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index 416d8ac3e9..dc25abcaba 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -720,7 +720,7 @@ falsification_tests: if_fails: 'The V3 witness can only come from the Python script (C301), or the Rust probe reads a box that never batched, or a reference that disagrees with itself, as batch-invariant.' - id: FALSIFY-PERF041-CLI-4971 name: apr_test_llm_parity_exits_by_the_verdict_and_writes_the_witness - prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --witness before exiting (not --json, which is apr's global flag: a parity arg with that id panicked on every run, and a parse through the whole apr parser holds the two apart), and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' + prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --witness before exiting (not --json, which is apr''s global flag: a parity arg with that id panicked on every run, and a parse through the whole apr parser holds the two apart), and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' test_harness: 'cargo test -p apr-cli --lib test_llm_parity' expected_output: "test result: ok" if_fails: 'The Rust probe has no front door, so the V3 witness is still produced by the Python script by hand; or a FAIL and an UNMEASURABLE leave the same exit code; or a witness records a host path or a made-up host.' diff --git a/crates/apr-cli/src/commands/test_llm_parity.rs b/crates/apr-cli/src/commands/test_llm_parity.rs index 7b3c015aec..37987a0553 100644 --- a/crates/apr-cli/src/commands/test_llm_parity.rs +++ b/crates/apr-cli/src/commands/test_llm_parity.rs @@ -524,8 +524,20 @@ mod tests { /// Through the whole `apr` parser, not `llm` alone: that is where the /// global `--json` lives, and a parity arg sharing its id parsed fine /// under [`parse`] yet panicked on every real run (gx10 g12, 61f0fa5798). + /// On a 16 MB stack, as in run_tests_top_k_default_3754.rs: clap's + /// destructuring of the full `Commands` enum overflows the default 2 MiB + /// test-thread stack in debug builds (gx10 g13, e1f24a0645). #[test] fn the_witness_flag_parses_beside_the_global_json() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(witness_and_global_json_cases) + .expect("spawn parse thread") + .join() + .expect("every case parses as its row says"); + } + + fn witness_and_global_json_cases() { use clap::Parser; let dir = tempfile::tempdir().expect("tempdir"); let out = dir.path().join("w.json"); From 0cc45a1acd107c6783816c7debc214c762c0b05d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 09:43:56 +0200 Subject: [PATCH 22/35] feat(contracts): serving-shape-parity-v1, the V3 bar and its admission rule (PMAT-4971) Order step 1 of the V3 bar the C314 quorum adopted on 2026-10-10 06:48Z (receipt quorum-la-0.71.1a-v3bar.md): the draft 0.3.0 moves into contracts/ at 1.0.0 with the bar's statement, falsifier, host classes (lambda RTX 4090 sm_89, gx10 GB10), conditions and order. FALSIFY-SSP-001..016 are re-pointed from the Python reference to the Rust case table in perf_gate::v3_shape. SSP-007 states what main's lambda witness gives today: four reasons, since #4888 refreshed it and c=16 now forms m=16. S7 (FALSIFY-CB-004 read from the witness), the one-near-tie cap and the declared_min floor of 64 are spec-before-code: SSP-017..019 are PLANNED until shape-check reads them. The cap's rationale is the author's wording. Not a gate: no merge, queue or release job runs the check. pv validate and pv lint contracts/ pass; README's CONTRACT_COUNT is regenerated. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e8e238ae8619df8854d068ce2ca5c5546022a694) (cherry picked from commit 79988b491bde552866c7358fb5e6ca608e4cf032) (cherry picked from commit 5fae9e470fdc4ec531681f5e414f5c5c79887fcc) --- README.md | 4 +- contracts/serving-shape-parity-v1.yaml | 267 +++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 contracts/serving-shape-parity-v1.yaml diff --git a/README.md b/README.md index 237de95dae..238df73c47 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ publishing — all backed by YAML provable contracts that fail CI on drift. | Metric | Count | Source of truth | |-------:|------:|---| | Workspace crates | **81** workspace crates | `cargo metadata --no-deps` (NOT `ls crates/` — 4 are `exclude`d, 1 has no Cargo.toml) | -| Provable contracts | **1903** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | +| Provable contracts | **1904** provable contracts | `contracts/census.json` `.n_files` — the set `pv lint` walks (`pv census`, ONT-001 ONT-1; regenerated by `make contracts`, written by `make readme-sync`, guarded by `scripts/check_readme_claims.sh`) | | CLI commands | **113** CLI commands | `contracts/apr-cli-commands-v1.yaml` §`commands` (parse the list; `apr --help` also lists `help` itself, and `grep -c '^ - name:'` gives 119 — other same-indent `name:` keys exist in the file) | | Book CLI chapters | **114** chapters | `ls book/src/cli/*.md` | | Book lib chapters | **73** chapters | `ls book/src/lib/*.md` (parity with `pub mod`) | @@ -403,7 +403,7 @@ falsification_tests: prediction: apr validate bad-model.apr exits non-zero ``` -The tree carries 1903 contracts across inference, training, quantization, attention, FFN, +The tree carries 1904 contracts across inference, training, quantization, attention, FFN, tokenization, model formats, CLI safety — and this README itself. ## Migration from old crates diff --git a/contracts/serving-shape-parity-v1.yaml b/contracts/serving-shape-parity-v1.yaml new file mode 100644 index 0000000000..1c3ef2a0fe --- /dev/null +++ b/contracts/serving-shape-parity-v1.yaml @@ -0,0 +1,267 @@ +# 0.71.1 exit criterion V3 (#4971): is a perf041 witness a serving-shape parity receipt? +# Moved here from the spec branch's draft (la-71/v3-serving-shape-parity @ adf6c4b4aa, +# docs/lookahead/contracts-draft/serving-shape-parity-v1.yaml, version 0.3.0). The draft's +# Python checker and fixtures are not carried: the check is perf_gate::v3_shape and its +# in-code case table, and `apr test llm shape-check` is its front door (C301). +# The bar below was adopted by the C314 quorum of 2026-10-10 06:48Z, recorded here before +# any of its code (order step 1). S7, the near-tie cap and the declared_min floor are +# spec-before-code: their rows are PLANNED until shape-check reads them (condition 1). +# This is not a gate: no merge, queue or release job runs the check. Wiring V3 into a +# release gate is a gate change with its own ruling. +metadata: + version: 1.0.0 + created: '2026-10-10' + author: la-0.71.1a (from the draft 0.3.0 by la-0.71, 2026-10-03) + kind: schema + status: partial + description: > + A perf041 witness is a V3 serving-shape parity receipt iff it is on the blessed + Qwen3.5-4B, measures bands c in {1,4,8,16} each formed at full width, carries + identity, agrees with the m=1 reference to declared_min unless a near-tie (top-2 + margin below epsilon) explains the divergence, and shows per-request decode at c=4 + at least half of c=1 (S7). V3 holds when one such witness from apr serve with CUDA + is admitted on each GPU host class 0.71.1 ships for. + references: + - crates/aprender-test-lib/src/perf_gate/v3_shape.rs + - crates/apr-cli/src/commands/test_llm_shape.rs + - crates/apr-cli/src/commands/test_llm_parity.rs + - crates/aprender-test-lib/src/llm/parity.rs + - 'contracts/apr-serve-openai-compat-v1.yaml FALSIFY-V3-SHAPE-CHECK-4971, FALSIFY-V3-SHAPE-CLI-4971, FALSIFY-PERF041-CLI-4971' + - 'Quorum receipt: /mnt/nvme-raid0/cop-inbox/handoff/quorum-la-0.71.1a-v3bar.md (C314, round 2, ADOPTED 2026-10-10 06:48Z)' + - 'Spec: docs/lookahead/0.71-v3-serving-shape-parity.md on branch la-71/v3-serving-shape-parity' + depends_on: + - continuous-batching-v1 +bar: + adopted: '2026-10-10T06:48Z, C314 quorum round 2 (Explore sonnet, haiku, sonnet adversarial; degraded: same-family), bound to la-711a/4971-v3a-margin @ 61f0fa5798' + statement: > + V3 holds when, on each GPU host class 0.71.1 ships for (lambda RTX 4090 sm_89 and gx10 + GB10), one `apr test llm parity` run against `apr serve` with CUDA on the blessed + Qwen3.5-4B at c = 1, 4, 8 and 16 writes a witness that `apr test llm shape-check` admits + with exit 0 under S1-S7. In that check S1-S6 are as drafted, with eps = 0.05 logit + fixed; at most one of the three batched bands may be admitted through the near-tie + exemption; and S7 (CB-004: per-request decode at c=4 at least 50% of c=1) is read from + per-request decode timing that the same witness records. + falsifier: > + From an admitted witness, set c=4 per-request decode to 49% of c=1: shape-check must + exit 5 and name CB-004 (FALSIFY-SSP-017). The cap row joins the case table: c=4 and + c=8 both near-tie exits 5, naming the cap and both bands (FALSIFY-SSP-018). + host_classes: + - 'lambda: RTX 4090, sm_89' + - 'gx10: GB10' + conditions: + - 'Until shape-check reads S7 and the cap, both are spec-before-code (rows PLANNED), and the bar cannot be met.' + - 'declared_min has a floor: every band declares at least the perf matrix''s witness.min_agree_tokens (64), so a self-declared declared_min of 1 is not admissible (FALSIFY-SSP-019).' + - 'The closer''s receipt names the host and its GPU from the witness env block, one per host class above.' + - 'The end-to-end plant (a deliberately wrong KV slot in the batched CUDA path must make parity + shape-check give NOT ADMISSIBLE) is a PROPOSE-TICKET with one ticket and one owner, logged 2026-10-10 06:48Z; its state is listed when the row closes.' + - 'Witnesses are self-reported and unsigned. The end-to-end plant is the guard against a server that lies; no new rule.' + - 'The rationale for the cap is recorded under equations.admissible.cap_rationale.' + spec_before_code: [S7, near_tie_cap, declared_min_floor] + order: + - 'Contract: this file, passing pv validate (la-0.71.1a).' + - 'Code: the probe records per-request decode timing; shape-check reads S7, the cap and the declared_min floor (la-0.71.1a).' + - 'PROPOSE-TICKET for the end-to-end plant.' + - 'A session other than la-0.71.1a runs both host classes and closes the row. The author never closes V3.' +equations: + admissible: + formula: | + admissible(w) := blessed(w.model) ∧ identity(w) complete + ∧ ∀c ∈ {1,4,8,16}: exactly one band(c) ∈ w + ∧ band(c).result = PASS ∧ band(c).m_formed = c + ∧ band(c).declared_min ≥ 64 + ∧ (band(c).divergence_at ≥ band(c).declared_min ∨ 0 ≤ band(c).top2_margin_at_divergence < ε) + ∧ |{c ∈ {4,8,16} : band(c) admitted only by the near-tie clause}| ≤ 1 + ∧ decode_tok_s(band(4)) ≥ 0.5 · decode_tok_s(band(1)) + ε := 0.05 logit (fixed by the bar; no longer [A]) + domain: 'w is the JSON witness written by `apr test llm parity` (perf041, PP-26 v3.1); decode_tok_s is per-request decode tokens per second, the median over the band''s requests' + invariants: + - 'a band formed at m < c is never admissible as shape c' + - 'a divergence from m=1 without a recorded margin below ε is never explained' + - 'an empty, blank, unreadable or bandless witness is RED, never NOT ADMISSIBLE' + - 'a witness with no per-request decode timing at c=1 or c=4 is not admissible: S7 is unmeasured, never assumed' + cap_rationale: > + Author's wording of quorum condition 6. A near-tie explains one early divergence as + chance: two candidates within ε of each other at that step, so any reduction-order + change may flip the argmax. Bands c=4, 8 and 16 share one batched path, so when more + than one of them diverges early the likelier cause is that path, not three + independent coin flips at near-ties. Letting every batched band in on a recorded + margin would let a systematic batched-path defect pass by recording margins. One of + three keeps the chance explanation and refuses it as the shape of the whole run. +falsification_tests: + - id: FALSIFY-SSP-001 + rule: 'control: full bands on the blessed model are admissible' + prediction: 'Case row `admissible` (the in-code control witness) is ADMISSIBLE: library exit 0, `apr test llm shape-check` exit 0, no reason.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A rule fires on a clean witness: a band, identity or model check misreads a field it should accept.' + - id: FALSIFY-SSP-002 + rule: 'a divergence at 3 with a recorded margin of 0.01 is explained' + prediction: 'Case row `near-tie`: a c=4 band that diverges from m=1 at 3 with top-2 margin 0.01 is ADMISSIBLE with no reason.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The near-tie clause ignores the recorded margin, or compares it against another ε.' + - id: FALSIFY-SSP-003 + rule: 'c=16 with m_formed=14 is not admissible' + prediction: 'Case row `short-band` is NOT ADMISSIBLE (library 1, CLI 5) with the single reason `c=16: m_formed 14 != c (shape not served)`.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The m_formed == c check is missing or reads the wrong field.' + - id: FALSIFY-SSP-004 + rule: 'a model other than the blessed Qwen3.5-4B is not admissible' + prediction: 'Case rows `wrong-model` and `prefix, not substring` are NOT ADMISSIBLE with the single reason that the model is not the blessed Qwen3.5-4B. The file name must start with qwen3.5-4b, compared without case; containing it is not enough.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The blessed-model check reads another field, or matches the prefix anywhere in the name (the draft''s surviving mutant startswith-to-in).' + - id: FALSIFY-SSP-005 + rule: 'a divergence with no recorded margin is not admissible' + prediction: 'Case row `unexplained-divergence` is NOT ADMISSIBLE with the single reason `c=4: diverges from m=1 at 3 < 64 with no near-tie margin recorded`.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A divergence with no recorded margin is treated as explained.' + - id: FALSIFY-SSP-006 + rule: 'an empty, malformed or missing witness is RED (findings F1)' + prediction: 'An empty string, malformed JSON, a non-object and a missing file are RED (library 2, CLI 4; a missing path or a directory is CLI 3), never NOT ADMISSIBLE.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'An unreadable witness reads as a verdict about the server.' + - id: FALSIFY-SSP-007 + rule: 'the real lambda witness on main (perf041 PASS) is not a V3 receipt' + prediction: > + `apr test llm shape-check evidence/perf041/lambda/witness.json` on main exits 5 with + four reasons: the model qwen2.5-coder-1.5b-instruct-q4_k_m.gguf is not the blessed + Qwen3.5-4B, and c=4, c=8 and c=16 each diverge from m=1 at 3 < 64 with no near-tie + margin recorded. The draft's five reasons included c=16 m_formed 14; #4888 + (8d44b58703, 2026-10-07) refreshed the witness, and c=16 now forms m=16. Once S7 + lands, a fifth reason names its missing decode timing. + test: 'by hand: apr test llm shape-check evidence/perf041/lambda/witness.json' + if_fails: 'A perf041 PASS is read as a V3 receipt: a V3 rule is missing, or main''s witness changed again.' + - id: FALSIFY-SSP-008 + rule: 'identity complete: a witness missing any one of binary_sha256, commit, host, prompt_sha256 is not admissible (one case per member)' + prediction: 'Case rows `no-identity-binary-sha256`, `no-identity-commit`, `no-identity-host` and `no-identity-prompt-sha256` each give the single reason `identity field missing`; `blank identity` shows a whitespace value counts as missing.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The identity check is missing or skips a member.' + - id: FALSIFY-SSP-009 + rule: 'a blessed model with no sha256 recorded is not admissible' + prediction: 'Case row `no-model-sha256` gives the single reason `model sha256 missing`; `model missing` gives both S1 reasons.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The sha256 half of S1 is unchecked, so a file name alone admits the model.' + - id: FALSIFY-SSP-010 + rule: 'a margin equal to ε does not explain a divergence (ε is a strict upper bound)' + prediction: 'Case row `margin-at-epsilon (F4 wording)`: a margin of exactly 0.05 is not a near-tie, and the reason names the recorded margin and ε (findings F4 closed). `margin just below epsilon` and `margin zero (an exact tie)` are explained; `negative margin` is not.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'ε was raised or the comparison became <=, or a negative margin passes as a near-tie.' + - id: FALSIFY-SSP-011 + rule: 'a band whose perf041 result is not PASS is not admissible' + prediction: 'Case row `failed-band` gives the single reason `c=8: perf041 result FAIL`; `result missing` names it as not recorded.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'S6 is unchecked: a band that perf041 failed is admitted.' + - id: FALSIFY-SSP-012 + rule: 'every band c in {1,4,8,16} is measured, exactly once (one case per member; findings F6)' + prediction: 'Case rows `missing-band-c1`, `-c4`, `-c8` and `-c16` each give `band c= not measured`; `F6 duplicate c` is not admissible and every copy is checked; `a band outside V3 is ignored`.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'S2 is unchecked or skips a band, or a second copy of a band masks the first.' + - id: FALSIFY-SSP-013 + rule: 'control: a divergence exactly at declared_min is admissible' + prediction: 'Case row `divergence-at-min` is ADMISSIBLE; `divergence one below min` is not.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The divergence comparison moved off the boundary.' + - id: FALSIFY-SSP-014 + rule: 'a witness whose band list is empty is RED' + prediction: 'Case row `no-bands` is RED (library 2, CLI 4), never four `not measured` reasons.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A witness that measured nothing is evaluated instead of RED.' + - id: FALSIFY-SSP-015 + rule: 'an UNMEASURABLE band is not admissible and reports only its result (findings F5)' + prediction: 'Case row `unmeasurable-band (F5: its result only)` gives the single reason `c=8: perf041 result UNMEASURABLE`; a PASS band with no divergence_at is not admissible (`PASS band without divergence_at`).' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A null divergence_at reads as 0, or an UNMEASURABLE band is admitted.' + - id: FALSIFY-SSP-016 + rule: 'a whitespace-only witness is RED' + prediction: 'A witness holding only a newline is RED (library 2, CLI 4).' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'The blank witness is parsed and reported as malformed JSON or NOT ADMISSIBLE.' + - id: FALSIFY-SSP-017 + rule: 'S7 (FALSIFY-CB-004): per-request decode at c=4 is at least half of c=1; the bar''s falsifier' + status: PLANNED + prediction: > + From the admissible control, set c=4 per-request decode to 49% of c=1: shape-check + exits 5 with a reason naming FALSIFY-CB-004 and both rates. At exactly 50% it is + admissible. A witness with no per-request decode timing at c=1 or c=4 is not + admissible, with a reason saying S7 is unmeasured. + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'Architectural serialization or lock contention ships as a V3 PASS, or a witness with no timing passes S7 by default.' + - id: FALSIFY-SSP-018 + rule: 'at most one of the bands c=4, 8, 16 is admitted through the near-tie clause' + status: PLANNED + prediction: 'From the admissible control, make c=4 and c=8 both diverge at 3 with margin 0.01: shape-check exits 5 with a reason naming the near-tie cap and both bands. One near-tie batched band (FALSIFY-SSP-002) stays admissible.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A systematic batched-path divergence passes by recording a margin in every band.' + - id: FALSIFY-SSP-019 + rule: 'declared_min has a floor of 64 (the perf matrix''s witness.min_agree_tokens)' + status: PLANNED + prediction: 'From the admissible control, set one band''s declared_min to 1 and its divergence_at to 3: shape-check exits 5 with a reason naming the floor. declared_min 64 stays admissible.' + test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' + expected_output: "test result: ok" + if_fails: 'A witness passes S5 by declaring its own agreement bar.' +qa_gate: + id: F-SSP-QA-001 + name: A V3 receipt is admitted only when every conjunct holds + description: > + The in-code case table in perf_gate::v3_shape carries every row of the draft's + cases.json that holds a JSON object, plus rows for findings F2, F3 and F6 and for + each boundary (divergence at and one below declared_min; margin at, just below and + at zero; a negative margin). The RED rows (empty, blank, malformed, non-object, + missing file) run in their own test. Each row gives its exit code and its exact + reasons, and the test is red on a row whose reasons differ by one. The CLI test + maps the verdicts onto apr's exit codes: 0, 5 NOT ADMISSIBLE, 4 RED, 3 no such file. + checks: + - planted_case_exits_as_its_row_says + - not_admissible_case_prints_exactly_its_own_reasons + - unreadable_witness_is_red + - cli_exit_codes_follow_the_verdict + - real_witness_not_admissible + pass_criteria: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli --lib test_llm_shape both print test result: ok; FALSIFY-SSP-007 by hand gives its reasons.' + falsification: 'The draft''s 32-mutant record (mutation_record) left two survivors; both are killed by rows in the Rust table: startswith-to-in by `prefix, not substring`, declared-min-default-64 by `F3 no declared_min`.' +findings: + - id: F1 + summary: 'The Python checker exited 1 (NOT ADMISSIBLE) on an unreadable witness.' + closed_by: 'v3_shape: unreadable is RED, exit 2 (CLI 4 or 3). FALSIFY-SSP-006.' + - id: F2 + summary: 'A JSON boolean passed the numeric checks.' + closed_by: 'v3_shape: typed fields; rows `F2 m_formed true`, `F2 margin false`, `F2 c as a string`.' + - id: F3 + summary: 'A band with no declared_min passed S5 with a default of 0.' + closed_by: 'v3_shape: not admissible; row `F3 no declared_min`. The floor of 64 (FALSIFY-SSP-019) tightens it further.' + - id: F4 + summary: 'The S5 reason said no margin was recorded when one was, at or above ε.' + closed_by: 'v3_shape: the reason names the margin and ε; row `margin-at-epsilon (F4 wording)`.' + - id: F5 + summary: 'An UNMEASURABLE band also reported a divergence at None.' + closed_by: 'v3_shape: only its result; row `unmeasurable-band (F5: its result only)`.' + - id: F6 + summary: 'When two bands shared a c, the last one won.' + closed_by: 'v3_shape: a duplicate c is not admissible and every copy is checked; row `F6 duplicate c`.' + - id: S7-scope + summary: 'The draft did not check S7 and had the receipt cite CB-004 beside the witness.' + closed_by: 'Superseded by the bar: S7 is read from the same witness (FALSIFY-SSP-017, PLANNED until the code lands).' +mutation_record: + note: 'History: the draft''s record, measured on the Python checker. The Rust port has not been mutation-tested yet.' + date: '2026-10-04' + mutants: 32 + patched_self_test: {killed: 30, killed_by_crash: 4, survived: 2} + survived_table: [startswith-to-in, declared-min-default-64] +verification: > + cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli + --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. FALSIFY-SSP-007 by hand is + not_measured through the Rust port yet; its reasons above are read off main's witness + and the code. From 09af4ba7fcb44caefdc48950968c13e03e27a7ac Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 09:56:33 +0200 Subject: [PATCH 23/35] feat(perf_gate): V3 reads S7, the near-tie cap and the declared_min floor (PMAT-4971) The probe now records each request's decode rate from its own token arrivals, and only on a stream the server declared live: the batched CUDA path drops the server's timings. A band's decode_tok_s is the median of its slots' rates, and None when any slot is untimed. shape-check reads the V3 bar's three new rules (C314 round 2): - S7 (FALSIFY-CB-004): per-request decode at c=4 is at least 50% of c=1's. A missing rate at either band is unmeasured, which is not admissible. - the near-tie cap: at most one of c=4, 8, 16 is admitted through the near-tie clause. - the declared_min floor of 64. A test checks that it equals the shipped perf matrix's witness.min_agree_tokens. serving-shape-parity-v1: FALSIFY-SSP-017..019 are no longer PLANNED. SSP-007 now predicts five reasons on main's lambda witness. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a2defb82c5135e1f6ca247fc3c4ff6b37618b815) (cherry picked from commit 70c6257fb5967af05555d8c6bcf705b50eacb882) (cherry picked from commit 7ed84b686f9b7d8e03f1e32ab9dfefb07cff8323) --- contracts/serving-shape-parity-v1.yaml | 39 +-- .../apr-cli/src/commands/test_llm_parity.rs | 1 + crates/apr-cli/src/commands/test_llm_shape.rs | 37 ++- crates/aprender-test-lib/src/llm/parity.rs | 95 ++++++-- crates/aprender-test-lib/src/perf_gate/mod.rs | 9 +- .../aprender-test-lib/src/perf_gate/parity.rs | 192 ++++++++++++++- .../src/perf_gate/v3_shape.rs | 224 +++++++++++++++++- 7 files changed, 531 insertions(+), 66 deletions(-) diff --git a/contracts/serving-shape-parity-v1.yaml b/contracts/serving-shape-parity-v1.yaml index 1c3ef2a0fe..2828fe2dcb 100644 --- a/contracts/serving-shape-parity-v1.yaml +++ b/contracts/serving-shape-parity-v1.yaml @@ -4,8 +4,8 @@ # Python checker and fixtures are not carried: the check is perf_gate::v3_shape and its # in-code case table, and `apr test llm shape-check` is its front door (C301). # The bar below was adopted by the C314 quorum of 2026-10-10 06:48Z, recorded here before -# any of its code (order step 1). S7, the near-tie cap and the declared_min floor are -# spec-before-code: their rows are PLANNED until shape-check reads them (condition 1). +# any of its code (order step 1). S7, the near-tie cap and the declared_min floor were +# spec-before-code; shape-check reads them since order step 2 (FALSIFY-SSP-017..019). # This is not a gate: no merge, queue or release job runs the check. Wiring V3 into a # release gate is a gate change with its own ruling. metadata: @@ -55,7 +55,7 @@ bar: - 'The end-to-end plant (a deliberately wrong KV slot in the batched CUDA path must make parity + shape-check give NOT ADMISSIBLE) is a PROPOSE-TICKET with one ticket and one owner, logged 2026-10-10 06:48Z; its state is listed when the row closes.' - 'Witnesses are self-reported and unsigned. The end-to-end plant is the guard against a server that lies; no new rule.' - 'The rationale for the cap is recorded under equations.admissible.cap_rationale.' - spec_before_code: [S7, near_tie_cap, declared_min_floor] + spec_before_code: [] # S7, near_tie_cap and declared_min_floor landed in order step 2 order: - 'Contract: this file, passing pv validate (la-0.71.1a).' - 'Code: the probe records per-request decode timing; shape-check reads S7, the cap and the declared_min floor (la-0.71.1a).' @@ -127,11 +127,11 @@ falsification_tests: rule: 'the real lambda witness on main (perf041 PASS) is not a V3 receipt' prediction: > `apr test llm shape-check evidence/perf041/lambda/witness.json` on main exits 5 with - four reasons: the model qwen2.5-coder-1.5b-instruct-q4_k_m.gguf is not the blessed - Qwen3.5-4B, and c=4, c=8 and c=16 each diverge from m=1 at 3 < 64 with no near-tie - margin recorded. The draft's five reasons included c=16 m_formed 14; #4888 - (8d44b58703, 2026-10-07) refreshed the witness, and c=16 now forms m=16. Once S7 - lands, a fifth reason names its missing decode timing. + five reasons: the model qwen2.5-coder-1.5b-instruct-q4_k_m.gguf is not the blessed + Qwen3.5-4B; c=4, c=8 and c=16 each diverge from m=1 at 3 < 64 with no near-tie + margin recorded; and S7 is unmeasured, because the witness records no per-request + decode_tok_s at c=1 and c=4. The draft's five reasons included c=16 m_formed 14; + #4888 (8d44b58703, 2026-10-07) refreshed the witness, and c=16 now forms m=16. test: 'by hand: apr test llm shape-check evidence/perf041/lambda/witness.json' if_fails: 'A perf041 PASS is read as a V3 receipt: a V3 rule is missing, or main''s witness changed again.' - id: FALSIFY-SSP-008 @@ -190,25 +190,25 @@ falsification_tests: if_fails: 'The blank witness is parsed and reported as malformed JSON or NOT ADMISSIBLE.' - id: FALSIFY-SSP-017 rule: 'S7 (FALSIFY-CB-004): per-request decode at c=4 is at least half of c=1; the bar''s falsifier' - status: PLANNED prediction: > From the admissible control, set c=4 per-request decode to 49% of c=1: shape-check exits 5 with a reason naming FALSIFY-CB-004 and both rates. At exactly 50% it is admissible. A witness with no per-request decode timing at c=1 or c=4 is not - admissible, with a reason saying S7 is unmeasured. + admissible, with a reason saying S7 is unmeasured. The probe writes the rates S7 + reads: a band's decode_tok_s is the median of its slots' rates, a slot is timed from + its own token arrivals only on a stream the server declared live, and a band with + one untimed slot carries no rate. test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' expected_output: "test result: ok" if_fails: 'Architectural serialization or lock contention ships as a V3 PASS, or a witness with no timing passes S7 by default.' - id: FALSIFY-SSP-018 rule: 'at most one of the bands c=4, 8, 16 is admitted through the near-tie clause' - status: PLANNED prediction: 'From the admissible control, make c=4 and c=8 both diverge at 3 with margin 0.01: shape-check exits 5 with a reason naming the near-tie cap and both bands. One near-tie batched band (FALSIFY-SSP-002) stays admissible.' test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' expected_output: "test result: ok" if_fails: 'A systematic batched-path divergence passes by recording a margin in every band.' - id: FALSIFY-SSP-019 rule: 'declared_min has a floor of 64 (the perf matrix''s witness.min_agree_tokens)' - status: PLANNED prediction: 'From the admissible control, set one band''s declared_min to 1 and its divergence_at to 3: shape-check exits 5 with a reason naming the floor. declared_min 64 stays admissible.' test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::v3_shape' expected_output: "test result: ok" @@ -222,8 +222,10 @@ qa_gate: each boundary (divergence at and one below declared_min; margin at, just below and at zero; a negative margin). The RED rows (empty, blank, malformed, non-object, missing file) run in their own test. Each row gives its exit code and its exact - reasons, and the test is red on a row whose reasons differ by one. The CLI test - maps the verdicts onto apr's exit codes: 0, 5 NOT ADMISSIBLE, 4 RED, 3 no such file. + reasons, and the test is red on a row whose reasons differ by one. The bar's rows + join it: S7 at 49% and at 50% of c=1, untimed and zero rates, the near-tie cap, and + the declared_min floor. The CLI test maps the verdicts onto apr's exit codes: 0, 5 + NOT ADMISSIBLE, 4 RED, 3 no such file. checks: - planted_case_exits_as_its_row_says - not_admissible_case_prints_exactly_its_own_reasons @@ -253,7 +255,7 @@ findings: closed_by: 'v3_shape: a duplicate c is not admissible and every copy is checked; row `F6 duplicate c`.' - id: S7-scope summary: 'The draft did not check S7 and had the receipt cite CB-004 beside the witness.' - closed_by: 'Superseded by the bar: S7 is read from the same witness (FALSIFY-SSP-017, PLANNED until the code lands).' + closed_by: 'Superseded by the bar: S7 is read from the same witness. The probe records each band''s per-request decode_tok_s from live streams and v3_shape reads it (FALSIFY-SSP-017).' mutation_record: note: 'History: the draft''s record, measured on the Python checker. The Rust port has not been mutation-tested yet.' date: '2026-10-04' @@ -262,6 +264,7 @@ mutation_record: survived_table: [startswith-to-in, declared-min-default-64] verification: > cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli - --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. FALSIFY-SSP-007 by hand is - not_measured through the Rust port yet; its reasons above are read off main's witness - and the code. + --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. FALSIFY-SSP-007 by hand + gave exit 5 and the four pre-S7 reasons on gx10 at 32282bcd5f (2026-10-10 07:50Z); + its fifth reason, S7 unmeasured, is read off the code until a run at the S7 commit + measures it. diff --git a/crates/apr-cli/src/commands/test_llm_parity.rs b/crates/apr-cli/src/commands/test_llm_parity.rs index 37987a0553..742be98fe7 100644 --- a/crates/apr-cli/src/commands/test_llm_parity.rs +++ b/crates/apr-cli/src/commands/test_llm_parity.rs @@ -314,6 +314,7 @@ mod tests { declared_min: 64, max_constant_run_declared: 16, reason: reason.map(str::to_string), + decode_tok_s: None, slots: vec![], } } diff --git a/crates/apr-cli/src/commands/test_llm_shape.rs b/crates/apr-cli/src/commands/test_llm_shape.rs index 8ff04ffafa..3a42a1e625 100644 --- a/crates/apr-cli/src/commands/test_llm_shape.rs +++ b/crates/apr-cli/src/commands/test_llm_shape.rs @@ -75,9 +75,10 @@ mod tests { use super::*; use serde_json::{json, Value}; - /// The library's control witness: every rule S1-S6 holds. + /// The library's control witness: every rule S1-S7, the floor and the + /// cap hold. fn admissible() -> Value { - let band = |c: u32| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127}); + let band = |c: u32, rate: f64| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127, "decode_tok_s": rate}); json!({ "_planted": "FIXTURE: not a measurement", "binary_sha256": "a".repeat(64), @@ -85,7 +86,7 @@ mod tests { "host": "lambda", "prompt_sha256": "b".repeat(64), "model": {"path": "Qwen3.5-4B-Q4_K_M.gguf", "sha256": "c".repeat(64)}, - "bands": [band(1), band(4), band(8), band(16)], + "bands": [band(1, 40.0), band(4, 30.0), band(8, 25.0), band(16, 20.0)], }) } @@ -146,6 +147,36 @@ mod tests { assert_eq!(code, 0, "{printed}"); } + /// The V3 bar's falsifier (FALSIFY-SSP-017): from the admitted control, + /// set c=4's per-request decode to 49% of c=1's. Exit 5, naming CB-004. + #[test] + fn c4_decode_at_49_percent_of_c1_exits_5_and_names_cb_004() { + let mut w = admissible(); + w["bands"][1]["decode_tok_s"] = json!(19.6); + let (code, printed) = exit_of(&w.to_string()); + assert_eq!(code, 5, "{printed}"); + assert!(printed.contains(" - S7 (FALSIFY-CB-004): "), "{printed}"); + w["bands"][1]["decode_tok_s"] = json!(20.0); + assert_eq!(exit_of(&w.to_string()).0, 0, "50% holds"); + } + + /// The bar's cap row (FALSIFY-SSP-018): c=4 and c=8 both admitted only by + /// a near-tie. Exit 5, naming the cap. + #[test] + fn two_near_tie_batched_bands_exit_5_and_name_the_cap() { + let mut w = admissible(); + for band in [1, 2] { + w["bands"][band]["divergence_at"] = json!(3); + w["bands"][band]["top2_margin_at_divergence"] = json!(0.01); + } + let (code, printed) = exit_of(&w.to_string()); + assert_eq!(code, 5, "{printed}"); + assert!( + printed.contains(" - near-tie cap: 2 batched bands (c=4, c=8)"), + "{printed}" + ); + } + /// F1 across the mapping: RED never shares NOT ADMISSIBLE's code. And it /// names the input, not a model: `InvalidFormat` would print "Invalid APR /// format" over a JSON witness. diff --git a/crates/aprender-test-lib/src/llm/parity.rs b/crates/aprender-test-lib/src/llm/parity.rs index 8eee310f47..35640e49d7 100644 --- a/crates/aprender-test-lib/src/llm/parity.rs +++ b/crates/aprender-test-lib/src/llm/parity.rs @@ -12,9 +12,10 @@ use std::path::PathBuf; use futures::future::join_all; use super::client::{ChatMessage, ChatRequest, LlmClient, Role}; +use crate::perf_gate::drain::StreamMode; use crate::perf_gate::parity::{ - batch_formed_since, check_reference, evaluate_band, log_offset, run_verdict, ProbeBand, - ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, SampleResult, + batch_formed_since, check_reference, evaluate_band, live_decode_tok_s, log_offset, run_verdict, + ProbeBand, ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, SampleResult, }; use crate::perf_gate::witness::BatchInvariance; @@ -96,12 +97,19 @@ impl ParityProbe { } } -/// One streamed request, as the probe records it. +/// One streamed request, as the probe records it. S7's decode rate comes +/// from the client's own arrival times, and only on a stream the server +/// declared live: the batched CUDA path drops the server's `timings`. async fn fire(client: &LlmClient, request: &ChatRequest) -> SampleResult { client .chat_completion_stream(request) .await .map(|r| ProbeSample { + decode_tok_s: live_decode_tok_s( + r.stream_mode == Some(StreamMode::Live), + &r.token_timestamps, + r.usage.completion_tokens, + ), logprobs: r.logprobs, completion_tokens: Some(r.usage.completion_tokens), finish_reason: r.finish_reason, @@ -150,20 +158,24 @@ mod tests { format!("data: {chunk}\n\n") } - fn sse_body(tokens: &[(String, f64)]) -> String { + /// The stream's frames. A `live` stream declares it on the role chunk, + /// as `apr serve` does (PP-27); otherwise the stream declares nothing. + fn sse_frames(tokens: &[(String, f64)], live: bool) -> Vec { let n = tokens.len(); let usage = json!({"prompt_tokens": 5, "completion_tokens": n, "total_tokens": n + 5}); let terminal = json!({"choices": [{"delta": {}, "finish_reason": "length"}], "usage": usage}); - std::iter::once( - "data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n".to_string(), - ) - .chain(tokens.iter().map(|(t, gap)| token_frame(t, *gap))) - .chain([ - format!("data: {terminal}\n\n"), - "data: [DONE]\n\n".to_string(), - ]) - .collect() + let mut role = json!({"choices": [{"delta": {"role": "assistant"}}]}); + if live { + role["stream_mode"] = json!("live"); + } + std::iter::once(format!("data: {role}\n\n")) + .chain(tokens.iter().map(|(t, gap)| token_frame(t, *gap))) + .chain([ + format!("data: {terminal}\n\n"), + "data: [DONE]\n\n".to_string(), + ]) + .collect() } /// Read one request through its body, so the client never sees a reset. @@ -189,8 +201,10 @@ mod tests { /// The script's `_replay_server`: canned SSE per request index, and the /// scheduler's line appended to the log from inside the handler, as a real - /// server does, so the band offsets are exercised for real. - async fn replay(plan: Plan, log: PathBuf) -> String { + /// server does, so the band offsets are exercised for real. A `live` + /// server declares it and paces its frames 5 ms apart; otherwise the + /// whole body goes in one write. + async fn replay(plan: Plan, log: PathBuf, live: bool) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("bind loopback"); @@ -209,13 +223,21 @@ mod tests { .expect("open log"); writeln!(file, "{line}").expect("append log"); } - let body = sse_body(&tokens); + let frames = sse_frames(&tokens, live); let head = format!( "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\ Content-Length: {}\r\nConnection: close\r\n\r\n", - body.len() + frames.iter().map(String::len).sum::() ); - let _ = sock.write_all((head + &body).as_bytes()).await; + if live { + let _ = sock.write_all(head.as_bytes()).await; + for frame in frames { + let _ = sock.write_all(frame.as_bytes()).await; + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + } else { + let _ = sock.write_all((head + &frames.concat()).as_bytes()).await; + } let _ = sock.shutdown().await; }); } @@ -227,11 +249,19 @@ mod tests { /// are the references, 2 is c=1, and 3 and 4 are c=2. async fn probe( plan: impl Fn(usize) -> (Vec<(String, f64)>, Option) + Send + Sync + 'static, + ) -> ParityRun { + probe_with(false, plan).await + } + + /// [`probe`] against a server that declares its streams `live` or not. + async fn probe_with( + live: bool, + plan: impl Fn(usize) -> (Vec<(String, f64)>, Option) + Send + Sync + 'static, ) -> ParityRun { let dir = tempfile::tempdir().expect("tempdir"); let log = dir.path().join("server.log"); std::fs::write(&log, "server starting\n").expect("seed log"); - let url = replay(Arc::new(plan), log.clone()).await; + let url = replay(Arc::new(plan), log.clone(), live).await; let probe = ParityProbe { prompt: "Write an essay on compilers.".to_string(), ladder: vec![1, 2], @@ -267,6 +297,33 @@ mod tests { assert_eq!(run.bands[1].divergence_at, Some(8)); } + /// S7's input, end to end: a stream the server declared live records a + /// per-request decode rate on every slot and band; the same stream + /// undeclared records none, so V3 reads S7 as unmeasured, never as fast. + /// Only the rate's sign is asserted: its size is the test box's timing. + #[tokio::test] + async fn a_live_stream_records_per_request_decode_and_an_undeclared_one_does_not() { + let live = probe_with(true, |i| (canned("a"), batch_line(i))).await; + assert_eq!(live.result, BatchInvariance::Pass, "{live:?}"); + for band in &live.bands { + let rate = band.decode_tok_s.expect("a live band is timed"); + assert!(rate.is_finite() && rate > 0.0, "{band:?}"); + assert!( + band.slots.iter().all(|s| s.decode_tok_s.is_some()), + "{band:?}" + ); + } + let undeclared = probe(|i| (canned("a"), batch_line(i))).await; + assert_eq!(undeclared.result, BatchInvariance::Pass, "{undeclared:?}"); + for band in &undeclared.bands { + assert_eq!(band.decode_tok_s, None, "{band:?}"); + assert!( + band.slots.iter().all(|s| s.decode_tok_s.is_none()), + "{band:?}" + ); + } + } + /// `witness_no_batch_formed_is_unmeasurable`, end to end. #[tokio::test] async fn a_box_that_never_batches_is_unmeasurable_never_a_pass() { diff --git a/crates/aprender-test-lib/src/perf_gate/mod.rs b/crates/aprender-test-lib/src/perf_gate/mod.rs index f7e4c9cbb7..deda6c664d 100644 --- a/crates/aprender-test-lib/src/perf_gate/mod.rs +++ b/crates/aprender-test-lib/src/perf_gate/mod.rs @@ -195,9 +195,9 @@ pub use join::{BandRatios, JoinKey, Ratio, RatioBound, RatioMethod}; pub use margin::{token_ids, top2_margin, TokenLogprob, TopLogprob, LOGPROBS_SOURCE}; pub use metrics::{agg_tok_s, aggregate_terms, BandMetrics, RequestSample}; pub use parity::{ - batch_formed_since, check_reference, evaluate_band, run_verdict, ProbeBand, ProbeMatrix, - ProbeModel, ProbePolicy, ProbeReference, ProbeSample, ProbeSampler, ProbeSlot, ProbeWitness, - SampleResult, + band_decode_tok_s, batch_formed_since, check_reference, evaluate_band, live_decode_tok_s, + run_verdict, ProbeBand, ProbeMatrix, ProbeModel, ProbePolicy, ProbeReference, ProbeSample, + ProbeSampler, ProbeSlot, ProbeWitness, SampleResult, }; pub use protocol::{ min_sampled_requests, warmup_requests, BandConfig, ClientModel, ProtocolParams, ProtocolSource, @@ -213,7 +213,8 @@ pub use receipt::{ pub use replicate::{log_ratio_lcb, t_lower_one_sided_95, ArmOrder, ReplicatePair, MIN_REPLICATES}; pub use samples::{read_samples_gz, write_samples_gz, SamplesFile}; pub use v3_shape::{ - check_v3_shape, check_v3_shape_file, ShapeVerdict, BLESSED_MODEL_PREFIX, NEAR_TIE_EPS, V3_BANDS, + check_v3_shape, check_v3_shape_file, ShapeVerdict, BLESSED_MODEL_PREFIX, DECLARED_MIN_FLOOR, + NEAR_TIE_CAP, NEAR_TIE_EPS, S7_MIN_RATIO, V3_BANDS, }; pub use window::{WindowController, WindowReport}; pub use witness::{BatchInvariance, BatchInvarianceWitness}; diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs index 376a588343..d386353371 100644 --- a/crates/aprender-test-lib/src/perf_gate/parity.rs +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -24,11 +24,13 @@ use std::collections::BTreeMap; use std::io::{Read, Seek, SeekFrom}; use std::path::Path; +use std::time::Duration; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use super::margin::{token_ids, TokenLogprob}; +use super::metrics::percentile; use super::protocol::{matrix_block, ProtocolParams, PERF_MATRIX_SOURCE}; use super::witness::{BatchInvariance, BatchInvarianceWitness}; @@ -76,6 +78,31 @@ pub struct ProbeSample { pub completion_tokens: Option, /// The terminal chunk's `finish_reason`. pub finish_reason: Option, + /// S7: this request's decode rate, from [`live_decode_tok_s`]. + pub decode_tok_s: Option, +} + +/// §4.4.3 `decode_tok_s` for one streamed request, as +/// [`RequestSample::decode_tok_s`](super::metrics::RequestSample::decode_tok_s) +/// defines it: `(completion tokens − 1) / (last token arrival − first)`, from +/// the client's arrival times. +/// +/// `None` unless the server declared the stream `live` (PP-27): a replayed +/// stream's arrival times time the replay, not the decode, and an undeclared +/// stream is not assumed live. `None` too with fewer than two tokens or +/// arrivals, or no time between them, where the rate is undefined, not zero. +/// The server's own `timings` cannot stand in: the batched path drops them +/// (`cuda_batch_scheduler.rs`), and S7 is a claim about that path. +#[must_use] +pub fn live_decode_tok_s(live: bool, arrivals: &[Duration], completion_tokens: u32) -> Option { + if !live || arrivals.len() < 2 || completion_tokens < 2 { + return None; + } + let span = arrivals + .last()? + .saturating_sub(*arrivals.first()?) + .as_secs_f64(); + (span > 0.0).then(|| f64::from(completion_tokens - 1) / span) } /// A request that came back, or why it did not. @@ -178,6 +205,9 @@ pub struct ProbeSlot { pub finish_reason: Option, /// Why the slot could not be compared. pub refused: Option, + /// S7: the request's decode rate, `null` when it was not timed. + #[serde(default)] + pub decode_tok_s: Option, } impl ProbeSlot { @@ -188,6 +218,7 @@ impl ProbeSlot { completion_tokens: ok.and_then(|s| s.completion_tokens), finish_reason: ok.and_then(|s| s.finish_reason.clone()), refused: accepted(sample, n_predict).err(), + decode_tok_s: ok.and_then(|s| s.decode_tok_s), } } } @@ -218,10 +249,27 @@ pub struct ProbeBand { pub max_constant_run_declared: u32, /// Why the band is not PASS. pub reason: Option, + /// S7: the median of the slots' decode rates, from [`band_decode_tok_s`]. + #[serde(default)] + pub decode_tok_s: Option, /// Every slot fired. pub slots: Vec, } +/// §4.4.3 — a band's per-request decode rate: the median of its slots'. +/// `None` when the band has no slot or any slot was not timed: the median of +/// the slots that happened to be timed is not the band's, and V3 reads `None` +/// as S7 unmeasured, never assumed. +#[must_use] +pub fn band_decode_tok_s(slots: &[ProbeSlot]) -> Option { + let mut rates = slots + .iter() + .map(|s| s.decode_tok_s) + .collect::>>()?; + rates.sort_by(f64::total_cmp); + percentile(&rates, 0.50) +} + impl ProbeBand { fn unmeasurable(c: u32, m_formed: u32, policy: &ProbePolicy, slots: Vec) -> Self { Self { @@ -235,6 +283,7 @@ impl ProbeBand { declared_min: policy.declared_min, max_constant_run_declared: policy.max_constant_run, reason: None, + decode_tok_s: band_decode_tok_s(&slots), slots, } } @@ -590,11 +639,19 @@ mod tests { words.iter().map(|w| tok(w, 1.0)).collect() } + /// The decode rate every [`sample`] was timed at. + const RATE: f64 = 40.0; + fn sample(tokens: Vec) -> SampleResult { + timed(tokens, Some(RATE)) + } + + fn timed(tokens: Vec, decode_tok_s: Option) -> SampleResult { Ok(ProbeSample { completion_tokens: u32::try_from(tokens.len()).ok(), finish_reason: Some("length".to_string()), logprobs: tokens, + decode_tok_s, }) } @@ -619,6 +676,53 @@ mod tests { assert_eq!(band.divergence_at, Some(8)); assert_eq!(band.top2_margin_at_divergence, None); assert_eq!(band.reason, None); + assert_eq!(band.decode_tok_s, Some(RATE)); + } + + /// S7's rate, as `RequestSample::decode_tok_s` defines it, and every case + /// where it is undefined rather than zero. + #[test] + fn live_decode_tok_s_times_only_a_live_stream_with_two_arrivals() { + let ms = Duration::from_millis; + let arrivals = [ms(100), ms(150), ms(300)]; + let near = |got: Option, want: f64| got.is_some_and(|r| (r - want).abs() < 1e-9); + assert!( + near(live_decode_tok_s(true, &arrivals, 9), 40.0), + "8 tokens over 0.2 s" + ); + assert!( + near(live_decode_tok_s(true, &arrivals, 2), 5.0), + "1 token over 0.2 s" + ); + assert_eq!( + live_decode_tok_s(false, &arrivals, 9), + None, + "a replay is not timed" + ); + assert_eq!(live_decode_tok_s(true, &arrivals[..1], 9), None); + assert_eq!(live_decode_tok_s(true, &[], 9), None); + assert_eq!(live_decode_tok_s(true, &arrivals, 1), None); + assert_eq!( + live_decode_tok_s(true, &[ms(5), ms(5)], 9), + None, + "no time between" + ); + } + + #[test] + fn a_band_rate_is_the_slots_median_and_unmeasured_if_any_slot_is() { + let slot = |decode_tok_s| ProbeSlot { + i: 0, + completion_tokens: Some(8), + finish_reason: None, + refused: None, + decode_tok_s, + }; + let odd = [slot(Some(10.0)), slot(Some(30.0)), slot(Some(20.0))]; + assert_eq!(band_decode_tok_s(&odd), Some(20.0)); + assert_eq!(band_decode_tok_s(&odd[..2]), Some(20.0)); + assert_eq!(band_decode_tok_s(&[slot(Some(10.0)), slot(None)]), None); + assert_eq!(band_decode_tok_s(&[]), None); } /// The script's `witness_constant_token_m3`: #2753, one id forever. @@ -673,6 +777,7 @@ mod tests { logprobs: Vec::new(), completion_tokens: Some(8), finish_reason: None, + decode_tok_s: None, }); let band = band_of(2, 2, &slots); assert_eq!(band.result, BatchInvariance::Unmeasurable); @@ -688,6 +793,10 @@ mod tests { slots[0] = Err("connection refused".to_string()); let band = band_of(2, 2, &slots); assert_eq!(band.result, BatchInvariance::Unmeasurable); + assert_eq!( + band.decode_tok_s, None, + "a slot that never came back was not timed" + ); assert!(band .reason .unwrap_or_default() @@ -820,6 +929,41 @@ mod tests { assert_eq!(max_batch_formed("Batch m=12"), 0); } + /// What V3's floor admits: `declared_min` at the perf matrix's 64. + const V3_POLICY: ProbePolicy = ProbePolicy { + declared_min: 64, + n_predict: 96, + max_constant_run: 5, + }; + + /// `n_predict` distinct tokens, every step decided by a margin of 1. + fn v3_reference() -> Vec { + (0..V3_POLICY.n_predict) + .map(|i| tok(&format!("t{i} "), 1.0)) + .collect() + } + + /// A band of `c` slots streaming `slot` against `m1`, each timed at `rate`. + fn v3_band(c: u32, m1: &[TokenLogprob], slot: &[TokenLogprob], rate: Option) -> ProbeBand { + let slots: Vec = (0..c).map(|_| timed(slot.to_vec(), rate)).collect(); + evaluate_band(c, c, m1, &slots, &V3_POLICY) + } + + fn v3_passed(c: u32, rate: f64) -> ProbeBand { + v3_band(c, &v3_reference(), &v3_reference(), Some(rate)) + } + + /// A clean run whose c=4 band decodes at `c4_rate` and the rest at 40. + fn v3_bands(c4_rate: Option) -> Vec { + let m1 = v3_reference(); + vec![ + v3_passed(1, RATE), + v3_band(4, &m1, &m1, c4_rate), + v3_passed(8, RATE), + v3_passed(16, RATE), + ] + } + fn witness(bands: Vec) -> ProbeWitness { let reference = ProbeReference { stable: true, @@ -840,10 +984,10 @@ mod tests { temperature: 0.0, seed: 0, ignore_eos: true, - max_tokens: POLICY.n_predict, + max_tokens: V3_POLICY.n_predict, top_logprobs: 2, }, - declared_min: POLICY.declared_min, + declared_min: V3_POLICY.declared_min, env: env_block(), result: run_verdict(&reference, &bands), reference, @@ -856,7 +1000,7 @@ mod tests { /// still record `divergence_at`, or V3 reads it as unmeasured. #[test] fn a_clean_run_writes_a_witness_the_v3_shape_check_admits() { - let w = witness(vec![passed(1), passed(4), passed(8), passed(16)]); + let w = witness(v3_bands(Some(RATE))); assert_eq!(w.result, BatchInvariance::Pass); let text = serde_json::to_string_pretty(&w).expect("serialize"); assert_eq!(check_v3_shape(&text), ShapeVerdict::Admissible, "{text}"); @@ -868,12 +1012,14 @@ mod tests { /// the margin. Without the margin it would not be. #[test] fn a_near_tie_divergence_is_admitted_by_v3_only_with_its_margin() { - let mut m1 = reference(); - m1[1] = tok("b", 0.01); - let mut flipped = reference(); + let mut m1 = v3_reference(); + m1[1] = tok("t1 ", 0.01); + let mut flipped = v3_reference(); flipped[1] = tok("~", 0.01); - let flip = evaluate_band(8, 8, &m1, &copies(8, &flipped), &POLICY); - let mut w = witness(vec![passed(1), passed(4), flip, passed(16)]); + let flip = v3_band(8, &m1, &flipped, Some(RATE)); + assert_eq!(flip.divergence_at, Some(1), "{flip:?}"); + let mut w = witness(v3_bands(Some(RATE))); + w.bands[2] = flip; let text = serde_json::to_string(&w).expect("serialize"); assert_eq!(check_v3_shape(&text), ShapeVerdict::Admissible, "{text}"); w.bands[2].top2_margin_at_divergence = None; @@ -886,7 +1032,7 @@ mod tests { #[test] fn missing_identity_stays_null_and_v3_names_it() { - let mut w = witness(vec![passed(1), passed(4), passed(8), passed(16)]); + let mut w = witness(v3_bands(Some(RATE))); w.commit = None; let text = serde_json::to_string(&w).expect("serialize"); assert!(text.contains("\"commit\":null"), "{text}"); @@ -896,6 +1042,34 @@ mod tests { ); } + /// FALSIFY-SSP-017 through the producer: the rates the probe records are + /// the rates V3 reads. c=4 at 49% of c=1 is refused by name; at 50% it is + /// admitted; one untimed c=4 slot leaves S7 unmeasured. + #[test] + fn the_band_rates_the_probe_writes_are_what_v3_s7_reads() { + let check = |c4_rate| { + check_v3_shape(&serde_json::to_string(&witness(v3_bands(c4_rate))).expect("serialize")) + }; + assert_eq!( + check(Some(19.6)), + ShapeVerdict::NotAdmissible(vec![ + "S7 (FALSIFY-CB-004): per-request decode at c=4 is 19.6 tok/s, 49.0% of c=1's 40 tok/s; needs at least 50%".to_string() + ]) + ); + assert_eq!(check(Some(20.0)), ShapeVerdict::Admissible); + let unmeasured = ShapeVerdict::NotAdmissible(vec![ + "S7 (FALSIFY-CB-004) is unmeasured: no positive per-request decode_tok_s at c=4" + .to_string(), + ]); + assert_eq!(check(None), unmeasured); + let mut w = witness(v3_bands(Some(RATE))); + w.bands[1].slots[2].decode_tok_s = None; + w.bands[1].decode_tok_s = band_decode_tok_s(&w.bands[1].slots); + let text = serde_json::to_string(&w).expect("serialize"); + assert!(text.contains("\"decode_tok_s\":null"), "{text}"); + assert_eq!(check_v3_shape(&text), unmeasured); + } + #[test] fn env_block_records_every_key_and_prompt_sha256_is_64_hex() { let env = env_block(); diff --git a/crates/aprender-test-lib/src/perf_gate/v3_shape.rs b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs index cdf77c612c..4dbfd08341 100644 --- a/crates/aprender-test-lib/src/perf_gate/v3_shape.rs +++ b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs @@ -15,10 +15,15 @@ //! | S4 | identity: `binary_sha256`, `commit`, `host`, `prompt_sha256` | //! | S5 | agreement with the `m = 1` reference to `declared_min`, unless a recorded top-2 margin below ε explains the divergence as a near-tie | //! | S6 | perf041 PASS in every band | +//! | S7 | FALSIFY-CB-004: per-request decode at c=4 at least half of c=1's, read from the bands' `decode_tok_s` | +//! | floor | every band's `declared_min` is at least 64, the perf matrix's `witness.min_agree_tokens` | +//! | cap | at most one of c=4, 8, 16 is admitted through the near-tie clause | //! -//! S7 (FALSIFY-CB-004, per-request decode at c=4 at least half of c=1) needs -//! timing the witness does not carry. The receipt cites it beside the witness, -//! and this check does not read it. +//! S7, the floor and the cap are the V3 bar's (C314 round 2, contract +//! `serving-shape-parity-v1`), as is ε = 0.05 logit, now fixed. A witness +//! with no per-request decode timing at c=1 or c=4 is not admissible: S7 is +//! unmeasured, never assumed. The probe times a request only on a stream the +//! server declared live ([`super::parity::live_decode_tok_s`]). //! //! The port closes the draft contract's findings: //! * **F1** an unreadable witness is RED (exit 2), never NOT ADMISSIBLE; @@ -42,14 +47,26 @@ pub const BLESSED_MODEL_PREFIX: &str = "qwen3.5-4b"; /// S2: the bands `apr serve` advertises. pub const V3_BANDS: [u32; 4] = [1, 4, 8, 16]; -/// S5: ε in logits, `[A]` until measured near-ties calibrate it. A recorded -/// top-2 margin explains a divergence only when `0 <= margin < ε`. +/// S5: ε in logits, fixed by the V3 bar. A recorded top-2 margin explains a +/// divergence only when `0 <= margin < ε`. pub const NEAR_TIE_EPS: f64 = 0.05; +/// The cap: batched bands (c > 1) that may be admitted through the near-tie +/// clause. Bands c=4, 8 and 16 share one batched path, so more than one early +/// divergence there points at the path, not at independent near-ties. +pub const NEAR_TIE_CAP: usize = 1; + +/// The floor under every band's `declared_min`: the perf matrix's +/// `witness.min_agree_tokens`, so a witness cannot declare its own bar. +pub const DECLARED_MIN_FLOOR: u32 = 64; + +/// S7 (FALSIFY-CB-004): per-request decode at c=4 over c=1's, at least. +pub const S7_MIN_RATIO: f64 = 0.5; + /// The check's answer. Exit codes follow the Python reference: 0, 1, 2. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ShapeVerdict { - /// Every rule S1–S6 holds. + /// Every rule S1–S7, the floor and the cap hold. Admissible, /// The witness was read, and these rules failed. NotAdmissible(Vec), @@ -115,6 +132,7 @@ struct ShapeBand { divergence_at: Option, declared_min: Option, top2_margin_at_divergence: Option, + decode_tok_s: Option, } /// Check the witness file at `path`. A file that cannot be read is RED (F1). @@ -182,7 +200,7 @@ fn blank(value: &str) -> bool { value.trim().is_empty() } -/// S2, F6 and the per-band rules, in band order. +/// S2, F6 and the per-band rules in band order, then the cap and S7. fn bands_reasons(bands: &[ShapeBand]) -> Vec { let mut by_c: BTreeMap> = BTreeMap::new(); for band in bands { @@ -204,10 +222,12 @@ fn bands_reasons(bands: &[ShapeBand]) -> Vec { out.extend(band_reasons(band)); } } + out.extend(near_tie_cap_reason(&by_c)); + out.extend(s7_reason(&by_c)); out } -/// S6, S3 and S5 for one band. +/// S6, S3, the floor and S5 for one band. fn band_reasons(band: &ShapeBand) -> Vec { let c = band.c; let result = band.result.as_deref(); @@ -223,10 +243,88 @@ fn band_reasons(band: &ShapeBand) -> Vec { Some(m) => out.push(format!("c={c}: m_formed {m} != c (shape not served)")), None => out.push(format!("c={c}: m_formed not recorded (shape not shown)")), } + if let Some(min) = band.declared_min.filter(|&min| min < DECLARED_MIN_FLOOR) { + out.push(format!( + "c={c}: declared_min {min} is below the floor of {DECLARED_MIN_FLOOR} \ + (witness.min_agree_tokens)" + )); + } out.extend(divergence_reason(band, result == Some("PASS"))); out } +/// S5's near-tie clause: a recorded top-2 margin in `[0, ε)`. +fn near_tie(margin: f64) -> bool { + (0.0..NEAR_TIE_EPS).contains(&margin) +} + +/// True when S5 holds for `band` only through the near-tie clause: it parts +/// from m=1 before `declared_min`, at a recorded near-tie. +fn admitted_by_near_tie(band: &ShapeBand) -> bool { + match ( + band.declared_min, + band.divergence_at, + band.top2_margin_at_divergence, + ) { + (Some(min), Some(at), Some(margin)) => at < min && near_tie(margin), + _ => false, + } +} + +/// The cap: at most [`NEAR_TIE_CAP`] batched V3 bands admitted through the +/// near-tie clause. c=1 is not batched, and a band outside V3 is ignored, as +/// everywhere else. +fn near_tie_cap_reason(by_c: &BTreeMap>) -> Option { + let tied: Vec = V3_BANDS + .iter() + .filter(|&&c| c > 1) + .flat_map(|c| by_c.get(c).into_iter().flatten()) + .filter(|band| admitted_by_near_tie(band)) + .map(|band| format!("c={}", band.c)) + .collect(); + (tied.len() > NEAR_TIE_CAP).then(|| { + format!( + "near-tie cap: {} batched bands ({}) are admitted only by a near-tie margin; \ + at most {NEAR_TIE_CAP} of c=4, 8, 16 may be", + tied.len(), + tied.join(", ") + ) + }) +} + +/// S7 (FALSIFY-CB-004): per-request decode at c=4 at least [`S7_MIN_RATIO`] +/// of c=1's. Read only when c=1 and c=4 each have exactly one band: a missing +/// or doubled band already has its reason, and which copy's rate S7 would read +/// is not defined. A band with no rate, or a rate that is not positive, leaves +/// S7 unmeasured, and an unmeasured S7 is not admissible. +fn s7_reason(by_c: &BTreeMap>) -> Option { + let one = |c: u32| match by_c.get(&c).map(Vec::as_slice) { + Some([band]) => Some(*band), + _ => None, + }; + let (c1, c4) = (one(1)?, one(4)?); + let rate = |band: &ShapeBand| band.decode_tok_s.filter(|&r| r > 0.0); + let (Some(d1), Some(d4)) = (rate(c1), rate(c4)) else { + let unmeasured: Vec<&str> = [(c1, "c=1"), (c4, "c=4")] + .into_iter() + .filter(|&(band, _)| rate(band).is_none()) + .map(|(_, name)| name) + .collect(); + return Some(format!( + "S7 (FALSIFY-CB-004) is unmeasured: no positive per-request decode_tok_s at {}", + unmeasured.join(" and ") + )); + }; + (d4 < S7_MIN_RATIO * d1).then(|| { + format!( + "S7 (FALSIFY-CB-004): per-request decode at c=4 is {d4} tok/s, {:.1}% of c=1's \ + {d1} tok/s; needs at least {}%", + 100.0 * d4 / d1, + 100.0 * S7_MIN_RATIO + ) + }) +} + /// S5, with F3 (no `declared_min`), F4 (name the recorded margin) and F5 (no /// divergence measurement on a band that did not pass: its result says it). fn divergence_reason(band: &ShapeBand, passed: bool) -> Option { @@ -248,7 +346,7 @@ fn divergence_reason(band: &ShapeBand, passed: bool) -> Option { None => Some(format!( "c={c}: diverges from m=1 at {at} < {min} with no near-tie margin recorded" )), - Some(margin) if (0.0..NEAR_TIE_EPS).contains(&margin) => None, + Some(margin) if near_tie(margin) => None, Some(margin) => Some(format!( "c={c}: diverges from m=1 at {at} < {min}; the recorded top-2 margin {margin} \ is not a near-tie (needs 0 <= margin < {NEAR_TIE_EPS})" @@ -262,9 +360,10 @@ mod tests { use serde_json::{json, Value}; /// The admissible control, as `fixtures/v3/admissible.json` on the spec - /// branch has it. Every case below changes one thing in it. + /// branch has it, plus S7's per-request decode rates (c=4 at 75% of c=1). + /// Every case below changes one thing in it. fn admissible() -> Value { - let band = |c: u32| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127}); + let band = |c: u32, rate: f64| json!({"c": c, "m_formed": c, "result": "PASS", "declared_min": 64, "divergence_at": 127, "decode_tok_s": rate}); json!({ "_planted": "FIXTURE: not a measurement", "binary_sha256": "a".repeat(64), @@ -272,7 +371,7 @@ mod tests { "host": "lambda", "prompt_sha256": "b".repeat(64), "model": {"path": "Qwen3.5-4B-Q4_K_M.gguf", "sha256": "c".repeat(64)}, - "bands": [band(1), band(4), band(8), band(16)], + "bands": [band(1, 40.0), band(4, 30.0), band(8, 25.0), band(16, 20.0)], }) } @@ -312,6 +411,20 @@ mod tests { } } + fn remove(index: usize, key: &'static str) -> impl Fn(&mut Value) { + move |w: &mut Value| { + band_mut(w, index).remove(key); + } + } + + /// Several changes, in order. + fn both(a: impl Fn(&mut Value), b: impl Fn(&mut Value)) -> impl Fn(&mut Value) { + move |w: &mut Value| { + a(w); + b(w); + } + } + type Mutation = Box; /// One planted case: a change to the control, the exit code it must give, @@ -443,10 +556,84 @@ mod tests { 0, vec![], ), + // FALSIFY-SSP-017, S7: the bar's falsifier, its boundary, and a + // witness that cannot say. + ( + "S7 c=4 at 49% of c=1 (the bar's falsifier)", + Box::new(set(1, "decode_tok_s", json!(19.6))), + 1, + vec!["S7 (FALSIFY-CB-004): per-request decode at c=4 is 19.6 tok/s, 49.0% of c=1's 40 tok/s; needs at least 50%"], + ), + ("S7 c=4 at exactly 50%", Box::new(set(1, "decode_tok_s", json!(20.0))), 0, vec![]), + ("S7 reads c=4, not c=8 or c=16", Box::new(both(set(2, "decode_tok_s", json!(1.0)), set(3, "decode_tok_s", json!(1.0)))), 0, vec![]), + ( + "S7 no timing at c=4", + Box::new(remove(1, "decode_tok_s")), + 1, + vec!["S7 (FALSIFY-CB-004) is unmeasured: no positive per-request decode_tok_s at c=4"], + ), + ( + "S7 null timing at c=1", + Box::new(set(0, "decode_tok_s", Value::Null)), + 1, + vec!["S7 (FALSIFY-CB-004) is unmeasured: no positive per-request decode_tok_s at c=1"], + ), + ( + "S7 zero rates are not rates", + Box::new(both(set(0, "decode_tok_s", json!(0.0)), set(1, "decode_tok_s", json!(0.0)))), + 1, + vec!["S7 (FALSIFY-CB-004) is unmeasured: no positive per-request decode_tok_s at c=1 and c=4"], + ), + ( + "S7 is not read beside a missing c=1 (its band says it)", + Box::new(both(drop_band(0), set(0, "decode_tok_s", Value::Null))), + 1, + vec!["band c=1 not measured"], + ), + ("F2 decode_tok_s as a string", Box::new(set(1, "decode_tok_s", json!("30"))), 2, vec![]), + // FALSIFY-SSP-018, the cap: one near-tie batched band is chance, + // two are the batched path. + ( + "near-tie cap: c=4 and c=8", + Box::new(both(diverge(1, 3, Some(json!(0.01))), diverge(2, 3, Some(json!(0.01))))), + 1, + vec!["near-tie cap: 2 batched bands (c=4, c=8) are admitted only by a near-tie margin; at most 1 of c=4, 8, 16 may be"], + ), + ( + "near-tie cap: all three batched bands", + Box::new(|w: &mut Value| { + for index in 1..4 { + diverge(index, 3, Some(json!(0.01)))(w); + } + }), + 1, + vec!["near-tie cap: 3 batched bands (c=4, c=8, c=16) are admitted only by a near-tie margin; at most 1 of c=4, 8, 16 may be"], + ), + ("near-tie at c=16 alone", Box::new(diverge(3, 3, Some(json!(0.01)))), 0, vec![]), + ("near-tie cap: c=1 is not batched", Box::new(both(diverge(0, 3, Some(json!(0.01))), diverge(1, 3, Some(json!(0.01))))), 0, vec![]), + ( + "near-tie cap: a margin on a band that agreed to declared_min is not counted", + Box::new(both(diverge(1, 64, Some(json!(0.01))), diverge(2, 3, Some(json!(0.01))))), + 0, + vec![], + ), + // FALSIFY-SSP-019, the floor: a witness does not set its own bar. + ( + "floor: declared_min 1", + Box::new(both(set(1, "declared_min", json!(1)), diverge(1, 3, None))), + 1, + vec!["c=4: declared_min 1 is below the floor of 64 (witness.min_agree_tokens)"], + ), + ( + "floor: declared_min 63", + Box::new(set(2, "declared_min", json!(63))), + 1, + vec!["c=8: declared_min 63 is below the floor of 64 (witness.min_agree_tokens)"], + ), ] } - /// FALSIFY-SSP-001..006, 008..016 and F1–F6: every planted case exits as + /// FALSIFY-SSP-001..006, 008..019 and F1–F6: every planted case exits as /// its row says and prints exactly its row's reasons. A RED row with no /// reasons listed checks the exit code and the `RED:` prefix. #[test] @@ -509,6 +696,17 @@ mod tests { let _ = std::fs::remove_dir(&dir); } + /// The floor is the shipped perf matrix's `witness.min_agree_tokens`, not + /// a second copy of it that can drift. + #[test] + fn the_floor_is_the_shipped_matrix_min_agree_tokens() { + let shipped = super::super::protocol::witness_min_agree_tokens_from( + super::super::protocol::PERF_MATRIX_SOURCE, + ) + .expect("the shipped witness block"); + assert_eq!(DECLARED_MIN_FLOOR, shipped); + } + #[test] fn exit_codes_and_labels_follow_the_python_reference() { let rows = [ From e43f6d0037d7d74f9e8cd9e6c1d35e0603d19a10 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 10:11:25 +0200 Subject: [PATCH 24/35] docs(contracts): serving-shape-parity-v1 records the S7 run on gx10 (PMAT-4971) At 72283efd5e on gx10 (aarch64, CPU only), FALSIFY-SSP-007 by hand gave exit 5 with its five reasons, S7 unmeasured among them. The perf_gate, llm::parity and apr-cli test_llm tests all passed. No GPU host class has run the probe yet. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 83ce5b232c68b9f17128d644bf7afdf0b099a945) (cherry picked from commit d3d31b891a0fe6aee802e6992a963a55ce67a0a1) (cherry picked from commit d34127cad5f53e2c9f145679d78d05536ae6e29d) --- contracts/serving-shape-parity-v1.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contracts/serving-shape-parity-v1.yaml b/contracts/serving-shape-parity-v1.yaml index 2828fe2dcb..f7cc20a25d 100644 --- a/contracts/serving-shape-parity-v1.yaml +++ b/contracts/serving-shape-parity-v1.yaml @@ -264,7 +264,7 @@ mutation_record: survived_table: [startswith-to-in, declared-min-default-64] verification: > cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli - --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. FALSIFY-SSP-007 by hand - gave exit 5 and the four pre-S7 reasons on gx10 at 32282bcd5f (2026-10-10 07:50Z); - its fifth reason, S7 unmeasured, is read off the code until a run at the S7 commit - measures it. + --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. At 72283efd5e (2026-10-10 + 08:11Z, gx10 aarch64, CPU only): perf_gate 251 passed, llm::parity 7 passed, apr-cli + test_llm 89 passed, and FALSIFY-SSP-007 by hand gave exit 5 with its five reasons, + S7 unmeasured among them. No GPU host class has run the probe yet. From 0a09a762bc5d77d6cae4610e6879ceaa85043e0a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 10:43:15 +0200 Subject: [PATCH 25/35] test(perf_gate): kill the three g16 survivors in the V3 probe (PMAT-4971) The first mutation run of the Rust V3 shape check (gx10 g16 at 8978becb22, perf_gate tests) left three survivors in parity.rs: - `arrivals.len() < 2` in live_decode_tok_s mutated to `==` and `<=`. The clause was redundant: `?` on first/last and `span > 0.0` already give None below two arrivals. It goes, and a two-arrival row pins that two arrivals are enough. - check_reference's `tokens` field could be deleted unnoticed. The reference test now asserts it, including when ref#1 is refused. Not yet built; a gx10 run verifies it before this is pushed. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6d913abb78a83295a02f4813a82d319d0808450c) (cherry picked from commit 281eed1395351ea2078e084b083977afd196dd68) (cherry picked from commit 788dcaf048894124eae246ed05dd31b947a9f613) --- crates/aprender-test-lib/src/perf_gate/parity.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs index d386353371..44ce2b80a6 100644 --- a/crates/aprender-test-lib/src/perf_gate/parity.rs +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -95,9 +95,10 @@ pub struct ProbeSample { /// (`cuda_batch_scheduler.rs`), and S7 is a claim about that path. #[must_use] pub fn live_decode_tok_s(live: bool, arrivals: &[Duration], completion_tokens: u32) -> Option { - if !live || arrivals.len() < 2 || completion_tokens < 2 { + if !live || completion_tokens < 2 { return None; } + // Fewer than two arrivals: no first (`?`) or no time between them (`span`). let span = arrivals .last()? .saturating_sub(*arrivals.first()?) @@ -694,6 +695,10 @@ mod tests { near(live_decode_tok_s(true, &arrivals, 2), 5.0), "1 token over 0.2 s" ); + assert!( + near(live_decode_tok_s(true, &[ms(100), ms(300)], 9), 40.0), + "two arrivals are enough" + ); assert_eq!( live_decode_tok_s(false, &arrivals, 9), None, @@ -846,6 +851,7 @@ mod tests { let ok = check_reference(&sample(reference()), &sample(reference()), &POLICY); assert!(ok.stable, "{ok:?}"); assert_eq!(ok.self_divergence_at, Some(8)); + assert_eq!(ok.tokens, Some(8), "the first decode's completion_tokens"); let mut early = reference(); early[2] = tok("X", 1.0); let unstable = check_reference(&sample(reference()), &sample(early), &POLICY); @@ -857,6 +863,11 @@ mod tests { &POLICY, ); assert!(!short.stable); + assert_eq!( + short.tokens, + Some(3), + "recorded when the first decode is refused" + ); assert!(short .reason .unwrap_or_default() From 4911d7da0eea034c8d78eb2135c8a39702fc8ca3 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 11:56:28 +0200 Subject: [PATCH 26/35] test(perf_gate): pin check_reference at exactly declared_min (PMAT-4971) g17 on gx10 left one survivor: `replace < with <= in check_reference` (parity.rs:186). No test had the two m=1 references agree to exactly declared_min, so the boundary was free. They now diverge at token 4 with declared_min 4, which must read as stable. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7b4252408b3f85f8d372845184422401d8900320) (cherry picked from commit 792b150f5d77a61fc86944423f025e4784aaa3f7) (cherry picked from commit 4b676a2a245824f5d91a5cd52df47b22c7ece705) --- crates/aprender-test-lib/src/perf_gate/parity.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs index 44ce2b80a6..1c57b9c3d9 100644 --- a/crates/aprender-test-lib/src/perf_gate/parity.rs +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -857,6 +857,14 @@ mod tests { let unstable = check_reference(&sample(reference()), &sample(early), &POLICY); assert!(!unstable.stable); assert_eq!(unstable.self_divergence_at, Some(2)); + let mut at_min = reference(); + at_min[4] = tok("X", 1.0); + let edge = check_reference(&sample(reference()), &sample(at_min), &POLICY); + assert!( + edge.stable, + "agreeing exactly to declared_min is enough: {edge:?}" + ); + assert_eq!(edge.self_divergence_at, Some(4)); let short = check_reference( &sample(reference()[..3].to_vec()), &sample(reference()), From ca9701a790bcf530eb358a60f166fc918514493a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 12:56:16 +0200 Subject: [PATCH 27/35] docs(contracts): serving-shape-parity-v1 records the Rust mutation runs (PMAT-4971) mutation_record said the Rust port had never been mutation-tested. It now carries the gx10 runs: g17 at 5227cda79c, 98 mutants, 95 caught, 1 missed, 2 unviable; g18 at ba959efa3e killed that one (5/5 on check_reference). The draft's Python record stays as python_draft. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d16d43d06b9cdd9a967aa4076a883305cad030c9) (cherry picked from commit 028efffb75742ef5eb922f9394630ca17f5a6180) (cherry picked from commit 8d7477f7f33afd438edcad901ae2d7af0368ef39) --- contracts/serving-shape-parity-v1.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contracts/serving-shape-parity-v1.yaml b/contracts/serving-shape-parity-v1.yaml index f7cc20a25d..63f9b2e184 100644 --- a/contracts/serving-shape-parity-v1.yaml +++ b/contracts/serving-shape-parity-v1.yaml @@ -257,11 +257,14 @@ findings: summary: 'The draft did not check S7 and had the receipt cite CB-004 beside the witness.' closed_by: 'Superseded by the bar: S7 is read from the same witness. The probe records each band''s per-request decode_tok_s from live streams and v3_shape reads it (FALSIFY-SSP-017).' mutation_record: - note: 'History: the draft''s record, measured on the Python checker. The Rust port has not been mutation-tested yet.' - date: '2026-10-04' - mutants: 32 - patched_self_test: {killed: 30, killed_by_crash: 4, survived: 2} - survived_table: [startswith-to-in, declared-min-default-64] + note: 'The Rust port, by cargo-mutants on gx10 (aarch64, CPU only) against the perf_gate unit tests. Scope: all of perf_gate/v3_shape.rs, plus live_decode_tok_s, band_decode_tok_s and check_reference in perf_gate/parity.rs. g16 at 8978becb22 was cut off by the 08:49Z reboot at 43 of 98 with 3 missed; 5227cda79c kills them. g17 at 5227cda79c left one, killed by ba959efa3e; g18 at ba959efa3e re-ran check_reference.' + date: '2026-10-10' + mutants: 98 + runs: + - {run: g17, commit: 5227cda79c, mutants: 98, caught: 95, missed: 1, unviable: 2, timeout: 0} + - {run: g18, commit: ba959efa3e, filter: check_reference, mutants: 5, caught: 5, missed: 0, unviable: 0, timeout: 0} + survived_table: [] + python_draft: {date: '2026-10-04', mutants: 32, killed: 30, killed_by_crash: 4, survived: 2, survived_table: [startswith-to-in, declared-min-default-64]} verification: > cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. At 72283efd5e (2026-10-10 From 6f97a1070ad06a98140dec9850c7b990592fba7d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sat, 10 Oct 2026 16:13:09 +0200 Subject: [PATCH 28/35] test(serve): logprobs_4971 asks ignore_eos, so the fixture generates tokens (PMAT-4971) The quantized fixture model's greedy first token is a stop token, so every reply in this module was empty: three per-token tests failed on completion_tokens == 0, at 4dbdb1371f before the recut as well (gx10 g22, 11 passed / 3 failed). With ignore_eos the module runs 14/14 on gx10 (g23, d16d43d06b plus this change). Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 05cdab9fb29450ccd9dfee8abdbc78ba93cfcefe) (cherry picked from commit 1f275af9265fe6b591ed884ee3218b01789c97bd) (cherry picked from commit 4c37a8e6b9078888c2ae274cf5f15862457dea98) --- crates/aprender-serve/src/api/tests/logprobs_4971.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/aprender-serve/src/api/tests/logprobs_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_4971.rs index 2dd81b8f2e..900d249ec1 100644 --- a/crates/aprender-serve/src/api/tests/logprobs_4971.rs +++ b/crates/aprender-serve/src/api/tests/logprobs_4971.rs @@ -17,9 +17,11 @@ use crate::api::create_router; const CHAT: &str = "/v1/chat/completions"; +/// `ignore_eos`: the fixture model's greedy first token is a stop token, so +/// without it every reply here is empty and the per-token checks see nothing. fn chat_body(extra: &str) -> String { format!( - r#"{{"model":"default","messages":[{{"role":"user","content":"Hi"}}],"max_tokens":4,"temperature":0{extra}}}"# + r#"{{"model":"default","messages":[{{"role":"user","content":"Hi"}}],"max_tokens":4,"temperature":0,"ignore_eos":true{extra}}}"# ) } From 6099a21e9e28a780ece3cf49b8448c213584187a Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 11 Oct 2026 06:56:34 +0200 Subject: [PATCH 29/35] fix(apr-cli): the model-path gate lists `llm ttft-verdict` beside shape-check and parity (PMAT-4971) The re-cut onto main brought #4954's TtftVerdict into LlmSubcommand, and V3's explicit match in validate.rs (no wildcard, so a new subcommand that names a model must be handled) did not list it: apr-cli failed to compile with E0004. ttft-verdict reads receipt directories, never a model, so it joins the arm that returns no paths. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 3ab7ebe679d02ef2d6e944a5486b6270224e1af1) (cherry picked from commit f811234be02d96dabb3aa96bd9b7eb0f8ab20d30) --- crates/apr-cli/src/validate.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/apr-cli/src/validate.rs b/crates/apr-cli/src/validate.rs index eca6c7a9e1..6cc6b14815 100644 --- a/crates/apr-cli/src/validate.rs +++ b/crates/apr-cli/src/validate.rs @@ -14,7 +14,8 @@ fn extract_extended_model_paths(command: &ExtendedCommands) -> Vec { vec![file.to_path_buf()] }, // `llm bench` measures an HTTP endpoint, not a local model file, - // and `llm shape-check` reads a perf041 witness JSON, not a model. + // `llm ttft-verdict` reads receipt directories (#4954) and + // `llm shape-check` reads a perf041 witness JSON, not a model. // `llm parity` drives an endpoint too: its `--model` is hashed // for the witness and never loaded, so the gate has nothing to // refuse. @@ -22,6 +23,7 @@ fn extract_extended_model_paths(command: &ExtendedCommands) -> Vec { // has to be handled here rather than absorbed by a wildcard. TestSubcommand::Llm { command } => match command { LlmSubcommand::Bench { .. } + | LlmSubcommand::TtftVerdict { .. } | LlmSubcommand::ShapeCheck { .. } | LlmSubcommand::Parity { .. } => vec![], }, From e102b9999baf09f18793703f94287dcd8978984e Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Sun, 11 Oct 2026 06:56:55 +0200 Subject: [PATCH 30/35] chore(contracts): ONT witness for serving-shape-parity-v1 on the re-cut (PMAT-4971) Regenerated by pv-sat on intel at 1f275af926 (the m2 re-cut onto main 72282b2e47); contracts lint_passes_on_real_contracts green on that tree. Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e38601b3c514b1e55d81c3a0af08d46f4cc2e5d6) (cherry picked from commit 7cba852523e56ebb9916ec23d345e7994d831ba3) --- ...1afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json index 7f0aa6a9fe..14b7fbf7c8 100644 --- a/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json +++ b/contracts/witness/f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67.json @@ -1,7 +1,7 @@ { - "census_id_set_sha256": "ced96e20e1de8700f4de7592803f602f279ccecc086cbb4fbd86b349b9028779", + "census_id_set_sha256": "04c86e59ff4920150a699521e5b23705cd4c75893e80e95ad4b6a5a68d5077bf", "relations_sha256": "f4a1686f98d541afc73cc61d63d1a9726ef28aaee70bad35dff472cef083bf67", - "reasoner_git_sha": "28006ac347c6c8ff337874e39711ea67dbfe8bcf", + "reasoner_git_sha": "1f275af9265fe6b591ed884ee3218b01789c97bd", "checkable_n": 84, "result": { "kind": "model", @@ -10,5 +10,5 @@ } }, "pc_reasoner": "fired", - "cpu_ms": 1 + "cpu_ms": 2 } From fd6f25fd834d2b7461e34d1635fba07e80eb9cd8 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 12:17:02 +0200 Subject: [PATCH 31/35] fix(serve): the APR CUDA chat route refuses logprobs instead of dropping them (PMAT-4971) handle_gpu_chat_completion generated through run_gpu_generation, which computes no logprobs, and answered 200 without them. It now calls logprobs_refusal_response before generating, as the APR CPU, WGPU and SafeTensors routes do. The handler is cuda-only, so the new test reads its shipping source and checks the refusal comes before generation. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 --- .../commands/serve/handler_gpu_completion.rs | 4 +++ crates/apr-cli/src/commands/serve/mod.rs | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 973e54454d..e31eaa6687 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -262,6 +262,10 @@ async fn handle_gpu_chat_completion( return err_response; } } + // #4971: run_gpu_generation computes no logprobs, so a request for them is refused. + if let Some(refused) = logprobs_refusal_response(&req, "APR CUDA (AprV2ModelCuda)") { + return refused; + } let messages = req.get("messages").and_then(|m| m.as_array()); let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false); diff --git a/crates/apr-cli/src/commands/serve/mod.rs b/crates/apr-cli/src/commands/serve/mod.rs index 4ec18077fa..3c6a977f0e 100644 --- a/crates/apr-cli/src/commands/serve/mod.rs +++ b/crates/apr-cli/src/commands/serve/mod.rs @@ -598,6 +598,38 @@ mod accelerator_guard_tests { ); } + /// #4971 (ASOC-INV-021): the APR CUDA chat route computes no logprobs, so a + /// request for them must be refused before generation, never answered 200 + /// without them. The handler is cuda-only and has no router test without a + /// GPU, so this reads its shipping source: comments dropped, `#[cfg(test)]` cut. + #[test] + fn the_cuda_chat_route_refuses_logprobs_before_it_generates() { + let whole = include_str!("handler_gpu_completion.rs"); + let shipping = whole.split("#[cfg(test)]").next().unwrap_or(whole); + let src: String = shipping + .lines() + .filter(|l| !l.trim_start().starts_with("//")) + .collect::>() + .join("\n"); + let start = src + .find("async fn handle_gpu_chat_completion(") + .expect("the APR CUDA chat handler is gone"); + let body = &src[start..]; + let body = &body[..body.find("\nfn ").unwrap_or(body.len())]; + let refuse = body.find(&format!("{}(&req,", "logprobs_refusal_response")); + let generate = body.find(&format!("{}(", "run_gpu_generation")); + match (refuse, generate) { + (Some(r), Some(g)) => assert!( + r < g, + "handle_gpu_chat_completion generates before it refuses logprobs" + ), + _ => panic!( + "handle_gpu_chat_completion does not refuse logprobs \ + (refusal {refuse:?}, generation {generate:?}): it answers 200 without them" + ), + } + } + #[test] fn the_resolver_is_actually_called_from_a_decision_path() { let src = include_str!("handler_gpu_completion.rs"); From 23e42db9afa06c5cf568bf9df935d7d827332986 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 12:17:19 +0200 Subject: [PATCH 32/35] fix(serve): a logprobs step stays graphed under BATCHED_GRAPH=1 (PMAT-4971) batched_step_token_ids sent a recording step down the eager path whenever BATCHED_GRAPH=1, so one batch mixed graphed and eager steps. The graph is captured against workspace.logits_buf (init_batched_graph_buffers sizes it exactly), and the replay synchronizes before its argmax, so the logits a recording step needs are already there. Both paths now pick their tokens, and a recording step downloads the logits afterwards. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 --- .../cuda/executor/layers/batched_forward.rs | 3 +- .../gguf/cuda/generate_batched_streaming.rs | 45 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs b/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs index 1f0211c79f..5505873c31 100644 --- a/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs +++ b/crates/aprender-serve/src/cuda/executor/layers/batched_forward.rs @@ -465,7 +465,8 @@ impl CudaExecutor { Self::download_logits_at(logits_ptr, len) } - /// #4971: the logits the last `forward_batched_to_token_ids` argmaxed (m × vocab_size, + /// #4971: the logits the last `forward_batched_to_token_ids` (or its graphed replay, + /// captured against the same buffer) argmaxed (m × vocab_size, /// slot s at s*vocab_size), downloaded after it, so a slot that records logprobs keeps /// the token the device argmax chose. `batched_gpu_argmax` only reads `logits_buf` and /// syncs the stream before it returns. diff --git a/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs b/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs index cf7fa7a9bc..13d5ef94ca 100644 --- a/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs +++ b/crates/aprender-serve/src/gguf/cuda/generate_batched_streaming.rs @@ -548,8 +548,10 @@ impl OwnedQuantizedModelCuda { /// embeddings and positions already in `state`. /// /// #4971: a live slot that records logprobs gets this step's record from the - /// same logits its token came from. The graphed path (BATCHED_GRAPH=1) - /// replays into its own buffers, so a recording step runs eager. + /// same logits its token came from. Both the eager and the graphed path + /// (BATCHED_GRAPH=1) leave this step's logits in `workspace.logits_buf`: the + /// graph is captured against that buffer (`init_batched_graph_buffers`), so a + /// recording step stays on the path the batch was already taking. fn batched_step_token_ids(&mut self, state: &mut BatchedDecodeState) -> Result> { // PMAT-056: Multi-stream root cause fixed (scatter moved to self.stream), // but graph replay still 25% slower than eager due to capture overhead. @@ -599,9 +601,8 @@ impl OwnedQuantizedModelCuda { } return Ok(token_ids); } - if use_graph && !recording { - return self - .executor + let token_ids = if use_graph { + self.executor .forward_batched_to_token_ids_graphed( &state.embed_buf, &state.pos_buf, @@ -614,23 +615,23 @@ impl OwnedQuantizedModelCuda { .map_err(|e| RealizarError::UnsupportedOperation { operation: "forward_batched_to_token_ids_graphed".to_string(), reason: format!("Batched forward failed: {e}"), - }); - } - let token_ids = self - .executor - .forward_batched_to_token_ids( - &state.embed_buf, - &state.pos_buf, - state.num_layers, - state.hidden_dim as u32, - state.intermediate_dim as u32, - state.vocab_size as u32, - state.eps, - ) - .map_err(|e| RealizarError::UnsupportedOperation { - operation: "forward_batched_to_token_ids".to_string(), - reason: format!("Batched forward failed: {e}"), - })?; + })? + } else { + self.executor + .forward_batched_to_token_ids( + &state.embed_buf, + &state.pos_buf, + state.num_layers, + state.hidden_dim as u32, + state.intermediate_dim as u32, + state.vocab_size as u32, + state.eps, + ) + .map_err(|e| RealizarError::UnsupportedOperation { + operation: "forward_batched_to_token_ids".to_string(), + reason: format!("Batched forward failed: {e}"), + })? + }; if recording { let logits = self .executor From fa0af16735fdfa53d37b900b6e5c268484d8f26c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 12:38:47 +0200 Subject: [PATCH 33/35] docs(contracts): serving-shape-parity-v1 cites no internal paths or names (PMAT-4971) The contract and the V3 module docs cited an internal review receipt path, working-session names, branch names and internal ruling IDs. They now say what was decided (adopted in review on 2026-10-10, bound to 61f0fa5798; the build is Rust only) without them. The CUDA logprobs test finds tinyllama under $HOME/.apr/models instead of two host-specific paths. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 --- contracts/apr-serve-openai-compat-v1.yaml | 4 ++-- contracts/serving-shape-parity-v1.yaml | 23 +++++++++---------- .../apr-cli/src/commands/test_llm_parity.rs | 2 +- .../src/api/tests/logprobs_cuda_4971.rs | 17 +++++++------- .../aprender-test-lib/src/perf_gate/parity.rs | 2 +- .../src/perf_gate/v3_shape.rs | 6 ++--- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/contracts/apr-serve-openai-compat-v1.yaml b/contracts/apr-serve-openai-compat-v1.yaml index dc25abcaba..3b8eaf7f8d 100644 --- a/contracts/apr-serve-openai-compat-v1.yaml +++ b/contracts/apr-serve-openai-compat-v1.yaml @@ -714,10 +714,10 @@ falsification_tests: if_fails: 'A V3 witness can only carry a margin someone typed in: a near-tie flip at c > 1 reads as NOT ADMISSIBLE, or a margin from the wrong stream or step (or a 0.0 for a step with no runner-up) makes a wrong-KV divergence read as a near-tie.' - id: FALSIFY-PERF041-PORT-4971 name: the_perf041_probe_decides_in_rust_as_the_script_does - prediction: 'perf_gate::parity ports the perf041 probe''s judgement (C301). The m=1 reference is stable only when its two decodes agree to declared_min; a c > 1 band whose log shows no batch of m >= 2 formed since the band''s offset is UNMEASURABLE, never PASS; a slot whose completion_tokens or logprob entries differ from n_predict makes its band UNMEASURABLE by name; a band FAILs when its slots part before declared_min or one emits a token max_constant_run times; and a run with no measured c > 1 band, or an unstable reference, is UNMEASURABLE. Each band records the m=1 reference''s top-2 margin at its divergence, so a clean run writes a witness the V3 shape check admits and a near-tie flip carries its margin. ProbeMatrix reads declared_min, max_constant_run, n_predict, the sampler and the ladder from perf-matrix.yaml and refuses, by name, a missing key, an empty ladder or a 0 in it, max_constant_run 0, and declared_min above n_predict. llm::parity fires the requests against a canned SSE server: identical streams with batches formed PASS every band, a box that never batches is UNMEASURABLE, frozen slots FAIL, a flip at a near-tie PASSes with its margin, and an untrustworthy reference ends the run with no bands. This is not a gate: no merge, queue or release job runs it.' + prediction: 'perf_gate::parity ports the perf041 probe''s judgement, in Rust. The m=1 reference is stable only when its two decodes agree to declared_min; a c > 1 band whose log shows no batch of m >= 2 formed since the band''s offset is UNMEASURABLE, never PASS; a slot whose completion_tokens or logprob entries differ from n_predict makes its band UNMEASURABLE by name; a band FAILs when its slots part before declared_min or one emits a token max_constant_run times; and a run with no measured c > 1 band, or an unstable reference, is UNMEASURABLE. Each band records the m=1 reference''s top-2 margin at its divergence, so a clean run writes a witness the V3 shape check admits and a near-tie flip carries its margin. ProbeMatrix reads declared_min, max_constant_run, n_predict, the sampler and the ladder from perf-matrix.yaml and refuses, by name, a missing key, an empty ladder or a 0 in it, max_constant_run 0, and declared_min above n_predict. llm::parity fires the requests against a canned SSE server: identical streams with batches formed PASS every band, a box that never batches is UNMEASURABLE, frozen slots FAIL, a flip at a near-tie PASSes with its margin, and an untrustworthy reference ends the run with no bands. This is not a gate: no merge, queue or release job runs it.' test_harness: 'cargo test -p aprender-test-lib --lib perf_gate::parity && cargo test -p aprender-test-lib --features llm --lib llm::parity' expected_output: "test result: ok" - if_fails: 'The V3 witness can only come from the Python script (C301), or the Rust probe reads a box that never batched, or a reference that disagrees with itself, as batch-invariant.' + if_fails: 'The V3 witness can only come from the Python script, or the Rust probe reads a box that never batched, or a reference that disagrees with itself, as batch-invariant.' - id: FALSIFY-PERF041-CLI-4971 name: apr_test_llm_parity_exits_by_the_verdict_and_writes_the_witness prediction: 'apr test llm parity runs the Rust perf041 probe against --url, reading --server-log, configured by the compiled-in perf-matrix.yaml (--ladder replaces only the ladder; 0 is refused by clap). It prints the verdict, the reference and one line per band, writes the witness to --witness before exiting (not --json, which is apr''s global flag: a parity arg with that id panicked on every run, and a parse through the whole apr parser holds the two apart), and exits 0 on PASS, 5 (ValidationFailed) on FAIL and 8 (InferenceFailed) on UNMEASURABLE, naming the first reason; FAIL and UNMEASURABLE never share a code. A server log, binary or model that is not a file exits 3 before any request. The witness records the model by file name only beside its sha256, the binary''s sha256, the host (flag, PERF041_HOST, then this host, else null) and the commit (flag, then PERF041_COMMIT, else null), and reads back whole. This is not a gate: no merge, queue or release job runs it.' diff --git a/contracts/serving-shape-parity-v1.yaml b/contracts/serving-shape-parity-v1.yaml index 63f9b2e184..659d2c2da8 100644 --- a/contracts/serving-shape-parity-v1.yaml +++ b/contracts/serving-shape-parity-v1.yaml @@ -1,9 +1,9 @@ # 0.71.1 exit criterion V3 (#4971): is a perf041 witness a serving-shape parity receipt? -# Moved here from the spec branch's draft (la-71/v3-serving-shape-parity @ adf6c4b4aa, +# Moved here from the spec branch's draft (adf6c4b4aa, # docs/lookahead/contracts-draft/serving-shape-parity-v1.yaml, version 0.3.0). The draft's # Python checker and fixtures are not carried: the check is perf_gate::v3_shape and its -# in-code case table, and `apr test llm shape-check` is its front door (C301). -# The bar below was adopted by the C314 quorum of 2026-10-10 06:48Z, recorded here before +# in-code case table, and `apr test llm shape-check` is its front door (the build is Rust only). +# The bar below was adopted in review on 2026-10-10 06:48Z, recorded here before # any of its code (order step 1). S7, the near-tie cap and the declared_min floor were # spec-before-code; shape-check reads them since order step 2 (FALSIFY-SSP-017..019). # This is not a gate: no merge, queue or release job runs the check. Wiring V3 into a @@ -11,7 +11,7 @@ metadata: version: 1.0.0 created: '2026-10-10' - author: la-0.71.1a (from the draft 0.3.0 by la-0.71, 2026-10-03) + author: 'aprender maintainers (from the draft 0.3.0 of 2026-10-03)' kind: schema status: partial description: > @@ -27,12 +27,11 @@ metadata: - crates/apr-cli/src/commands/test_llm_parity.rs - crates/aprender-test-lib/src/llm/parity.rs - 'contracts/apr-serve-openai-compat-v1.yaml FALSIFY-V3-SHAPE-CHECK-4971, FALSIFY-V3-SHAPE-CLI-4971, FALSIFY-PERF041-CLI-4971' - - 'Quorum receipt: /mnt/nvme-raid0/cop-inbox/handoff/quorum-la-0.71.1a-v3bar.md (C314, round 2, ADOPTED 2026-10-10 06:48Z)' - - 'Spec: docs/lookahead/0.71-v3-serving-shape-parity.md on branch la-71/v3-serving-shape-parity' + - 'Spec: docs/lookahead/0.71-v3-serving-shape-parity.md on the spec branch' depends_on: - continuous-batching-v1 bar: - adopted: '2026-10-10T06:48Z, C314 quorum round 2 (Explore sonnet, haiku, sonnet adversarial; degraded: same-family), bound to la-711a/4971-v3a-margin @ 61f0fa5798' + adopted: '2026-10-10T06:48Z, in review, bound to commit 61f0fa5798' statement: > V3 holds when, on each GPU host class 0.71.1 ships for (lambda RTX 4090 sm_89 and gx10 GB10), one `apr test llm parity` run against `apr serve` with CUDA on the blessed @@ -57,10 +56,10 @@ bar: - 'The rationale for the cap is recorded under equations.admissible.cap_rationale.' spec_before_code: [] # S7, near_tie_cap and declared_min_floor landed in order step 2 order: - - 'Contract: this file, passing pv validate (la-0.71.1a).' - - 'Code: the probe records per-request decode timing; shape-check reads S7, the cap and the declared_min floor (la-0.71.1a).' + - 'Contract: this file, passing pv validate.' + - 'Code: the probe records per-request decode timing; shape-check reads S7, the cap and the declared_min floor.' - 'PROPOSE-TICKET for the end-to-end plant.' - - 'A session other than la-0.71.1a runs both host classes and closes the row. The author never closes V3.' + - 'Someone other than the author runs both host classes and closes the row. The author never closes V3.' equations: admissible: formula: | @@ -79,7 +78,7 @@ equations: - 'an empty, blank, unreadable or bandless witness is RED, never NOT ADMISSIBLE' - 'a witness with no per-request decode timing at c=1 or c=4 is not admissible: S7 is unmeasured, never assumed' cap_rationale: > - Author's wording of quorum condition 6. A near-tie explains one early divergence as + Review condition 6, in the author's words. A near-tie explains one early divergence as chance: two candidates within ε of each other at that step, so any reduction-order change may flip the argmax. Bands c=4, 8 and 16 share one batched path, so when more than one of them diverges early the likelier cause is that path, not three @@ -267,7 +266,7 @@ mutation_record: python_draft: {date: '2026-10-04', mutants: 32, killed: 30, killed_by_crash: 4, survived: 2, survived_table: [startswith-to-in, declared-min-default-64]} verification: > cargo test -p aprender-test-lib --lib perf_gate::v3_shape and cargo test -p apr-cli - --lib test_llm_shape, on gx10 at la-711a/4971-v3a-margin. At 72283efd5e (2026-10-10 + --lib test_llm_shape, on gx10 at the V3 branch. At 72283efd5e (2026-10-10 08:11Z, gx10 aarch64, CPU only): perf_gate 251 passed, llm::parity 7 passed, apr-cli test_llm 89 passed, and FALSIFY-SSP-007 by hand gave exit 5 with its five reasons, S7 unmeasured among them. No GPU host class has run the probe yet. diff --git a/crates/apr-cli/src/commands/test_llm_parity.rs b/crates/apr-cli/src/commands/test_llm_parity.rs index 742be98fe7..558b358150 100644 --- a/crates/apr-cli/src/commands/test_llm_parity.rs +++ b/crates/apr-cli/src/commands/test_llm_parity.rs @@ -1,6 +1,6 @@ //! `apr test llm parity`, the front door of `apr_test::llm::parity` //! (#4971 V3-d): the perf041 batched-parity probe, ported from -//! `scripts/perf041_batched_parity_probe.py` (C301). +//! `scripts/perf041_batched_parity_probe.py`, so no Python runs it. //! //! It decodes one prompt twice alone (`m = 1`, the reference), then as `c` //! identical concurrent requests for each `c` of the ladder, reads the batch diff --git a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs index 5a4578988c..012d4c1c94 100644 --- a/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs +++ b/crates/aprender-serve/src/api/tests/logprobs_cuda_4971.rs @@ -20,20 +20,19 @@ use crate::api::{create_router, AppState}; const CHAT: &str = "/v1/chat/completions"; -const TINYLLAMA: [&str; 2] = [ - "/home/noah/.apr/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", - "/mnt/nvme-raid0/cache/apr-home/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", -]; +/// `$HOME/.apr/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf`. +fn tinyllama_path() -> Option { + let home = std::env::var_os("HOME")?; + let path = std::path::Path::new(&home) + .join(".apr/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"); + path.exists().then_some(path) +} /// tinyllama on CUDA device 0 behind the direct path, or `None` (and a SKIP /// line) when the model or the device is missing. fn cuda_state() -> Option { use crate::gguf::{MappedGGUFModel, OwnedQuantizedModel, OwnedQuantizedModelCuda}; - let Some(path) = TINYLLAMA - .iter() - .map(std::path::Path::new) - .find(|p| p.exists()) - else { + let Some(path) = tinyllama_path() else { eprintln!("SKIP: tinyllama not present"); return None; }; diff --git a/crates/aprender-test-lib/src/perf_gate/parity.rs b/crates/aprender-test-lib/src/perf_gate/parity.rs index 1c57b9c3d9..7ca7cfbf1d 100644 --- a/crates/aprender-test-lib/src/perf_gate/parity.rs +++ b/crates/aprender-test-lib/src/perf_gate/parity.rs @@ -4,7 +4,7 @@ //! (`m = 1`, the reference), then as `c` identical concurrent requests for each //! `c` of the ladder, and writes a witness whose bands the V3 shape check //! ([`super::v3_shape`]) reads. This module is that probe's judgement, ported -//! (C301): what makes a sample comparable, when the reference is stable, how a +//! to Rust: what makes a sample comparable, when the reference is stable, how a //! band is scored, and the run's verdict. Every function here is pure in what //! came back over the wire, so it compiles and is tested under default //! features. The requests themselves are fired behind the `llm` feature. diff --git a/crates/aprender-test-lib/src/perf_gate/v3_shape.rs b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs index 4dbfd08341..a04f6d296a 100644 --- a/crates/aprender-test-lib/src/perf_gate/v3_shape.rs +++ b/crates/aprender-test-lib/src/perf_gate/v3_shape.rs @@ -4,7 +4,7 @@ //! itself? A V3 receipt asks more of the same witness, and this module is the //! check. It ports the hand-run `v3_shape_check.py` from the spec branch //! (`docs/lookahead/0.71-v3-serving-shape-parity.md`, draft contract -//! `serving-shape-parity-v1`, branch `la-71/v3-serving-shape-parity`) and +//! `serving-shape-parity-v1`) and //! carries its case table. No Python ships with it. //! //! | Rule | The witness holds | @@ -19,7 +19,7 @@ //! | floor | every band's `declared_min` is at least 64, the perf matrix's `witness.min_agree_tokens` | //! | cap | at most one of c=4, 8, 16 is admitted through the near-tie clause | //! -//! S7, the floor and the cap are the V3 bar's (C314 round 2, contract +//! S7, the floor and the cap are the V3 bar's (adopted in review, contract //! `serving-shape-parity-v1`), as is ε = 0.05 logit, now fixed. A witness //! with no per-request decode timing at c=1 or c=4 is not admissible: S7 is //! unmeasured, never assumed. The probe times a request only on a stream the @@ -678,7 +678,7 @@ mod tests { check_v3_shape("{}"), ShapeVerdict::Red("RED: the witness measured no bands".to_string()) ); - let missing = Path::new("/nonexistent/la-0.71.1a/v3-shape/witness.json"); + let missing = Path::new("/nonexistent/v3-shape/witness.json"); let verdict = check_v3_shape_file(missing); assert_eq!(verdict.exit_code(), 2, "{verdict:?}"); assert!(verdict.reasons()[0].starts_with("RED: cannot read witness ")); From ed43b1e26d0d5b0dd075cdea76902232a61b266d Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 13:15:39 +0200 Subject: [PATCH 34/35] docs(audits): re-derive the chat SSE row after the logprobs change (PMAT-4971) openai_handlers.rs gained the logprobs stream path, so the POST /v1/chat/completions (stream:true) row now cites 919-989,1058-1240, and its stream:true branches at 1336 and 1435 (each line checked against the source it named before). Refs #4971 Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 --- docs/audits/surface_audit.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index 9e537d7804..437761eaf2 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -72,7 +72,7 @@ apr,POST /stream/generate,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/a apr,POST /tokenize,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:96,high apr,POST /v1/batch/completions,9,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:152,medium apr,POST /v1/chat/completions,4,x86_64-linux;nvidia-cuda,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:134,high -apr,"POST /v1/chat/completions (stream:true, SSE)",4,x86_64-linux;nvidia-cuda,llama.cpp,no,3,http-apr-serve,"crates/aprender-serve/src/api/openai_handlers.rs:878-948,1017-1151 (stream:true branches 1234,1332; cuda stream branches crates/aprender-serve/src/api/cuda_chat_backend.rs:147,441,602,755,1335,1366; ignore_eos fail-closed 16,561,712)",high +apr,"POST /v1/chat/completions (stream:true, SSE)",4,x86_64-linux;nvidia-cuda,llama.cpp,no,3,http-apr-serve,"crates/aprender-serve/src/api/openai_handlers.rs:919-989,1058-1240 (stream:true branches 1336,1435; cuda stream branches crates/aprender-serve/src/api/cuda_chat_backend.rs:147,441,602,755,1335,1366; ignore_eos fail-closed 16,561,712)",high apr,POST /v1/chat/completions/stream,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:139,high apr,POST /v1/completions,2,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:131,high apr,"POST /v1/completions (stream:true, SSE)",2,x86_64-linux,llama.cpp,no,3,http-apr-serve,crates/aprender-serve/src/api/gpu_completions_handler.rs:405 (SSE :340-378),high From db2f7299028667dab63fdaee078dcc4c3ca307a3 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Mon, 12 Oct 2026 15:18:35 +0200 Subject: [PATCH 35/35] style: format the Qwen3.5 chat backend fragment (#4971) qwen35_chat_backend.rs is include!d, so cargo fmt never reaches it or its tests module; both are now formatted with the crate's rustfmt.toml. No code change. Agent: la-0.71.1a Co-Authored-By: Claude Opus 5.5 --- crates/aprender-serve/src/api/qwen35_chat_backend.rs | 9 ++++++++- .../src/api/qwen35_chat_backend_tests.rs | 10 ++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend.rs b/crates/aprender-serve/src/api/qwen35_chat_backend.rs index 72bd3e2518..c9c5ee01f0 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend.rs @@ -308,7 +308,14 @@ async fn try_qwen35_backend( let tracer = crate::api::serve_trace::tracer_for(trace_level); let time_layers = trace_level == Some("layer"); let turn = tokio::task::spawn_blocking(move || { - qwen35_turn(&session, &input_ids, &gen_config, top_n, tracer, time_layers) + qwen35_turn( + &session, + &input_ids, + &gen_config, + top_n, + tracer, + time_layers, + ) }) .await; let reply = match turn { diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs index 7019710c74..fff26037cc 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs @@ -153,7 +153,10 @@ async fn a_chat_request_answers_what_apr_run_answers() { "serve must hand the model apr run's tokens: {body}" ); // #3719: the chat route says what this turn ran on; a --no-gpu session ran the CPU. - assert_eq!(json["used_gpu"], false, "the chat response reports its device: {body}"); + assert_eq!( + json["used_gpu"], false, + "the chat response reports its device: {body}" + ); } #[tokio::test(flavor = "multi_thread")] @@ -399,7 +402,10 @@ async fn gpu_a_chat_request_answers_from_the_gpu_session() { "the GPU session answers what apr run --gpu answers: {body}" ); // #3719: the response itself says the GPU ran it, so a client needs no /health probe. - assert_eq!(json["used_gpu"], true, "the chat response reports its device: {body}"); + assert_eq!( + json["used_gpu"], true, + "the chat response reports its device: {body}" + ); assert!( served.session.lock().expect("lock").on_gpu(), "no fallback during the request"