From 0590d5012f5f851e667837708013cbf76b19d680 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 17:14:36 +0200 Subject: [PATCH 01/68] fix(f2): key the F2 receipt on the executable's sha256, not the version string (#4290) f2_receipt::decide skipped the CPU reference forward whenever a stored receipt's {model_sha256, apr_version, device} matched. apr_version is CARGO_PKG_VERSION, so v0.69.3-rc.1, rc.2 (which rewrites the generation loop, #4263), the final tag and every dev build on release/0.69.3 all shared receipts: rc.2 would skip F2 on a receipt rc.1 wrote. The key gains build_id = "exe-sha256:" of the running executable (streamed, hashed once per process). Not the git sha: a dirty dev build shares its HEAD's sha and the release assets report +no-git. An executable that cannot be hashed gets an empty id, which never skips and is never written. Schema 1 -> 2; build_id is serde(default) so a schema-1 file parses and re-validates as a schema mismatch. Case table in f2_receipt_tests.rs: same version + different build -> BuildIdMismatch; two +no-git builds of one version -> BuildIdMismatch; identical build -> Skip; unhashable exe -> BuildUnidentified. Closes #4290 Co-Authored-By: Claude Opus 5.5 --- .../src/gguf/inference/forward/f2_receipt.rs | 99 ++++++++++++++++- .../inference/forward/f2_receipt_tests.rs | 103 +++++++++++++++++- .../gguf/inference/forward/forward_qwen35.rs | 11 +- 3 files changed, 205 insertions(+), 8 deletions(-) diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs index 788892cd6c..82aeace049 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -1,4 +1,4 @@ -//! The F2 hybrid guard's receipt: validate once per (model, apr version, +//! The F2 hybrid guard's receipt: validate once per (model, build, //! device), and let later runs read the answer instead of re-deriving it. //! //! # Why (#3604, operator ruling 2026-09-20) @@ -48,8 +48,19 @@ pub struct F2ReceiptKey { /// prefix or a size+mtime fingerprint would let a planted receipt with the /// wrong hash pass, which is the first falsifier. pub model_sha256: String, - /// The version of the crate that ran the guard. + /// The version of the crate that ran the guard. Informational: every + /// build that shares a version string shares it (rc.1, rc.2, the final + /// tag, every dev build), so it cannot be the build key (#4290). pub apr_version: String, + /// sha256 of the EXECUTABLE that ran the guard, `exe-sha256:`. This + /// is the build key (#4290): a git sha misses dirty dev builds and the + /// release assets carry `+no-git`, but two builds whose binaries differ + /// by a byte cannot share this. Empty means the executable could not be + /// hashed, and an empty build id never skips and is never written. + /// `default` so a schema-1 file (which has no such field) still parses + /// and re-validates as a SCHEMA mismatch instead of as unreadable. + #[serde(default)] + pub build_id: String, /// The device the GPU half ran on, as the driver names it. pub device: String, } @@ -70,7 +81,10 @@ pub struct F2Receipt { } /// The current receipt schema. Bump it and every old receipt re-validates. -pub const F2_RECEIPT_SCHEMA: u32 = 1; +/// +/// 2 (#4290): the key gained `build_id`; every schema-1 receipt was keyed on +/// the version string alone and re-validates. +pub const F2_RECEIPT_SCHEMA: u32 = 2; /// Why a run is validating instead of reading the receipt. #[derive(Debug, Clone, PartialEq, Eq)] @@ -100,6 +114,17 @@ pub enum F2ValidateReason { /// This crate's version. expected: String, }, + /// This run's executable could not be hashed, so no receipt can vouch + /// for it. + BuildUnidentified, + /// The receipt was written by a different build (the executable's + /// sha256 differs), even if the version string is the same. + BuildIdMismatch { + /// The build id in the file. + found: String, + /// This executable's build id. + expected: String, + }, /// The receipt was written for a different device. DeviceMismatch { /// The device in the file. @@ -128,6 +153,18 @@ impl std::fmt::Display for F2ValidateReason { Self::AprVersionMismatch { found, expected } => { write!(f, "receipt written by apr {found}, this is {expected}") }, + Self::BuildUnidentified => { + write!( + f, + "this executable could not be hashed, so no receipt applies" + ) + }, + Self::BuildIdMismatch { found, expected } => write!( + f, + "receipt written by build {}…, this build is {}…", + &found[..found.len().min(23)], + &expected[..expected.len().min(23)] + ), Self::DeviceMismatch { found, expected } => { write!(f, "receipt written for {found}, this device is {expected}") }, @@ -164,6 +201,11 @@ pub fn decide( if revalidate { return F2Decision::Validate(F2ValidateReason::Revalidate); } + // No build identity, no skip: a receipt cannot vouch for a binary whose + // bytes nobody could read (#4290). + if expected.build_id.is_empty() { + return F2Decision::Validate(F2ValidateReason::BuildUnidentified); + } let receipt = match found { Err(e) => return F2Decision::Validate(F2ValidateReason::Unreadable(e)), Ok(None) => return F2Decision::Validate(F2ValidateReason::NoReceipt), @@ -189,6 +231,12 @@ pub fn decide( expected: expected.apr_version.clone(), }); } + if receipt.key.build_id != expected.build_id { + return F2Decision::Validate(F2ValidateReason::BuildIdMismatch { + found: receipt.key.build_id, + expected: expected.build_id.clone(), + }); + } if receipt.key.device != expected.device { return F2Decision::Validate(F2ValidateReason::DeviceMismatch { found: receipt.key.device, @@ -310,6 +358,51 @@ pub fn apr_version() -> String { env!("CARGO_PKG_VERSION").to_string() } +/// The build key: sha256 of the running executable, `exe-sha256:`, or +/// an empty string when it cannot be read (#4290). Hashed once per process. +/// +/// Not the version string (rc.1, rc.2, the final tag and every dev build share +/// it) and not the git sha (a dirty dev build shares its HEAD's sha, and the +/// release assets report `+no-git`). The binary's own bytes are the only +/// identity no two differing builds can share. +#[must_use] +pub fn build_id() -> String { + static ID: std::sync::OnceLock = std::sync::OnceLock::new(); + ID.get_or_init(|| { + std::env::current_exe() + .ok() + .and_then(|p| file_sha256(&p).ok()) + .map_or_else(String::new, |h| format!("exe-sha256:{h}")) + }) + .clone() +} + +/// sha256 of a file's bytes, lower-case hex, streamed so a large binary is +/// never held in memory whole. +/// +/// # Errors +/// The file could not be opened or read. +pub fn file_sha256(path: &Path) -> std::io::Result { + use sha2::{Digest, Sha256}; + use std::io::Read as _; + let mut f = std::fs::File::open(path)?; + let mut h = Sha256::new(); + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = f.read(&mut buf)?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + let mut s = String::with_capacity(64); + for b in h.finalize() { + use std::fmt::Write as _; + let _ = write!(s, "{b:02x}"); + } + Ok(s) +} + /// Now, in unix seconds; 0 if the clock is before the epoch. #[must_use] pub fn unix_now() -> u64 { diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs index 448adc179b..1d76d83338 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt_tests.rs @@ -9,14 +9,15 @@ //! device. use super::{ - apr_version, decide, model_sha256, read_receipt, receipt_path, write_receipt, F2Decision, - F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, + apr_version, build_id, decide, file_sha256, model_sha256, read_receipt, receipt_path, + write_receipt, F2Decision, F2Receipt, F2ReceiptKey, F2ValidateReason, F2_RECEIPT_SCHEMA, }; fn key() -> F2ReceiptKey { F2ReceiptKey { model_sha256: "a".repeat(64), apr_version: "0.68.2".to_string(), + build_id: format!("exe-sha256:{}", "c".repeat(64)), device: "NVIDIA GeForce RTX 4090".to_string(), } } @@ -199,6 +200,7 @@ fn the_receipt_file_carries_all_three_keys_flat_so_a_human_can_read_them() { for needle in [ "\"model_sha256\"", "\"apr_version\"", + "\"build_id\"", "\"device\"", "\"schema\"", "\"positions_judged\"", @@ -258,3 +260,100 @@ fn two_writers_on_one_model_each_rename_a_whole_file() { } let _ = std::fs::remove_dir_all(&dir); } + +// ---------------------------------------------------------------- #4290 +// The build key. Case table: same version + different build -> Validate; +// a +no-git build (no sha to tell them apart) with a different exe hash -> +// Validate; the identical build -> Skip. Reverting `decide` to the +// version-only key turns the first two green-to-Skip; that mutant was run. + +#[test] +fn falsifier_same_version_different_build_revalidates_rc1_receipt_on_rc2() { + let k = key(); + let mut planted = receipt_for(&k); + // rc.1 wrote it: same model, same device, same "0.68.2" version string. + planted.key.build_id = format!("exe-sha256:{}", "d".repeat(64)); + match decide(Ok(Some(planted)), &k, false) { + F2Decision::Validate(F2ValidateReason::BuildIdMismatch { found, expected }) => { + assert_eq!(found, format!("exe-sha256:{}", "d".repeat(64))); + assert_eq!(expected, k.build_id); + }, + other => { + panic!("a same-version receipt from another build must re-validate, got {other:?}") + }, + } +} + +#[test] +fn falsifier_two_no_git_builds_of_one_version_do_not_share_a_receipt() { + // Release assets report `v0.69.3+no-git`: the version string and the + // commit string are identical across rc.1/rc.2/final. Only the bytes differ. + let mut k = key(); + k.apr_version = "0.69.3".to_string(); + let mut planted = receipt_for(&k); + planted.key.build_id = format!("exe-sha256:{}", "e".repeat(64)); + assert!(matches!( + decide(Ok(Some(planted)), &k, false), + F2Decision::Validate(F2ValidateReason::BuildIdMismatch { .. }) + )); +} + +#[test] +fn the_identical_build_skips() { + let k = key(); + let r = receipt_for(&k); + assert!(matches!( + decide(Ok(Some(r)), &k, false), + F2Decision::Skip { .. } + )); +} + +#[test] +fn an_unhashable_executable_never_skips_even_on_a_matching_receipt() { + let mut k = key(); + k.build_id = String::new(); + let r = receipt_for(&k); // a receipt that "matches" the empty id + assert_eq!( + decide(Ok(Some(r)), &k, false), + F2Decision::Validate(F2ValidateReason::BuildUnidentified) + ); +} + +#[test] +fn a_schema_1_receipt_without_build_id_parses_and_revalidates_as_schema() { + let json = format!( + r#"{{"schema":1,"model_sha256":"{}","apr_version":"0.68.2","device":"NVIDIA GeForce RTX 4090","validated_at":1,"positions_judged":64}}"#, + "a".repeat(64) + ); + let old: F2Receipt = serde_json::from_str(&json).expect("schema-1 receipt parses"); + assert_eq!( + decide(Ok(Some(old)), &key(), false), + F2Decision::Validate(F2ValidateReason::SchemaMismatch { + found: 1, + expected: F2_RECEIPT_SCHEMA + }) + ); +} + +#[test] +fn build_id_is_the_sha256_of_the_running_executable() { + let id = build_id(); + let exe = std::env::current_exe().expect("current_exe"); + let want = format!("exe-sha256:{}", file_sha256(&exe).expect("hash exe")); + assert_eq!(id, want); + assert_eq!(id.len(), "exe-sha256:".len() + 64); + // Not the version string, in any form. + assert!(!id.contains(env!("CARGO_PKG_VERSION"))); +} + +#[test] +fn file_sha256_streams_to_the_same_answer_as_the_in_memory_hash() { + let dir = std::env::temp_dir().join(format!("f2-4290-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("mkdir"); + let p = dir.join("blob"); + // Larger than the 1 MiB read buffer, so the loop runs more than once. + let bytes: Vec = (0..(3 << 20) + 17).map(|i| (i % 251) as u8).collect(); + std::fs::write(&p, &bytes).expect("write"); + assert_eq!(file_sha256(&p).expect("hash"), model_sha256(&bytes)); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs index ec870db618..179f0a20ad 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs @@ -1615,7 +1615,7 @@ fn f2_validate_qwen35_receipted( device_name: &str, ) -> F2Outcome { use crate::gguf::f2_receipt::{ - apr_version, decide, model_sha256, read_receipt, receipt_dir, receipt_path, + apr_version, build_id, decide, model_sha256, read_receipt, receipt_dir, receipt_path, revalidate_requested, unix_now, write_receipt, F2Decision, F2Receipt, F2ReceiptKey, F2_RECEIPT_SCHEMA, }; @@ -1624,6 +1624,7 @@ fn f2_validate_qwen35_receipted( let key = F2ReceiptKey { model_sha256: model_sha256(model_bytes), apr_version: apr_version(), + build_id: build_id(), device: device_name.to_string(), }; let sha256_ms = hash_start.elapsed().as_secs_f64() * 1000.0; @@ -1638,9 +1639,10 @@ fn f2_validate_qwen35_receipted( F2Decision::Skip { receipt } => { let age_s = unix_now().saturating_sub(receipt.validated_at); eprintln!( - "F2 guard: receipt matches (model sha256 {}…, apr {}, {}) — validated {}s ago on {} positions; CPU reference forward skipped [source=receipt, sha256 {:.0} ms]. `apr run --revalidate` forces a fresh run.", + "F2 guard: receipt matches (model sha256 {}…, apr {} build {}, {}) — validated {}s ago on {} positions; CPU reference forward skipped [source=receipt, sha256 {:.0} ms]. `apr run --revalidate` forces a fresh run.", &key.model_sha256[..12], key.apr_version, + &key.build_id[..key.build_id.len().min(23)], key.device, age_s, receipt.positions_judged, @@ -1665,7 +1667,7 @@ fn f2_validate_qwen35_receipted( let source = match verdict { F2Verdict::Accepted { positions_judged } => { - match path.as_deref() { + match path.as_deref().filter(|_| !key.build_id.is_empty()) { Some(p) => { let receipt = F2Receipt { schema: F2_RECEIPT_SCHEMA, @@ -1683,6 +1685,9 @@ fn f2_validate_qwen35_receipted( ), } } + None if key.build_id.is_empty() => eprintln!( + "F2 guard: passed in {validate_ms:.0} ms; this executable could not be hashed, so no receipt — every run validates." + ), None => eprintln!( "F2 guard: passed in {validate_ms:.0} ms; no cache directory (no HOME, XDG_CACHE_HOME or APR_F2_RECEIPT_DIR), so no receipt — every run validates." ), From 64b20f89c26247311ea098572ba82f4957f861c1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 17:43:49 +0200 Subject: [PATCH 02/68] perf(qwen35-cuda): forward_single reuses one device residual instead of allocating per token forward_single called GpuBuffer::from_host for every token: one device allocation plus a blocking copy on the hot decode path. The model now owns a hidden_dim residual buffer, allocated once at construction, and refills it with copy_from_host_async on the executor's execution stream. The buffer is taken, lent to forward_from_residual, and restored on every path, refusals included. Test: qwen35_cuda_residual_pointer_is_stable_across_tokens_and_refusals checks that the device pointer stays the same across 3 tokens, an OOV refusal and a position-limit refusal, with GPU argmax == CPU argmax on each token. Closes #4316 Refs #4249 Co-Authored-By: Claude Opus 5.5 --- .../src/cuda/executor/executor_api.rs | 8 +++ .../src/gguf/cuda/forward_qwen35_cuda.rs | 57 ++++++++++++++--- .../gguf/cuda/forward_qwen35_cuda_tests.rs | 64 +++++++++++++++++++ 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/crates/aprender-serve/src/cuda/executor/executor_api.rs b/crates/aprender-serve/src/cuda/executor/executor_api.rs index e835c5a4d9..ba6c0b27b4 100644 --- a/crates/aprender-serve/src/cuda/executor/executor_api.rs +++ b/crates/aprender-serve/src/cuda/executor/executor_api.rs @@ -482,6 +482,14 @@ impl CudaExecutor { &self.context } + /// The EXECUTION stream: the one every `*_into` kernel launcher and + /// [`Self::sync_stream`] use. A host-to-device copy enqueued here is ordered + /// in front of the next kernel with no host sync (#4316). + #[must_use] + pub fn execution_stream(&self) -> &CudaStream { + &self.stream + } + /// Get reference to compute stream (WAPR-PERF-014: reuse stream for KV scatter/attention) #[must_use] pub fn compute_stream(&self) -> &CudaStream { diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs index 8ff1f85732..ee1eac9853 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -257,6 +257,13 @@ pub struct Qwen35CudaModel<'a> { out_normed: GpuBuffer, /// `[vocab_size]`, the logits — the ONE buffer a token's forward downloads. logits_buf: GpuBuffer, + /// `[hidden_dim]`, the residual stream a token runs through every layer + /// (#4316). Allocated ONCE here and refilled per token by an in-stream + /// async copy of the embedding row, so its device pointer never changes: + /// a per-token `GpuBuffer::from_host` was a cuMemAlloc + blocking HtoD + + /// cuMemFree every token and a moving pointer no captured graph can + /// replay. `None` only while [`Self::forward_single`] has lent it out. + residual: Option>, dims: Qwen35CudaDims, /// Positions the device KV caches hold. max_seq_len: usize, @@ -589,6 +596,7 @@ impl<'a> Qwen35CudaModel<'a> { let attn_scratch = Self::build_attn_scratch(&executor, dims)?; let out_normed = Self::zeros(&executor, dims.hidden_dim as usize)?; let logits_buf = Self::zeros(&executor, dims.vocab_size as usize)?; + let residual = Self::zeros(&executor, dims.hidden_dim as usize)?; let state = Self::build_state(&executor, &layers, dims, max_seq_len)?; Ok(Self { model, @@ -601,6 +609,7 @@ impl<'a> Qwen35CudaModel<'a> { lm_head, out_normed, logits_buf, + residual: Some(residual), dims, max_seq_len, }) @@ -1408,15 +1417,47 @@ impl<'a> Qwen35CudaModel<'a> { }); } - let dev = GpuBuffer::from_host( - self.executor.context(), - &embedding[start..start + hidden_dim], - ) - .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + // Lend the persistent residual out for the token and put it back on + // EVERY path, so a failed token cannot cost the next one its buffer. + let mut dev = self + .residual + .take() + .ok_or_else(|| RealizarError::InvalidShape { + reason: "qwen35_cuda: the residual buffer is already lent out".to_string(), + })?; + let row = &embedding[start..start + hidden_dim]; + // SAFETY: `copy_from_host_async` requires the host slice to outlive the + // copy. `row` is the model's embedding table, borrowed immutably for + // `'a`, which outlives `self`. Every path out of the token either syncs + // this stream (below) or leaves the copy pending on a stream that + // `CudaExecutor`'s Drop synchronizes (context-wide) before `self`, and + // so the `'a` borrow, can end. + let copied = unsafe { dev.copy_from_host_async(row, self.executor.execution_stream()) } + .map_err(|e| gpu_err("qwen35_cuda_forward", &e)); + let out = copied.and_then(|()| self.forward_from_residual(state, &dev, position)); + self.residual = Some(dev); + out + } + + /// The device pointer of the persistent residual buffer — the address a + /// captured graph would bake in (#4316). `None` only mid-token. + #[must_use] + pub fn residual_device_ptr(&self) -> Option { + self.residual.as_ref().map(|b| b.as_ptr() as u64) + } + + /// Everything [`Self::forward_single`] does after the embedding row is in + /// `dev`: the layers, the output norm, the `lm_head`, the one sync. + fn forward_from_residual( + &mut self, + state: &mut Qwen35CudaState, + dev: &GpuBuffer, + position: usize, + ) -> Result> { for il in 0..self.layers.len() { match self.layers[il] { - CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, &dev)?, - CudaLayer::Attention(_) => self.attention_layer(state, il, &dev, position)?, + CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, dev)?, + CudaLayer::Attention(_) => self.attention_layer(state, il, dev, position)?, } } // The tail stays on the device (#3090 review). `hidden_to_logits` would @@ -1429,7 +1470,7 @@ impl<'a> Qwen35CudaModel<'a> { let d = self.dims; self.executor .rmsnorm_into( - &dev, + dev, &self.output_norm, &self.out_normed, d.hidden_dim, diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs index 2d1ea33116..1fbf9e3577 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs @@ -1711,3 +1711,67 @@ fn qwen35_cuda_refuses_only_value_heads_that_do_not_group() { Qwen35CudaModel::check_head_grouping(dims(0, 16, 128, 128)) .expect_err("zero key heads must be refused, not divided by"); } + +/// #4316: the residual stream is ONE device allocation for the model's life. +/// Its pointer is the same before the first token, after every token, and +/// after a refused token (out-of-vocabulary id, and a position past the KV +/// cache) — the property a captured CUDA graph needs, and the one a +/// per-token `GpuBuffer::from_host` broke. The next token after the refusals +/// must still produce the CPU's argmax. +#[test] +#[serial_test::serial] +fn qwen35_cuda_residual_pointer_is_stable_across_tokens_and_refusals() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + gpu.pin_reference_gemv(); + let mut gpu_state = gpu.new_state().expect("device state"); + let mut cpu_state = qwen.new_state(LONG_PROMPT.len() + 1); + + let p0 = gpu + .residual_device_ptr() + .expect("residual allocated at construction"); + assert_ne!(p0, 0, "a null residual pointer is not an allocation"); + + for (pos, &token) in LONG_PROMPT.iter().take(3).enumerate() { + let want = qwen + .forward_single_qwen35(token, &mut cpu_state, pos) + .expect("cpu forward"); + let got = gpu + .forward_single(token, &mut gpu_state, pos) + .expect("gpu forward"); + assert_eq!( + crate::gguf::ops::argmax(&got), + crate::gguf::ops::argmax(&want), + "pos {pos}: argmax" + ); + assert_eq!( + gpu.residual_device_ptr(), + Some(p0), + "pos {pos}: pointer moved" + ); + // One past the last row of the embedding table (= the logits width). + let bad_token = u32::try_from(got.len()).expect("vocab fits u32"); + + // A refused token, both kinds, between every real one. + assert!(gpu + .forward_single(bad_token, &mut gpu_state, pos + 1) + .is_err()); + assert_eq!( + gpu.residual_device_ptr(), + Some(p0), + "pos {pos}: an out-of-vocabulary refusal lost or moved the residual" + ); + let past = gpu_state.max_seq_len(); + assert!(gpu.forward_single(token, &mut gpu_state, past).is_err()); + assert_eq!( + gpu.residual_device_ptr(), + Some(p0), + "pos {pos}: a past-the-cache refusal lost or moved the residual" + ); + } +} From ca5907eb384d2efc88d38e1862b5ea618c1f8393 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 18:23:38 +0200 Subject: [PATCH 03/68] =?UTF-8?q?refactor(chat,serve):=20DELETE-DUP=20?= =?UTF-8?q?=E2=80=94=20three=20callerless=20dense=20paths=20(0.69.4=20port?= =?UTF-8?q?=20of=20rc.2=208be3565d2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These have no caller anywhere in the workspace on this tree either: - apr-cli ChatSession::generate_gguf (#[allow(dead_code)], ~90 lines): the bespoke OwnedQuantizedModel generate_with_cache loop. - apr-cli generate_and_print_fallback (no-inference cfg): unreachable!() stub with no caller. - aprender-serve gguf/inference/cached/sequence.rs: never included by any module, byte-identical to single_true_batched_owned.rs. No behaviour change. Verified: fmt, clippy -D warnings (apr-cli + aprender-serve lib), cargo test -p apr-cli --lib chat 263 pass. Refs #4266 Co-Authored-By: Claude Opus 5.5 --- .../src/commands/chat_generate_session.rs | 5 - .../src/commands/chat_generate_session_02.rs | 90 ------ .../src/gguf/inference/cached/sequence.rs | 302 ------------------ 3 files changed, 397 deletions(-) delete mode 100644 crates/aprender-serve/src/gguf/inference/cached/sequence.rs diff --git a/crates/apr-cli/src/commands/chat_generate_session.rs b/crates/apr-cli/src/commands/chat_generate_session.rs index c0a3b4f01f..91aa59577a 100644 --- a/crates/apr-cli/src/commands/chat_generate_session.rs +++ b/crates/apr-cli/src/commands/chat_generate_session.rs @@ -216,11 +216,6 @@ fn run_repl(path: &Path, config: &ChatConfig) -> Result<(), CliError> { Ok(()) } -/// Generate a response, update history, and print (fallback mode). -#[cfg(not(feature = "inference"))] -fn generate_and_print_fallback(_session: &mut ChatSession, _input: &str, _config: &ChatConfig) { - unreachable!("ChatSession::new always returns Err without inference feature") -} enum CommandResult { Continue, diff --git a/crates/apr-cli/src/commands/chat_generate_session_02.rs b/crates/apr-cli/src/commands/chat_generate_session_02.rs index 70d4fba7b4..00e554cdf5 100644 --- a/crates/apr-cli/src/commands/chat_generate_session_02.rs +++ b/crates/apr-cli/src/commands/chat_generate_session_02.rs @@ -447,94 +447,4 @@ impl ChatSession { None } - - #[allow(dead_code)] - fn generate_gguf(&self, prompt: &[u32], config: &ChatConfig) -> Result, String> { - use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel, QuantizedGenerateConfig}; - - // Use MappedGGUFModel -> OwnedQuantizedModel for proper attention - // This has RoPE position encoding, causal mask, and GQA support - let mapped = MappedGGUFModel::from_path(&self.model_path) - .map_err(|e| format!("Failed to mmap GGUF: {e}"))?; - - let model = OwnedQuantizedModel::from_mapped(&mapped) - .map_err(|e| format!("Failed to create GGUF model: {e}"))?; - - // With KV cache, we can generate more tokens efficiently - let practical_max = config.max_tokens; - - let is_gqa = model.config().num_kv_heads < model.config().num_heads; - let gqa_note = if is_gqa { - format!(" (GQA: {} kv_heads)", model.config().num_kv_heads) - } else { - String::new() - }; - - // C-06 (Meyer DbC): EOS from model config, not hardcoded. - let stop_tokens = model - .config() - .eos_token_id - .map(|id| vec![id]) - .unwrap_or_default(); - - let gen_config = QuantizedGenerateConfig { - max_tokens: practical_max, - temperature: config.temperature, - top_k: 40, - stop_tokens, - trace: config.trace, - ..Default::default() - }; - - // Try CUDA GPU path first (200+ tok/s target) - // Uses generate_gpu_resident which is the tested/working GPU path - #[cfg(feature = "cuda")] - if !config.force_cpu { - use realizar::gguf::OwnedQuantizedModelCuda; - if OwnedQuantizedModelCuda::is_available() { - // Print model info before attempting CUDA (model consumed on success) - let num_layers = model.config().num_layers; - - match OwnedQuantizedModelCuda::new(model, 0) { - Ok(mut cuda_model) => { - let gpu_name = cuda_model.device_name().to_string(); - let vram_mb = cuda_model.vram_mb(); - println!( - "{}", - format!( - "[GGUF CUDA: {} ({} MB VRAM), {} layers, {} tokens{}]", - gpu_name, vram_mb, num_layers, practical_max, gqa_note - ) - .bright_green() - ); - // Use generate_gpu_resident (tested working path) not generate_full_cuda_with_cache - return cuda_model - .generate_gpu_resident(prompt, &gen_config) - .map_err(|e| format!("CUDA generate failed: {e}")); - } - Err(e) => { - // REG-15 (#2971): a parity-gate failure is never a silent downgrade. `apr chat` - // has no forced-GPU flag until R-0b's `--backend`, so the request is unforced here. - let handled = crate::commands::parity_admission::on_cuda_load_error(&format!("{e}"), false)?; - if !handled { - println!("{}", format!("[CUDA init failed: {}, falling back to CPU]", e).yellow()); - } - // Re-create model for CPU fallback (model was consumed) - let model = OwnedQuantizedModel::from_mapped(&mapped) - .map_err(|e| format!("Failed to recreate model: {e}"))?; - - return model - .generate_with_cache(prompt, &gen_config) - .map_err(|e| format!("GGUF generate failed: {e}")); - } - } - } - } - - // CPU path with KV cache (12+ tok/s) - used when CUDA feature disabled or unavailable - // Use KV cache path for O(n) instead of O(n²) - model - .generate_with_cache(prompt, &gen_config) - .map_err(|e| format!("GGUF generate failed: {e}")) - } } diff --git a/crates/aprender-serve/src/gguf/inference/cached/sequence.rs b/crates/aprender-serve/src/gguf/inference/cached/sequence.rs deleted file mode 100644 index 7fc073d544..0000000000 --- a/crates/aprender-serve/src/gguf/inference/cached/sequence.rs +++ /dev/null @@ -1,302 +0,0 @@ - -#[cfg(feature = "gpu")] -impl OwnedQuantizedModelCached { - - /// True batched multi-head attention (IMP-118) - /// - /// Uses true batched GEMM for Q@K^T and weights@V operations, - /// processing all heads efficiently without per-head dispatch overhead. - /// - /// # Arguments - /// * `q` - Query tensor [num_heads, seq_len, head_dim] - /// * `k` - Key tensor [num_heads, seq_len, head_dim] - /// * `v` - Value tensor [num_heads, seq_len, head_dim] - /// * `seq_len` - Sequence length - /// * `num_heads` - Number of attention heads - /// * `head_dim` - Dimension per head - /// - /// # Returns - /// Output tensor [num_heads, seq_len, head_dim] - pub fn true_batched_multihead_attention( - &self, - q: &[f32], - k: &[f32], - v: &[f32], - seq_len: usize, - num_heads: usize, - head_dim: usize, - ) -> Result> { - let expected_size = num_heads * seq_len * head_dim; - if q.len() != expected_size { - return Err(RealizarError::InvalidShape { - reason: format!( - "Q size {} doesn't match num_heads={} * seq_len={} * head_dim={}", - q.len(), - num_heads, - seq_len, - head_dim - ), - }); - } - - let scale = 1.0 / (head_dim as f32).sqrt(); - - // Step 1: Transpose K to [num_heads, head_dim, seq_len] - let mut k_transposed = vec![0.0f32; num_heads * head_dim * seq_len]; - for h in 0..num_heads { - let head_offset = h * seq_len * head_dim; - let k_t_offset = h * head_dim * seq_len; - for pos in 0..seq_len { - for d in 0..head_dim { - k_transposed[k_t_offset + d * seq_len + pos] = - k[head_offset + pos * head_dim + d]; - } - } - } - - // Step 2: True batched Q @ K^T -> [num_heads, seq_len, seq_len] - let scores = - self.true_batched_gemm(q, &k_transposed, num_heads, seq_len, head_dim, seq_len)?; - - // Step 3: Scale and apply causal softmax - let mut scaled_scores = scores; - for s in &mut scaled_scores { - *s *= scale; - } - - // Apply causal mask and softmax per-head using trueno SIMD (IMP-305e) - let weights = self.batched_causal_softmax_trueno(&scaled_scores, num_heads, seq_len)?; - - // Step 4: True batched weights @ V -> [num_heads, seq_len, head_dim] - let attn_output = - self.true_batched_gemm(&weights, v, num_heads, seq_len, seq_len, head_dim)?; - - Ok(attn_output) - } - - /// GPU-accelerated fused causal attention (IMP-119) - /// - /// Uses GPU for long sequences where compute dominates transfer overhead. - /// Combines Q@K^T → softmax → @V using GPU matmul operations. - /// - /// # Arguments - /// * `q` - Query tensor [seq_len, head_dim] - /// * `k` - Key tensor [seq_len, head_dim] - /// * `v` - Value tensor [seq_len, head_dim] - /// * `seq_len` - Sequence length - /// * `head_dim` - Head dimension - /// * `scale` - Attention scale factor (typically 1/sqrt(head_dim)) - /// - /// # Returns - /// Output tensor [seq_len, head_dim] - pub fn gpu_fused_causal_attention( - &self, - q: &[f32], - k: &[f32], - v: &[f32], - seq_len: usize, - head_dim: usize, - scale: f32, - ) -> Result> { - // For GPU-accelerated fused attention, we use a strategy that balances - // GPU matmul benefits with avoiding large intermediate allocations - // - // Strategy: - // 1. Use GPU for Q@K^T (benefits from parallelism) - // 2. Apply causal mask + softmax on CPU (memory-efficient) - // 3. Use GPU for attention_weights @ V - - let mut scheduler = self.get_scheduler()?; - - // Step 1: Transpose K to [head_dim, seq_len] - let mut k_transposed = vec![0.0f32; head_dim * seq_len]; - for pos in 0..seq_len { - for d in 0..head_dim { - k_transposed[d * seq_len + pos] = k[pos * head_dim + d]; - } - } - - // Step 2: GPU Q @ K^T -> [seq_len, seq_len] - let scores = scheduler - .matmul(q, &k_transposed, seq_len, head_dim, seq_len) - .map_err(|e| RealizarError::UnsupportedOperation { - operation: "gpu_fused_causal_attention Q@K^T".to_string(), - reason: format!("GPU matmul failed: {}", e), - })?; - - // Step 3: Scale and apply causal softmax (CPU - memory efficient) - let mut weights = vec![0.0f32; seq_len * seq_len]; - for i in 0..seq_len { - // Find max for numerical stability - let mut max_val = f32::NEG_INFINITY; - for j in 0..=i { - let score = scores[i * seq_len + j] * scale; - if score > max_val { - max_val = score; - } - } - - // Compute softmax with causal mask - let mut sum = 0.0f32; - for j in 0..=i { - let score = scores[i * seq_len + j] * scale; - weights[i * seq_len + j] = (score - max_val).exp(); - sum += weights[i * seq_len + j]; - } - - // Normalize - if sum > 0.0 { - for j in 0..=i { - weights[i * seq_len + j] /= sum; - } - } - // j > i remain zero (causal mask) - } - - // Step 4: GPU attention_weights @ V -> [seq_len, head_dim] - let output = scheduler - .matmul(&weights, v, seq_len, seq_len, head_dim) - .map_err(|e| RealizarError::UnsupportedOperation { - operation: "gpu_fused_causal_attention weights@V".to_string(), - reason: format!("GPU matmul failed: {}", e), - })?; - - Ok(output) - } - - /// GPU-accelerated fused multi-head attention (IMP-119) - /// - /// Processes all heads using GPU acceleration for long sequences. - /// - /// # Arguments - /// * `q` - Query tensor [seq_len, hidden_dim] - /// * `k` - Key tensor [seq_len, hidden_dim] - /// * `v` - Value tensor [seq_len, hidden_dim] - /// * `seq_len` - Sequence length - /// - /// # Returns - /// Output tensor [seq_len, hidden_dim] - pub fn gpu_fused_multihead_attention( - &self, - q: &[f32], - k: &[f32], - v: &[f32], - seq_len: usize, - ) -> Result> { - let num_heads = self.model.config.num_heads; - // GH-479: Use config methods (Qwen3 head_dim != hidden/heads) - let head_dim = self.model.config.head_dim(); - let q_dim = num_heads * head_dim; - let scale = 1.0 / (head_dim as f32).sqrt(); - - // Reshape Q, K, V to [num_heads, seq_len, head_dim] - let q_reshaped = self - .model - .reshape_for_parallel_heads(q, seq_len, num_heads, head_dim)?; - let k_reshaped = self - .model - .reshape_for_parallel_heads(k, seq_len, num_heads, head_dim)?; - let v_reshaped = self - .model - .reshape_for_parallel_heads(v, seq_len, num_heads, head_dim)?; - - // Process each head with GPU-accelerated fused attention - let mut attn_output = vec![0.0f32; num_heads * seq_len * head_dim]; - - for h in 0..num_heads { - let head_offset = h * seq_len * head_dim; - let q_head = &q_reshaped[head_offset..head_offset + seq_len * head_dim]; - let k_head = &k_reshaped[head_offset..head_offset + seq_len * head_dim]; - let v_head = &v_reshaped[head_offset..head_offset + seq_len * head_dim]; - - // GPU fused attention for this head - let head_output = - self.gpu_fused_causal_attention(q_head, k_head, v_head, seq_len, head_dim, scale)?; - - attn_output[head_offset..head_offset + seq_len * head_dim] - .copy_from_slice(&head_output); - } - - // Reshape back to [seq_len, q_dim] - let mut output = vec![0.0f32; seq_len * q_dim]; - for h in 0..num_heads { - let head_start = h * seq_len * head_dim; - for pos in 0..seq_len { - let src_start = head_start + pos * head_dim; - let dst_start = pos * q_dim + h * head_dim; - output[dst_start..dst_start + head_dim] - .copy_from_slice(&attn_output[src_start..src_start + head_dim]); - } - } - - Ok(output) - } - - /// Adaptive fused attention with CPU/GPU dispatch (IMP-119) - /// - /// Automatically selects CPU or GPU based on sequence length. - /// - Short sequences (< threshold): Use CPU fused attention (lower overhead) - /// - Long sequences (>= threshold): Use GPU fused attention (better throughput) - /// - /// # Arguments - /// * `q` - Query tensor [seq_len, head_dim] - /// * `k` - Key tensor [seq_len, head_dim] - /// * `v` - Value tensor [seq_len, head_dim] - /// * `seq_len` - Sequence length - /// * `head_dim` - Head dimension - /// * `scale` - Attention scale factor - /// - /// # Returns - /// Output tensor [seq_len, head_dim] - pub fn adaptive_fused_attention( - &self, - q: &[f32], - k: &[f32], - v: &[f32], - seq_len: usize, - head_dim: usize, - scale: f32, - ) -> Result> { - // Threshold based on empirical analysis from IMP-108 and IMP-115: - // - GPU dispatch overhead is ~300ms per HybridScheduler init (cached: ~0ms) - // - CPU fused attention is ~50µs for seq_len=64 - // - GPU wins when compute volume justifies transfer overhead - // - // With scheduler caching (IMP-112), the crossover is much lower - const GPU_SEQ_LEN_THRESHOLD: usize = 64; - - if seq_len >= GPU_SEQ_LEN_THRESHOLD { - // Long sequence: Use GPU for better throughput - self.gpu_fused_causal_attention(q, k, v, seq_len, head_dim, scale) - } else { - // Short sequence: Use CPU to avoid any overhead - self.fused_causal_attention(q, k, v, seq_len, head_dim, scale) - } - } - - /// Generate tokens with adaptive attention (IMP-121) - /// - /// Uses adaptive attention that automatically selects CPU or GPU - /// based on sequence length for optimal performance. - /// - /// # Arguments - /// * `prompt` - Input token IDs - /// * `config` - Generation configuration - /// - /// # Returns - /// Generated token sequence including prompt - pub fn generate_with_adaptive_attention( - &self, - prompt: &[u32], - config: &QuantizedGenerateConfig, - ) -> Result> { - // Delegate to generate_with_cache which uses efficient KV cache. - // Adaptive attention (IMP-122) is tracked separately for long-context prefill optimization. - // Current implementation handles typical inference workloads efficiently. - self.model.generate_with_cache(prompt, config) - } -} - -include!("scheduler.rs"); -include!("attention.rs"); -include!("single_flattened_batched_owned.rs"); From 9e1db133cdd94da955e086657578d9c8d4a05929 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 18:29:00 +0200 Subject: [PATCH 04/68] =?UTF-8?q?refactor(serve):=20DELETE-DUP=20=E2=80=94?= =?UTF-8?q?=20two=20test-only=20generate=20loops=20and=20the=20tests=20tha?= =?UTF-8?q?t=20only=20fed=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate_with_smallvec (forward/batched.rs) and generate_with_adaptive_attention (cached/single_true_batched_owned.rs) had no caller outside their own tests on this tree: no src caller, no example, no contract/doc/script reference. Deleted with the 6 tests that existed only to call them (batch_tests.rs x4, imp_115a 117f, imp_121a 121c). No behaviour change. Verified: fmt, clippy -p aprender-serve --lib -D warnings, cargo test -p aprender-serve --lib imp_115a imp_121a batch_tests 81 pass. (clippy --tests is red on this tree in untouched files — falsification_crux_c_34.rs, cancel_scope_2376.rs — none in the edited files.) Refs #4266 Co-Authored-By: Claude Opus 5.5 --- .../cached/single_true_batched_owned.rs | 21 ----- .../src/gguf/inference/forward/batch_tests.rs | 54 ------------ .../src/gguf/inference/forward/batched.rs | 85 ------------------- .../aprender-serve/src/gguf/tests/imp_115a.rs | 56 ------------ .../aprender-serve/src/gguf/tests/imp_121a.rs | 48 ----------- 5 files changed, 264 deletions(-) diff --git a/crates/aprender-serve/src/gguf/inference/cached/single_true_batched_owned.rs b/crates/aprender-serve/src/gguf/inference/cached/single_true_batched_owned.rs index 7fc073d544..f028fa7a98 100644 --- a/crates/aprender-serve/src/gguf/inference/cached/single_true_batched_owned.rs +++ b/crates/aprender-serve/src/gguf/inference/cached/single_true_batched_owned.rs @@ -274,27 +274,6 @@ impl OwnedQuantizedModelCached { } } - /// Generate tokens with adaptive attention (IMP-121) - /// - /// Uses adaptive attention that automatically selects CPU or GPU - /// based on sequence length for optimal performance. - /// - /// # Arguments - /// * `prompt` - Input token IDs - /// * `config` - Generation configuration - /// - /// # Returns - /// Generated token sequence including prompt - pub fn generate_with_adaptive_attention( - &self, - prompt: &[u32], - config: &QuantizedGenerateConfig, - ) -> Result> { - // Delegate to generate_with_cache which uses efficient KV cache. - // Adaptive attention (IMP-122) is tracked separately for long-context prefill optimization. - // Current implementation handles typical inference workloads efficiently. - self.model.generate_with_cache(prompt, config) - } } include!("scheduler.rs"); diff --git a/crates/aprender-serve/src/gguf/inference/forward/batch_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/batch_tests.rs index 1e09cc3dc2..96bc714fd7 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/batch_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/batch_tests.rs @@ -34,60 +34,6 @@ fn test_config() -> GGUFConfig { } } -// ============================================================================ -// generate_with_smallvec tests -// ============================================================================ - -#[test] -fn test_generate_with_smallvec_empty_prompt_error() { - let config = test_config(); - let model = create_test_model_with_config(&config); - let gen_config = QuantizedGenerateConfig::deterministic(5); - - let result = model.generate_with_smallvec(&[], &gen_config); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(format!("{:?}", err).contains("empty")); -} - -#[test] -fn test_generate_with_smallvec_greedy_sampling() { - let config = test_config(); - let model = create_test_model_with_config(&config); - let gen_config = QuantizedGenerateConfig::deterministic(3); - - let result = model.generate_with_smallvec(&[1], &gen_config); - assert!(result.is_ok()); - let tokens = result.unwrap(); - // Should have at least the prompt token - assert!(!tokens.is_empty()); - assert_eq!(tokens[0], 1); -} - -#[test] -fn test_generate_with_smallvec_temperature_sampling() { - let config = test_config(); - let model = create_test_model_with_config(&config); - let gen_config = QuantizedGenerateConfig::default() - .with_max_tokens(2) - .with_temperature(1.0) - .with_top_k(5); - - let result = model.generate_with_smallvec(&[1], &gen_config); - assert!(result.is_ok()); -} - -#[test] -fn test_generate_with_smallvec_stop_token() { - let config = test_config(); - let model = create_test_model_with_config(&config); - // Use token 0 as stop token (likely to be generated from zero weights) - let gen_config = QuantizedGenerateConfig::deterministic(10).with_stop_tokens(vec![0]); - - let result = model.generate_with_smallvec(&[1], &gen_config); - assert!(result.is_ok()); -} - // ============================================================================ // batch_generate tests // ============================================================================ diff --git a/crates/aprender-serve/src/gguf/inference/forward/batched.rs b/crates/aprender-serve/src/gguf/inference/forward/batched.rs index c8844d7340..75ed71fd70 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/batched.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/batched.rs @@ -377,89 +377,4 @@ impl OwnedQuantizedModel { Ok(tokens) } - /// Generate tokens with SmallVec optimization (IMP-117) - /// - /// Uses SmallVec for token storage to avoid heap allocations when: - /// - Prompt + max_tokens <= TOKEN_BUFFER_INLINE_CAP - /// - /// # Arguments - /// * `prompt` - Input token buffer (can be SmallVec or slice) - /// * `config` - Generation configuration - /// - /// # Returns - /// Generated token sequence as TokenBuffer (SmallVec) - /// - /// # Errors - /// Returns error if forward pass fails - pub fn generate_with_smallvec( - &self, - prompt: &[u32], - config: &QuantizedGenerateConfig, - ) -> Result { - if prompt.is_empty() { - return Err(RealizarError::InvalidShape { - reason: "Prompt cannot be empty".to_string(), - }); - } - - let max_seq_len = prompt.len() + config.max_tokens; - let mut cache = OwnedQuantizedKVCache::from_config(&self.config, max_seq_len); - - // Use SmallVec for token storage - inline for small sequences - let mut tokens: TokenBuffer = TokenBuffer::from_slice(prompt); - - // Process prompt tokens (prefill) - for (pos, &token_id) in prompt.iter().enumerate() { - let _ = self.forward_single_with_cache(token_id, &mut cache, pos)?; - } - - // Generate new tokens - for gen_idx in 0..config.max_tokens { - // aprender#2376(3): CANCELLATION POLL. The HTTP client may be gone; - // stop here instead of burning a core to max_tokens for nobody. - if config.cancel.is_cancelled() { - break; - } - let position = prompt.len() + gen_idx; - let last_token = *tokens.last().ok_or_else(|| RealizarError::InvalidShape { - reason: "Token buffer empty during generation".to_string(), - })?; - - let mut logits = self.forward_single_with_cache(last_token, &mut cache, position)?; - - // PMAT-814: apply repetition penalty in place over the recent context - // BEFORE both greedy argmax and sampling (no-op when repeat_penalty == 1.0). - crate::gguf::OwnedQuantizedModel::apply_repeat_penalty( - &mut logits, - &tokens, - config.repeat_penalty, - config.repeat_last_n, - ); - - // Sample next token - let next_token = if config.temperature == 0.0 || config.top_k == 1 { - ops::argmax(&logits) - } else { - crate::gguf::OwnedQuantizedModel::sample_topk( - &logits, - config.temperature, - config.top_k, - ) - }; - - // Check stop condition - if config.stop_tokens.contains(&next_token) { - break; - } - - tokens.push(next_token); - - // Check max length - if tokens.len() >= max_seq_len { - break; - } - } - - Ok(tokens) - } } diff --git a/crates/aprender-serve/src/gguf/tests/imp_115a.rs b/crates/aprender-serve/src/gguf/tests/imp_115a.rs index deaa8821b4..c469aa04dc 100644 --- a/crates/aprender-serve/src/gguf/tests/imp_115a.rs +++ b/crates/aprender-serve/src/gguf/tests/imp_115a.rs @@ -392,61 +392,5 @@ fn test_imp_117e_token_buffer_from_slice() { assert!(!buffer.spilled(), "IMP-117e: Small slice should not spill"); } -#[test] -fn test_imp_117f_generate_with_token_buffer() { - // IMP-117f: Test generate_with_smallvec returns correct SmallVec type - use crate::gguf::{TokenBuffer, TOKEN_BUFFER_INLINE_CAP}; - - let config = GGUFConfig { - architecture: "test".to_string(), - constraints: crate::gguf::ArchConstraints::from_architecture("test"), - hidden_dim: 64, - intermediate_dim: 128, - num_layers: 1, - num_heads: 4, - num_kv_heads: 4, - vocab_size: 100, - context_length: 256, - rope_theta: 10000.0, - eps: 1e-5, - rope_type: 0, - explicit_head_dim: None, - query_pre_attn_scalar: None, - bos_token_id: None, - eos_token_id: None, - }; - - let model = create_test_model_with_config(&config); - - // Test with small prompt that fits in inline capacity - let prompt: TokenBuffer = TokenBuffer::from_slice(&[1, 2, 3, 4, 5]); - assert!( - prompt.len() < TOKEN_BUFFER_INLINE_CAP, - "IMP-117f: Test prompt should be within inline capacity" - ); - - // Generate tokens using the SmallVec-based API - let gen_config = QuantizedGenerateConfig { - max_tokens: 10, - temperature: 0.0, - top_k: 1, - stop_tokens: Vec::new(), - trace: false, - ..Default::default() - }; - - let result = model.generate_with_smallvec(&prompt, &gen_config); - assert!( - result.is_ok(), - "IMP-117f: generate_with_smallvec should succeed" - ); - - let generated = result.expect("generation should succeed"); - assert!( - generated.len() > prompt.len(), - "IMP-117f: Generated tokens should include prompt + new tokens" - ); -} - include!("imp_118a.rs"); include!("imp_119c.rs"); diff --git a/crates/aprender-serve/src/gguf/tests/imp_121a.rs b/crates/aprender-serve/src/gguf/tests/imp_121a.rs index 65dbe661d7..a33df44cd9 100644 --- a/crates/aprender-serve/src/gguf/tests/imp_121a.rs +++ b/crates/aprender-serve/src/gguf/tests/imp_121a.rs @@ -120,54 +120,6 @@ fn test_imp_121b_cached_sync_adaptive_multihead() { ); } -#[test] -#[cfg(feature = "gpu")] -#[serial_test::serial] -fn test_imp_121c_generate_with_adaptive_attention() { - // IMP-121c: Cached model should have generate_with_adaptive_attention - let config = GGUFConfig { - architecture: "test".to_string(), - constraints: crate::gguf::ArchConstraints::from_architecture("test"), - hidden_dim: 64, - intermediate_dim: 128, - num_layers: 1, - num_heads: 4, - num_kv_heads: 4, - vocab_size: 100, - context_length: 256, - rope_theta: 10000.0, - eps: 1e-5, - rope_type: 0, - explicit_head_dim: None, - query_pre_attn_scalar: None, - bos_token_id: None, - eos_token_id: None, - }; - - let model = create_test_model_with_config(&config); - let cached_model = OwnedQuantizedModelCached::new(model); - - let prompt = vec![1u32, 2, 3, 4, 5]; - let gen_config = QuantizedGenerateConfig { - max_tokens: 5, - temperature: 0.0, - top_k: 1, - stop_tokens: Vec::new(), - trace: false, - ..Default::default() - }; - - // Generate with adaptive attention (should use CPU for short prompts) - let result = cached_model - .generate_with_adaptive_attention(&prompt, &gen_config) - .expect("generate_with_adaptive_attention should succeed"); - - assert!( - result.len() > prompt.len(), - "IMP-121c: Generated output should include new tokens" - ); -} - #[test] #[cfg(feature = "gpu")] #[serial_test::serial] From 39590bc633c83d85f45ccebb2434255f979b15b2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 18:47:28 +0200 Subject: [PATCH 05/68] evidence(0.69.4): Qwen3.5-9B temp-0 refs at 850/4k/32k (#4261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.69.3 G1 showed 9B SKIPPED. These are the refs owed before 0.69.4 acceptance: apr 0.69.3-rc.1 (7ff50ec2a) temp-0 ids for the pinned 9B GGUF (sha256 03b74727…) on the G1 prompts, every cell ran=gpu fell_back=false. - rc == base 0.69.1 id-for-id at 850 and 4k. Base at 32k was stopped (per-token prefill, ~40 min GPU hold, rc.2 acceptance queued behind it). - llama.cpp on the GGUF's own template (thinking off, same prompt-token counts 1006/4133/31848): identical at 32k; diverges at 850 pos 13 and 4k pos 2, both measured near-ties (top-3 within 0.08 nats; 0.117 nats, apr's pick is llama's #2) - margins.json. Refs #4261, #4249 Co-Authored-By: Claude Opus 5.5 --- docs/roadmaps/entries/PMAT-4261.yaml | 18 ++ docs/roadmaps/roadmap.yaml | 18 ++ .../release/0.69.4/refs-qwen35-9b/README.md | 40 +++ .../0.69.4/refs-qwen35-9b/build_refs.py | 67 +++++ .../refs-qwen35-9b/harness/margin-only.sh | 11 + .../0.69.4/refs-qwen35-9b/harness/margin.py | 16 ++ .../refs-qwen35-9b/harness/oracle-only.sh | 11 + .../0.69.4/refs-qwen35-9b/harness/oracle.py | 16 ++ .../0.69.4/refs-qwen35-9b/harness/run.sh | 25 ++ .../0.69.4/refs-qwen35-9b/margins.json | 62 +++++ .../release/0.69.4/refs-qwen35-9b/refs.json | 256 ++++++++++++++++++ 11 files changed, 540 insertions(+) create mode 100644 docs/roadmaps/entries/PMAT-4261.yaml create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/README.md create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/build_refs.py create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/harness/margin.py create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/harness/oracle.py create mode 100755 evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/margins.json create mode 100644 evidence/release/0.69.4/refs-qwen35-9b/refs.json diff --git a/docs/roadmaps/entries/PMAT-4261.yaml b/docs/roadmaps/entries/PMAT-4261.yaml new file mode 100644 index 0000000000..92be0f553a --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4261.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4261 + github_issue: 4261 + item_type: task + title: 'Qwen3.5-9B has no temp-0 refs: 0.69.3 G1 shows 9B SKIPPED; refs owed before 0.69.4 acceptance' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T16:47:18Z + updated: 2026-09-24T16:47:18Z + spec: null + acceptance_criteria: + - 'Temp-0 refs for Qwen3.5-9B-Q4_K_M (sha256 03b74727…) at 850/4k/32k exist and are receipted: apr rc ids per prompt, compared to base 0.69.1 and to llama.cpp on the official template, every divergence measured as a near-tie or reported.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:docs + notes: 'evidence/release/0.69.4/refs-qwen35-9b/README.md' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 90671942ca..08a3c10e6c 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -20610,3 +20610,21 @@ roadmap: labels: - kind:code notes: null +- id: PMAT-4261 + github_issue: 4261 + item_type: task + title: 'Qwen3.5-9B has no temp-0 refs: 0.69.3 G1 shows 9B SKIPPED; refs owed before 0.69.4 acceptance' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T16:47:18Z + updated: 2026-09-24T16:47:18Z + spec: null + acceptance_criteria: + - 'Temp-0 refs for Qwen3.5-9B-Q4_K_M (sha256 03b74727…) at 850/4k/32k exist and are receipted: apr rc ids per prompt, compared to base 0.69.1 and to llama.cpp on the official template, every divergence measured as a near-tie or reported.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:docs + notes: 'evidence/release/0.69.4/refs-qwen35-9b/README.md' diff --git a/evidence/release/0.69.4/refs-qwen35-9b/README.md b/evidence/release/0.69.4/refs-qwen35-9b/README.md new file mode 100644 index 0000000000..d971a7662d --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/README.md @@ -0,0 +1,40 @@ +# Qwen3.5-9B temp-0 refs at 850 / 4k / 32k (#4261, 0.69.4 epic #4249) + +0.69.3 G1 showed 9B as SKIPPED (cop scope cut: 4B only). This directory is the 9B reference set owed +before 0.69.4 acceptance. + +## Inputs (all pinned) +- Model: `Qwen3.5-9B-Q4_K_M.gguf`, sha256 `03b74727a860a56338e042c4420bb3f04b2fec5734175f4cb9fa853daf52b7e8` (the sha in #4261). +- Prompts: the 0.69.3 G1 prompts (f5-g1), sha256 in `refs.json` per prompt (`p850` 85bfde6e…, `p4k` ac2b35c4…, `p32k` 722b514d…). +- apr rc: `apr 0.69.3 (7ff50ec2a)` = v0.69.3-rc.1, sha256 `9a04c8cc…632fe` (the same binary as the 4B G1 receipt). +- apr base: `apr 0.69.1 (eed4a959a)`, sha256 `301336320a942572…`. +- Oracle: llama.cpp `llama-server` build 10987 (d1d3c3396), `--jinja`, the GGUF's own chat template with + `enable_thinking: false`, prompt ids fed to `/completion` as ids, `temperature 0, top_k 1`. +- Every apr cell: `apr run -i

.txt --chat --temperature 0 -n 64 --json --backend cuda`, one gpu-q hold per cell, RTX 4090 (lambda). + Every cell's envelope is `backend {requested: gpu, ran: gpu, fell_back: false}`. `nvidia-smi --query-compute-apps` was empty inside every hold. + +## Result (`python3 build_refs.py ` prints this; rc=0) +| prompt | prompt tokens | apr rc ids | rc vs base 0.69.1 | rc vs llama.cpp | rc prefill trace | +|---|---|---|---|---|---| +| p850 | 1006 | 23 (stop) | IDENTICAL (23 ids) | agree on first 13 ids | 1006 tok / 492 ms / 2045 tok/s | +| p4k | 4133 | 64 (length) | IDENTICAL (64 ids) | agree on first 2 ids | 4133 tok / 2228 ms / 1855 tok/s | +| p32k | 31848 | 16 (stop) | not run (see below) | IDENTICAL (16 ids) | 31848 tok / 20074 ms / 1587 tok/s | + +- **Template equivalence:** llama.cpp's official template gives the same prompt-token count as apr on all three prompts + (1006 / 4133 / 31848). The first oracle pass used llama-server's default `enable_thinking: true` and was 2 tokens short + (`\n` vs apr's `\n\n\n\n`). It is not used here. +- **The two divergences are near-ties, measured, not assumed** (`margins.json`, llama.cpp top-5 logprobs at the divergence, the shared prefix fed as ids): + - p850 pos 13: `;` −1.132, ` or` −1.202 (apr's pick), `,` −1.214 (llama's pick in the full run). The top 3 are within 0.08 nats. + llama.cpp itself picked a different one of those three between its full run and this probe. + - p4k pos 2: ` \`` −0.738 (llama), ` is` −0.855 (apr's pick). That is 0.117 nats, and apr's token is llama's #2. + - Both continuations are equivalent rewordings of the same answer (see `refs.json` `apr_rc_text`). p32k agrees id-for-id. +- **base 0.69.1 at p32k was stopped by me after 527 s** (exit 143). 0.69.1 prefills one token at a time, so 31848 tokens would + have held the GPU for ~40 min, and aprender-98's 0.69.3 rc.2 acceptance cell was queued behind it. rc == base holds + at 850 and 4k. At 32k the ref is the llama.cpp oracle, which agrees id-for-id. + +## Files +- `refs.json`: per prompt, the apr rc ids and text, the llama.cpp ids, the prompt sha256, and the first-divergence index. +- `margins.json`: the near-tie measurement above. +- `build_refs.py`: builds `refs.json` and the table from a harness `out/` directory. A cell that did not run is written as missing, never as a match. +- `harness/`: `run.sh` (all cells; its llama stage ran with the thinking-on oracle and was re-run), `oracle-only.sh` + `oracle.py` + (the thinking-off oracle used here), `margin-only.sh` + `margin.py`. The paths are the lambda scratch dir `/mnt/nvme-raid0/tmp/g1-9b`. diff --git a/evidence/release/0.69.4/refs-qwen35-9b/build_refs.py b/evidence/release/0.69.4/refs-qwen35-9b/build_refs.py new file mode 100644 index 0000000000..c0c76f60c9 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/build_refs.py @@ -0,0 +1,67 @@ +"""#4261: build the Qwen3.5-9B temp-0 refs (refs.json) and the receipt table from one harness run. + +Usage: python3 build_refs.py . Reads rc-9b-

.json / base-9b-

.json (apr run --json) +and llama-9b-

.json (llama-server on the GGUF's own template). Every cell carries its own ids, so a +cell that did not run is written as missing, never as a match. +""" +import hashlib, json, os, re, sys + +OUT = sys.argv[1] +PROMPTS = ("p850", "p4k", "p32k") +TRACE = re.compile(r"\[qwen35\] batched prefill: (\d+) tokens in (\d+) ms \((\d+) tok/s") + + +def load(key): + path = f"{OUT}/{key}.json" + if not os.path.exists(path) or os.path.getsize(path) == 0: + return None + d = json.load(open(path)) + toks = d["tokens"] + d["tokens"] = json.loads(toks) if isinstance(toks, str) else toks + err = f"{OUT}/{key}.err" + d["_err"] = open(err).read() if os.path.exists(err) else "" + return d + + +def first_diff(a, b): + return next((i for i, (x, y) in enumerate(zip(a, b)) if x != y), None if len(a) == len(b) else min(len(a), len(b))) + + +refs, rows, bad = {}, [], 0 +for p in PROMPTS: + rc, base, ll = load(f"rc-9b-{p}"), load(f"base-9b-{p}"), load(f"llama-9b-{p}") + cell = {"prompt_sha256": hashlib.sha256(open(f"{OUT}/../{p}.txt", "rb").read()).hexdigest()} + if rc is None: + bad += 1 + rows.append(f"| {p} | - | **MISSING** | | | |") + refs[p] = cell + continue + assert rc["backend"]["ran"] == "gpu" and not rc["backend"]["fell_back"], f"{p}: rc did not run on GPU" + t = TRACE.search(rc["_err"]) + cell.update(prompt_tokens=rc["prompt_tokens"], apr_rc_ids=rc["tokens"], apr_rc_text=rc["text"], + apr_rc_finish=rc["finish_reason"]) + if base is None: + vb = "not run" + else: + d = first_diff(rc["tokens"], base["tokens"]) + vb = f"IDENTICAL ({len(rc['tokens'])} ids)" if d is None else f"**DIFFER** at {d}" + bad += d is not None + cell["apr_base_ids_equal"] = d is None + if ll is None: + vl = "not run" + else: + d = first_diff(rc["tokens"], ll["tokens"]) + same_prompt = ll["prompt_tokens"] == rc["prompt_tokens"] + vl = (f"IDENTICAL ({len(rc['tokens'])} ids)" if d is None else f"agree on first {d} ids") + \ + ("" if same_prompt else f"; prompt tokens {ll['prompt_tokens']} vs apr {rc['prompt_tokens']}") + cell.update(llama_ids=ll["tokens"], llama_prompt_tokens=ll["prompt_tokens"], llama_first_diff=d) + refs[p] = cell + tr = f"{t.group(1)} tok / {t.group(2)} ms / {t.group(3)} tok/s" if t else "**NO TRACE LINE**" + rows.append(f"| {p} | {rc['prompt_tokens']} | {len(rc['tokens'])} ({rc['finish_reason']}) | {vb} | {vl} | {tr} |") + +json.dump(refs, open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "refs.json"), "w"), indent=1) +print("| prompt | prompt tokens | apr rc ids | rc vs base 0.69.1 | rc vs llama.cpp | rc prefill trace |") +print("|---|---|---|---|---|---|") +print("\n".join(rows)) +print("\nrc/base regressions:", bad) +sys.exit(1 if bad else 0) diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh new file mode 100644 index 0000000000..f5ebf993ed --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -u +cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 +M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +gpu-q --prio 1 -- bash -c ' + nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/llama.smi-inlock 2>&1 + llama-server -m '"$M"' -ngl 99 -c 40960 --jinja --port 18261 -np 1 > out/llama-server-margin.log 2>&1 & sp=$! + for _ in $(seq 1 120); do curl -sf http://127.0.0.1:18261/health >/dev/null && break; sleep 1; done + python3 margin.py > out/margin.log 2>&1; orc=$? + kill "$sp"; wait "$sp"; exit $orc' +echo "margin rc=$?" >> out/runs.log diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/margin.py b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin.py new file mode 100644 index 0000000000..5f059498e8 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin.py @@ -0,0 +1,16 @@ +"""#4261: llama.cpp top-5 probs at the first apr/llama divergence, fed the shared prefix as ids.""" +import json, urllib.request +U = "http://127.0.0.1:18261" +def post(path, body): + r = urllib.request.Request(U + path, json.dumps(body).encode(), {"Content-Type": "application/json"}) + return json.load(urllib.request.urlopen(r, timeout=600)) +res = {} +for p, i in (("p850", 13), ("p4k", 2)): + l = json.load(open(f"out/llama-9b-{p}.json")) + ids = l["prompt_ids"] + l["tokens"][:i] + c = post("/completion", {"prompt": ids, "n_predict": 1, "temperature": 0, "top_k": 1, "n_probs": 5, + "cache_prompt": False, "post_sampling_probs": False}) + top = c["completion_probabilities"][0]["top_logprobs"] + res[p] = {"pos": i, "top": [(t["id"], t["token"], t["logprob"]) for t in top]} + print(p, res[p], flush=True) +json.dump(res, open("out/margins.json", "w"), indent=1) diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh new file mode 100644 index 0000000000..34f478bf69 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -u +cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 +M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +gpu-q --prio 1 -- bash -c ' + nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/llama.smi-inlock 2>&1 + llama-server -m '"$M"' -ngl 99 -c 40960 --jinja --port 18261 -np 1 > out/llama-server.log 2>&1 & sp=$! + for _ in $(seq 1 120); do curl -sf http://127.0.0.1:18261/health >/dev/null && break; sleep 1; done + python3 oracle.py p850 p4k p32k > out/oracle.log 2>&1; orc=$? + kill "$sp"; wait "$sp"; exit $orc' +echo "llama rc=$?" >> out/runs.log diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle.py b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle.py new file mode 100644 index 0000000000..e5abe1c5cd --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle.py @@ -0,0 +1,16 @@ +"""llama.cpp temp-0 oracle for #4261: official GGUF template -> ids -> /completion, keyed per prompt.""" +import json, sys, urllib.request +U = "http://127.0.0.1:18261" +def post(path, body): + r = urllib.request.Request(U + path, json.dumps(body).encode(), {"Content-Type": "application/json"}) + return json.load(urllib.request.urlopen(r, timeout=2400)) +for p in sys.argv[1:]: + text = open(f"{p}.txt").read() + rendered = post("/apply-template", {"messages": [{"role": "user", "content": text}], "chat_template_kwargs": {"enable_thinking": False}})["prompt"] + ids = post("/tokenize", {"content": rendered, "add_special": False, "parse_special": True})["tokens"] + c = post("/completion", {"prompt": ids, "n_predict": 64, "temperature": 0, "top_k": 1, + "cache_prompt": False, "return_tokens": True, "seed": 0}) + json.dump({"prompt": p, "rendered_tail": rendered[-120:], "prompt_ids": ids, "prompt_tokens": len(ids), + "tokens": c["tokens"], "text": c["content"], "timings": c.get("timings")}, + open(f"out/llama-9b-{p}.json", "w")) + print(p, len(ids), len(c["tokens"]), flush=True) diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh new file mode 100755 index 0000000000..79eb4ba602 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# #4261: Qwen3.5-9B temp-0 refs at 850/4k/32k. One gpu-q hold per cell; outputs keyed bin-model-prompt. +set -u +export TMPDIR=/mnt/nvme-raid0/tmp/g1-9b/tmp +mkdir -p "$TMPDIR" +cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 +B=/mnt/nvme-raid0/agent-wt/f5-bins +M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +declare -A BIN=([rc]=$B/apr-0.69.3-rc-7ff50ec2a [base]=$B/apr-0.69.1-base-eed4a959a) +one() { + local k="$1-9b-$2" + [ -s "out/$k.json" ] && return 0 + local t0; t0=$(date +%s) + gpu-q --prio 1 -- bash -c "nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/$k.smi-inlock 2>&1; exec timeout 2400 ${BIN[$1]} run $M -i $2.txt --chat --temperature 0 -n 64 --json --backend cuda" > "out/$k.json" 2> "out/$k.err" + echo "$k rc=$? wall=$(( $(date +%s) - t0 ))" >> out/runs.log +} +for p in p850 p4k p32k; do one rc "$p"; one base "$p"; done +gpu-q --prio 1 -- bash -c ' + nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/llama.smi-inlock 2>&1 + llama-server -m '"$M"' -ngl 99 -c 40960 --jinja --port 18261 -np 1 > out/llama-server.log 2>&1 & sp=$! + for _ in $(seq 1 120); do curl -sf http://127.0.0.1:18261/health >/dev/null && break; sleep 1; done + python3 oracle.py p850 p4k p32k > out/oracle.log 2>&1; orc=$? + kill "$sp"; wait "$sp"; exit $orc' +echo "llama rc=$?" >> out/runs.log +echo DONE >> out/runs.log diff --git a/evidence/release/0.69.4/refs-qwen35-9b/margins.json b/evidence/release/0.69.4/refs-qwen35-9b/margins.json new file mode 100644 index 0000000000..61ef764fd2 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/margins.json @@ -0,0 +1,62 @@ +{ + "p850": { + "pos": 13, + "top": [ + [ + 26, + ";", + -1.1317181587219238 + ], + [ + 466, + " or", + -1.201866626739502 + ], + [ + 11, + ",", + -1.2139896154403687 + ], + [ + 13, + ".", + -2.8791298866271973 + ], + [ + 310, + " to", + -4.0089192390441895 + ] + ] + }, + "p4k": { + "pos": 2, + "top": [ + [ + 1510, + " `", + -0.7380622029304504 + ], + [ + 369, + " is", + -0.8553221821784973 + ], + [ + 6725, + " named", + -3.7118380069732666 + ], + [ + 421, + " that", + -3.843252420425415 + ], + [ + 3294, + " above", + -3.900766611099243 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/release/0.69.4/refs-qwen35-9b/refs.json b/evidence/release/0.69.4/refs-qwen35-9b/refs.json new file mode 100644 index 0000000000..14f7919cc5 --- /dev/null +++ b/evidence/release/0.69.4/refs-qwen35-9b/refs.json @@ -0,0 +1,256 @@ +{ + "p850": { + "prompt_sha256": "85bfde6ec906cd917b6d3227945350cde7942537367b93265c159d283ffbfb22", + "prompt_tokens": 1006, + "apr_rc_ids": [ + 760, + 3766, + 1414, + 369, + 264, + 1970, + 41643, + 321, + 1503, + 524, + 6435, + 264, + 3768, + 466, + 264, + 1118, + 1500, + 314, + 1970, + 310, + 12278, + 13, + 248046 + ], + "apr_rc_text": "The provided text is a code snippet and does not contain a filename or a first line of code to quote.", + "apr_rc_finish": "stop", + "apr_base_ids_equal": true, + "llama_ids": [ + 760, + 3766, + 1414, + 369, + 264, + 1970, + 41643, + 321, + 1503, + 524, + 6435, + 264, + 3768, + 11, + 748, + 424, + 369, + 11656, + 310, + 803, + 279, + 999, + 466, + 12278, + 1141, + 1118, + 1500, + 314, + 1970, + 2652, + 52907, + 13, + 248046 + ], + "llama_prompt_tokens": 1006, + "llama_first_diff": 13 + }, + "p4k": { + "prompt_sha256": "ac2b35c46b17cc4bf93fc387cdfae336b81c55b545c2fbe57fdf37bed06156ba", + "prompt_tokens": 4133, + "apr_rc_ids": [ + 760, + 999, + 369, + 1510, + 4920, + 942, + 23893, + 7092, + 12, + 12882, + 13050, + 4691, + 70, + 1644, + 2805, + 7800, + 14, + 13048, + 8705, + 16451, + 18, + 20, + 60593, + 41900, + 473, + 24822, + 63, + 321, + 1141, + 1118, + 1500, + 369, + 1510, + 13257, + 5670, + 802, + 12, + 18, + 20, + 24, + 21, + 28133, + 18, + 20, + 24, + 21, + 1590, + 6937, + 290, + 318, + 25148, + 290, + 8, + 829, + 7320, + 364, + 279, + 1167, + 16451, + 18, + 13, + 20, + 24197, + 383 + ], + "apr_rc_text": "The file is `crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_prefill.rs` and its first line is `//! PMAT-3596 (#3596): batched (chunked) prefill for the Qwen3.5 hybrid on", + "apr_rc_finish": "length", + "apr_base_ids_equal": true, + "llama_ids": [ + 760, + 999, + 1510, + 4920, + 942, + 23893, + 7092, + 12, + 12882, + 13050, + 4691, + 70, + 1644, + 2805, + 7800, + 14, + 13048, + 8705, + 16451, + 18, + 20, + 60593, + 41900, + 473, + 24822, + 63, + 18091, + 279, + 709, + 1510, + 25148, + 10630, + 5305, + 7561, + 321, + 1141, + 1118, + 1500, + 314, + 1970, + 369, + 1510, + 13257, + 5670, + 802, + 12, + 18, + 20, + 24, + 21, + 28133, + 18, + 20, + 24, + 21, + 1590, + 6937, + 290, + 318, + 25148, + 290, + 8, + 829, + 7320 + ], + "llama_prompt_tokens": 4133, + "llama_first_diff": 2 + }, + "p32k": { + "prompt_sha256": "722b514d484274745a95126143a5550b854b64777a8785309800e95eb7529f83", + "prompt_tokens": 31848, + "apr_rc_ids": [ + 2665, + 999, + 303, + 279, + 3766, + 1414, + 18091, + 279, + 32671, + 709, + 1510, + 25148, + 10630, + 5305, + 27653, + 248046 + ], + "apr_rc_text": "No file in the provided text defines the Rust function `chunk_rows_for`.", + "apr_rc_finish": "stop", + "llama_ids": [ + 2665, + 999, + 303, + 279, + 3766, + 1414, + 18091, + 279, + 32671, + 709, + 1510, + 25148, + 10630, + 5305, + 27653, + 248046 + ], + "llama_prompt_tokens": 31848, + "llama_first_diff": null + } +} \ No newline at end of file From 9e7225663fb1556485ff489b8273c72f89a40605 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 21:54:19 +0200 Subject: [PATCH 06/68] fix(f2): split decide's key comparison into key_mismatch (complexity ratchet); name run.sh's wall-clock DET002 (#4290, #4261) guard-tree RED on #4318: f2_receipt.rs::decide cyclomatic 9 / cognitive 26, new vs the comparand. guard-cargo RED: bashrs DET002 on the #4261 harness's `date +%s`, whose wall time IS the measurement. Co-Authored-By: Claude Opus 5.5 --- .../src/gguf/inference/forward/f2_receipt.rs | 36 ++++++++++++------- .../0.69.4/refs-qwen35-9b/harness/run.sh | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs index 82aeace049..b4b737aa88 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/f2_receipt.rs @@ -219,31 +219,41 @@ pub fn decide( } // Each key is compared and NAMED on its own. A combined "keys differ" would // hide which of the three moved, and the three falsifiers are one per key. - if receipt.key.model_sha256 != expected.model_sha256 { - return F2Decision::Validate(F2ValidateReason::ModelSha256Mismatch { - found: receipt.key.model_sha256, + match key_mismatch(&receipt.key, expected) { + Some(reason) => F2Decision::Validate(reason), + None => F2Decision::Skip { receipt }, + } +} + +/// The first key of `found` that differs from `expected`, as the reason that +/// names it; `None` when all four agree. Split out of [`decide`] so each key's +/// comparison reads on its own line (complexity ratchet, #4290). +fn key_mismatch(found: &F2ReceiptKey, expected: &F2ReceiptKey) -> Option { + if found.model_sha256 != expected.model_sha256 { + return Some(F2ValidateReason::ModelSha256Mismatch { + found: found.model_sha256.clone(), expected: expected.model_sha256.clone(), }); } - if receipt.key.apr_version != expected.apr_version { - return F2Decision::Validate(F2ValidateReason::AprVersionMismatch { - found: receipt.key.apr_version, + if found.apr_version != expected.apr_version { + return Some(F2ValidateReason::AprVersionMismatch { + found: found.apr_version.clone(), expected: expected.apr_version.clone(), }); } - if receipt.key.build_id != expected.build_id { - return F2Decision::Validate(F2ValidateReason::BuildIdMismatch { - found: receipt.key.build_id, + if found.build_id != expected.build_id { + return Some(F2ValidateReason::BuildIdMismatch { + found: found.build_id.clone(), expected: expected.build_id.clone(), }); } - if receipt.key.device != expected.device { - return F2Decision::Validate(F2ValidateReason::DeviceMismatch { - found: receipt.key.device, + if found.device != expected.device { + return Some(F2ValidateReason::DeviceMismatch { + found: found.device.clone(), expected: expected.device.clone(), }); } - F2Decision::Skip { receipt } + None } /// Where receipts live. `APR_F2_RECEIPT_DIR` wins (tests and operators pin it), diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh index 79eb4ba602..90213bad14 100755 --- a/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh @@ -10,7 +10,7 @@ declare -A BIN=([rc]=$B/apr-0.69.3-rc-7ff50ec2a [base]=$B/apr-0.69.1-base-eed4a9 one() { local k="$1-9b-$2" [ -s "out/$k.json" ] && return 0 - local t0; t0=$(date +%s) + local t0; t0=$(date +%s) # bashrs disable-line=DET002 (wall-clock timing IS the measurement) gpu-q --prio 1 -- bash -c "nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/$k.smi-inlock 2>&1; exec timeout 2400 ${BIN[$1]} run $M -i $2.txt --chat --temperature 0 -n 64 --json --backend cuda" > "out/$k.json" 2> "out/$k.err" echo "$k rc=$? wall=$(( $(date +%s) - t0 ))" >> out/runs.log } From 95224f4cb4c4b70f581319911e520bf32209c5c1 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 14:29:15 +0200 Subject: [PATCH 07/68] =?UTF-8?q?fix(serve):=20forward-port=20#4278=20to?= =?UTF-8?q?=20the=200.69.4=20train=20=E2=80=94=20APR=20CPU=20serve=20stops?= =?UTF-8?q?=20at=20EOS/<|im=5Fend|>=20and=20honours=20top=5Fp=20(#4265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4278 (d65a915be) landed on release/0.69.3 only. main, batch/0.70.0 and batch/0.69.4-epic-4249 still built the APR CPU GenerateConfig with stop_tokens: vec![] and top_p: 0.9 (handler_apr_cpu_completion.rs:139,213). Generation therefore ran to max_tokens and ignored the request's top_p. This cherry-pick resolves three conflicts against the newer base: - apr_cpu_generate_config drops `seed`. The #3760 field is release-only; GenerateConfig on this base has no seed and already carries `cancel`. - build_cpu_sse_stream keeps this base's 3-arg signature. The 4th arg is release-only. top_p is still threaded into spawn_cpu_streaming_task. - roadmap.yaml is re-aggregated from the carried PMAT-4269 fragment. Evidence: the 8 tests_apr_cpu_gen_config_4265 tests pass. A mutant that makes apr_cpu_stop_tokens return vec![] fails 3 of them. Clippy -D warnings is clean on apr-cli --lib. (cherry picked from commit d65a915beaf28486b70d6498baa9a7f1efd098bc) Refs #4265 Co-Authored-By: Claude Opus 5.5 --- .../serve/handler_apr_cpu_completion.rs | 40 +++--- .../commands/serve/handler_gpu_completion.rs | 8 +- crates/apr-cli/src/commands/serve/handlers.rs | 108 ++++++++++++--- .../serve/tests_apr_cpu_gen_config_4265.rs | 125 ++++++++++++++++++ docs/roadmaps/entries/PMAT-4269.yaml | 17 +++ docs/roadmaps/roadmap.yaml | 17 +++ 6 files changed, 269 insertions(+), 46 deletions(-) create mode 100644 crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs create mode 100644 docs/roadmaps/entries/PMAT-4269.yaml diff --git a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs index 0cc52d93d0..475b2e890d 100644 --- a/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_apr_cpu_completion.rs @@ -26,10 +26,11 @@ async fn handle_apr_cpu_completion( let max_tokens = req.max_tokens.min(4096); let prompt = req.prompt.clone(); let temperature = req.temperature.unwrap_or(0.0); + let top_p = req.top_p; // GH-284: Run inference off the async runtime to avoid blocking let result = tokio::task::spawn_blocking(move || { - run_apr_cpu_inference(&s, &prompt, max_tokens, temperature) + run_apr_cpu_inference(&s, &prompt, max_tokens, temperature, top_p) }) .await; @@ -114,6 +115,7 @@ fn spawn_cpu_streaming_task( prompt: String, max_tokens: usize, temperature: f32, + top_p: Option, tx: tokio::sync::mpsc::Sender>, ) { tokio::task::spawn_blocking(move || { @@ -129,16 +131,8 @@ fn spawn_cpu_streaming_task( None => prompt.chars().map(|c| c as u32).collect(), }; - let gen_config = realizar::apr_transformer::GenerateConfig { - max_tokens, - temperature, - top_p: 0.9, - top_k: 0, - repetition_penalty: 1.0, - trace: false, - stop_tokens: vec![], - cancel: realizar::generate::CancelToken::never(), - }; + let gen_config = + apr_cpu_generate_config(max_tokens, temperature, top_p, apr_cpu_stop_tokens(&s)); let Ok(t) = transformer.lock() else { if tx.blocking_send(Err("Lock poisoned".to_string())).is_err() { @@ -171,6 +165,7 @@ fn spawn_cpu_token_text_stream( prompt: String, max_tokens: usize, temperature: f32, + top_p: Option, tx: tokio::sync::mpsc::Sender>, ) { // Test seam: replay a scripted token sequence through the channel. @@ -203,16 +198,8 @@ fn spawn_cpu_token_text_stream( None => prompt.chars().map(|c| c as u32).collect(), }; - let gen_config = realizar::apr_transformer::GenerateConfig { - max_tokens, - temperature, - top_p: 0.9, - top_k: 0, - repetition_penalty: 1.0, - trace: false, - stop_tokens: vec![], - cancel: realizar::generate::CancelToken::never(), - }; + let gen_config = + apr_cpu_generate_config(max_tokens, temperature, top_p, apr_cpu_stop_tokens(&s)); let Ok(t) = transformer.lock() else { if tx.blocking_send(Err("Lock poisoned".to_string())).is_err() { @@ -335,6 +322,7 @@ async fn handle_apr_cpu_chat_completion( let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false); let max_tokens = req.get("max_tokens").and_then(serde_json::Value::as_u64).unwrap_or(32) as usize; let temperature = req.get("temperature").and_then(serde_json::Value::as_f64).unwrap_or(0.0) as f32; + let top_p = req.get("top_p").and_then(serde_json::Value::as_f64).map(|v| v as f32); let Some(msgs) = messages else { return Json(serde_json::json!({"error": "Missing messages"})).into_response(); @@ -345,7 +333,7 @@ async fn handle_apr_cpu_chat_completion( // GH-284: True SSE streaming path if stream_mode { let (tx, rx) = tokio::sync::mpsc::channel::>(16); - spawn_cpu_streaming_task(s.clone(), prompt, max_tokens.min(4096), temperature, tx); + spawn_cpu_streaming_task(s.clone(), prompt, max_tokens.min(4096), temperature, top_p, tx); return build_cpu_sse_stream(rx, s.tokenizer.clone(), s.model_name.clone()); } @@ -356,7 +344,7 @@ async fn handle_apr_cpu_chat_completion( let max_t = max_tokens.min(4096); let result = tokio::task::spawn_blocking(move || { - run_apr_cpu_inference(&s_for_blocking, &prompt_owned, max_t, temperature) + run_apr_cpu_inference(&s_for_blocking, &prompt_owned, max_t, temperature, top_p) }) .await; @@ -432,7 +420,8 @@ async fn handle_apr_cpu_ollama_chat( let (max_tokens, temperature) = ollama_sampling(&req.options); let prompt_eval_count = count_prompt_tokens(&s, &prompt); let (tx, rx) = tokio::sync::mpsc::channel::>(16); - spawn_cpu_token_text_stream(s, prompt, max_tokens, temperature, tx); + let top_p = req.options.as_ref().and_then(|o| o.top_p); + spawn_cpu_token_text_stream(s, prompt, max_tokens, temperature, top_p, tx); return super::ollama::ollama_ndjson_stream( super::ollama::OllamaStreamKind::Chat, model, @@ -475,7 +464,8 @@ async fn handle_apr_cpu_ollama_generate( let (max_tokens, temperature) = ollama_sampling(&req.options); let prompt_eval_count = count_prompt_tokens(&s, &prompt); let (tx, rx) = tokio::sync::mpsc::channel::>(16); - spawn_cpu_token_text_stream(s, prompt, max_tokens, temperature, tx); + let top_p = req.options.as_ref().and_then(|o| o.top_p); + spawn_cpu_token_text_stream(s, prompt, max_tokens, temperature, top_p, tx); return super::ollama::ollama_ndjson_stream( super::ollama::OllamaStreamKind::Generate, model, diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 2769e6999b..66faaca34d 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -49,7 +49,7 @@ async fn handle_gpu_completion( // CPU fallback also in spawn_blocking let result = tokio::task::spawn_blocking(move || { - run_apr_cpu_inference(&s, &prompt, max_tokens, 0.0) + run_apr_cpu_inference(&s, &prompt, max_tokens, 0.0, None) }) .await; @@ -116,6 +116,7 @@ async fn gpu_cpu_fallback( prompt: String, max_tokens: usize, temperature: f32, + top_p: Option, start: Instant, ) -> axum::response::Response { use axum::{response::IntoResponse, Json}; @@ -132,7 +133,7 @@ async fn gpu_cpu_fallback( }; let result = tokio::task::spawn_blocking(move || { - run_apr_cpu_inference(&s, &prompt, max_tokens, temperature) + run_apr_cpu_inference(&s, &prompt, max_tokens, temperature, top_p) }) .await; @@ -266,6 +267,7 @@ async fn handle_gpu_chat_completion( let stream_mode = req.get("stream").and_then(serde_json::Value::as_bool).unwrap_or(false); let max_tokens = req.get("max_tokens").and_then(serde_json::Value::as_u64).unwrap_or(32) as usize; let temperature = req.get("temperature").and_then(serde_json::Value::as_f64).unwrap_or(0.0) as f32; + let top_p = req.get("top_p").and_then(serde_json::Value::as_f64).map(|v| v as f32); let Some(msgs) = messages else { return Json(serde_json::json!({"error": "Missing messages"})).into_response(); @@ -289,7 +291,7 @@ async fn handle_gpu_chat_completion( let output_tokens = match gen_result { Ok(Ok(t)) => t, Ok(Err(gpu_err)) => { - return gpu_cpu_fallback(gpu_err, &cpu_state, prompt, max_tokens_clamped, temperature, start).await; + return gpu_cpu_fallback(gpu_err, &cpu_state, prompt, max_tokens_clamped, temperature, top_p, start).await; } Err(e) => { return Json(serde_json::json!({"error": format!("GPU task failed: {e}")})).into_response(); diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index 5e696eaeef..a1868d8985 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -1085,6 +1085,7 @@ fn run_apr_cpu_inference( prompt: &str, max_tokens: usize, temperature: f32, + top_p: Option, ) -> std::result::Result { let transformer = state .transformer @@ -1101,16 +1102,8 @@ fn run_apr_cpu_inference( }; let input_token_count = input_tokens.len(); - let gen_config = realizar::apr_transformer::GenerateConfig { - max_tokens, - temperature, - top_p: 0.9, - top_k: 0, - repetition_penalty: 1.0, - trace: false, - stop_tokens: vec![], - cancel: realizar::generate::CancelToken::never(), - }; + let gen_config = + apr_cpu_generate_config(max_tokens, temperature, top_p, apr_cpu_stop_tokens(state)); let gen_start = Instant::now(); let output_tokens = { @@ -1128,24 +1121,30 @@ fn run_apr_cpu_inference( // as the reply, and counted it as `completion_tokens` (#3718). let new_tokens = output_tokens.get(input_tokens.len()..).unwrap_or(&[]); + // #3718: the chat handler hardcoded "stop", so a reply cut at `max_tokens` + // (which the handler caps at 4096) read as finished. The loop's stop set is + // `gen_config.stop_tokens` plus token 0, which it pushes before breaking + // (`is_eos_token`, apr_transformer/generation.rs). + let mut stop_ids = gen_config.stop_tokens.clone(); + stop_ids.push(0); + + // #4265: the loop pushes the stop id it ended on; it is not reply text. + let reply_tokens = apr_cpu_reply_tokens(new_tokens, &stop_ids); + // Decode: embedded APR tokenizer → sibling tokenizer.json → character-level fallback let text = if let Some(ref tok) = state.embedded_tokenizer { - tok.decode(new_tokens) + tok.decode(reply_tokens) } else if let Some(ref tok) = state.tokenizer { - tok.tokenizer.decode(new_tokens).unwrap_or_default() + tok.tokenizer.decode(reply_tokens).unwrap_or_default() } else { - new_tokens + reply_tokens .iter() .filter_map(|&t| char::from_u32(t)) .collect() }; - // #3718: the chat handler hardcoded "stop", so a reply cut at `max_tokens` - // (which the handler caps at 4096) read as finished. The loop's stop set is - // `gen_config.stop_tokens` (empty here) plus token 0, which it pushes before - // breaking (`is_eos_token`, apr_transformer/generation.rs). let finish_reason = - realizar::infer::run_report::FinishReason::from_decode(new_tokens, &[0], max_tokens); + realizar::infer::run_report::FinishReason::from_decode(new_tokens, &stop_ids, max_tokens); Ok(AprInferenceOutput { text, @@ -1156,6 +1155,76 @@ fn run_apr_cpu_inference( }) } +/// #4265: chat-turn terminators. The APR CPU chat handlers format ChatML +/// (`format_chatml`), so `<|im_end|>` is the one that ends a reply there; the +/// others end a turn in the templates GGUF imports carry. +#[cfg(feature = "inference")] +const APR_CPU_TURN_END_TOKENS: &[&str] = + &["<|im_end|>", "<|endoftext|>", "<|eot_id|>", ""]; + +/// #4265: the token ids that end an APR CPU generation: every EOS the loaded +/// tokenizer declares plus the chat-turn terminators it has an id for. The +/// generation loop already stops at id 0 on its own (`is_eos_token`). +#[cfg(feature = "inference")] +fn apr_cpu_stop_tokens(state: &AprServerState) -> Vec { + let mut stop = Vec::new(); + if let Some(tok) = &state.embedded_tokenizer { + stop.extend(tok.eos_id); + stop.extend( + APR_CPU_TURN_END_TOKENS + .iter() + .filter_map(|t| tok.special_tokens.get(*t).copied()), + ); + } + if let Some(tok) = &state.tokenizer { + stop.extend(tok.eos_token_id); + stop.extend(tok.vocab.iter().enumerate().filter_map(|(id, t)| { + APR_CPU_TURN_END_TOKENS + .contains(&t.as_str()) + .then_some(id as u32) + })); + } + stop.sort_unstable(); + stop.dedup(); + stop +} + +/// The reply text's tokens: `new_tokens` without the stop id the loop ended on. +/// `stop_ids` must be the loop's whole stop set, token 0 included — the loop +/// ends on 0 even when it is not in `stop_tokens` (#4265). +#[cfg(feature = "inference")] +fn apr_cpu_reply_tokens<'a>(new_tokens: &'a [u32], stop_ids: &[u32]) -> &'a [u32] { + match new_tokens.split_last() { + Some((last, head)) if stop_ids.contains(last) => head, + _ => new_tokens, + } +} + +/// #4265: the one `GenerateConfig` every APR CPU path (blocking, SSE, NDJSON) builds. +#[cfg(feature = "inference")] +fn apr_cpu_generate_config( + max_tokens: usize, + temperature: f32, + top_p: Option, + stop_tokens: Vec, +) -> realizar::apr_transformer::GenerateConfig { + realizar::apr_transformer::GenerateConfig { + max_tokens, + temperature, + // The same default every other serve backend applies when the request is silent. + top_p: top_p.unwrap_or(realizar::gguf::QuantizedGenerateConfig::default().top_p), + top_k: 0, + repetition_penalty: 1.0, + trace: false, + stop_tokens, + cancel: realizar::generate::CancelToken::never(), + } +} + +#[cfg(all(test, feature = "inference"))] +#[path = "tests_apr_cpu_gen_config_4265.rs"] +mod tests_apr_cpu_gen_config_4265; + /// Load APR model, tokenizer, and transformer into shared server state. #[cfg(feature = "inference")] fn load_apr_model_state(model_path: &Path, config: &ServerConfig) -> Result { @@ -1298,6 +1367,9 @@ struct AprCompletionRequest { max_tokens: usize, #[serde(default)] temperature: Option, + /// #4265: honoured on the APR CPU path; absent means the shared serve default. + #[serde(default)] + top_p: Option, } #[cfg(feature = "inference")] diff --git a/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs b/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs new file mode 100644 index 0000000000..c59302aba6 --- /dev/null +++ b/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs @@ -0,0 +1,125 @@ +//! #4265: the APR CPU serve path stops at the model's EOS / chat-turn end and +//! honours the request's `top_p`, the same as every other serve backend. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use super::*; +use std::collections::HashMap; + +fn state_with( + embedded: Option, + eos_from_tokenizer_json: Option, +) -> AprServerState { + AprServerState { + transformer: None, + model_type: "test".to_string(), + architecture: "test".to_string(), + is_transformer: false, + tokenizer: eos_from_tokenizer_json.map(|eos| SafeTensorsTokenizerInfo { + tokenizer: std::sync::Arc::new( + realizar::tokenizer::BPETokenizer::new(vec!["".to_string()], vec![], "") + .expect("tiny tokenizer"), + ), + vocab: vec![], + bos_token_id: None, + eos_token_id: Some(eos), + }), + embedded_tokenizer: embedded, + model_name: "apr".to_string(), + demo_scripted_tokens: None, + } +} + +fn embedded(eos: Option, specials: &[(&str, u32)]) -> realizar::apr::BpeTokenizer { + realizar::apr::BpeTokenizer { + token_to_id: HashMap::new(), + id_to_token: vec![], + merge_rules: vec![], + bos_id: None, + eos_id: eos, + special_tokens: specials + .iter() + .map(|(s, i)| ((*s).to_string(), *i)) + .collect(), + } +} + +#[test] +fn stop_set_holds_embedded_eos_and_chat_turn_end() { + // A Qwen-shaped GGUF import: EOS <|endoftext|> = 151643, turn end <|im_end|> = 151645. + let s = state_with( + Some(embedded( + Some(151_643), + &[("<|im_end|>", 151_645), ("<|im_start|>", 151_644)], + )), + None, + ); + let mut stop = apr_cpu_stop_tokens(&s); + stop.sort_unstable(); + assert_eq!( + stop, + vec![151_643, 151_645], + "<|im_start|> is not a stop token" + ); +} + +#[test] +fn stop_set_holds_sibling_tokenizer_json_eos() { + let s = state_with(None, Some(2)); + assert_eq!(apr_cpu_stop_tokens(&s), vec![2]); +} + +#[test] +fn no_tokenizer_means_no_invented_stop_ids() { + assert!(apr_cpu_stop_tokens(&state_with(None, None)).is_empty()); +} + +#[test] +fn config_carries_the_stop_set_and_the_requests_top_p() { + let c = apr_cpu_generate_config(16, 0.7, Some(0.5), vec![151_645]); + assert_eq!(c.stop_tokens, vec![151_645]); + assert!( + (c.top_p - 0.5).abs() < f32::EPSILON, + "request top_p ignored: {}", + c.top_p + ); +} + +#[test] +fn absent_top_p_takes_the_other_backends_default() { + let want = realizar::gguf::QuantizedGenerateConfig::default().top_p; + let c = apr_cpu_generate_config(16, 0.7, None, vec![]); + assert!( + (c.top_p - want).abs() < f32::EPSILON, + "top_p {} != default {want}", + c.top_p + ); +} + +#[test] +fn reply_drops_the_stop_id_the_loop_ended_on() { + let stop = [151_645, 0]; + assert_eq!(apr_cpu_reply_tokens(&[9, 8, 151_645], &stop), &[9, 8]); + // Token 0 ends the loop even when no tokenizer named it (is_eos_token). + assert_eq!(apr_cpu_reply_tokens(&[9, 8, 0], &stop), &[9, 8]); +} + +#[test] +fn reply_keeps_a_budget_cut_and_an_inner_stop_id() { + let stop = [151_645, 0]; + assert_eq!(apr_cpu_reply_tokens(&[9, 8, 7], &stop), &[9, 8, 7]); + assert_eq!(apr_cpu_reply_tokens(&[151_645, 8], &stop), &[151_645, 8]); + assert!(apr_cpu_reply_tokens(&[], &stop).is_empty()); +} + +#[test] +fn stop_set_holds_sibling_tokenizer_json_turn_end_by_vocab_index() { + let mut s = state_with(None, Some(2)); + if let Some(tok) = s.tokenizer.as_mut() { + tok.vocab = ["a", "<|im_start|>", "b", "<|im_end|>"] + .iter() + .map(|t| (*t).to_string()) + .collect(); + } + // <|im_end|> sits at index 3; <|im_start|> (index 1) is not a stop. + assert_eq!(apr_cpu_stop_tokens(&s), vec![2, 3]); +} diff --git a/docs/roadmaps/entries/PMAT-4269.yaml b/docs/roadmaps/entries/PMAT-4269.yaml new file mode 100644 index 0000000000..434eb50e31 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4269.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4269 + github_issue: 4269 + item_type: task + title: 'one-engine workstream M: port qwen3_moe run, ST CUDA run, APR CPU serve, APR Q4K scheduler, ST serve and ST chat onto realizar::session::Session (child of #4263)' + status: in_progress + priority: high + assigned_to: aprender-cb + created: 2026-09-24T11:30:00Z + updated: 2026-09-24T11:30:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Roadmap id authorized by the 0.69.3 captain (aprender-36). Targets release/0.69.3 via the #4263 one-engine branch; #4265 (APR CPU stop/top_p) closed from this workstream.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 08a3c10e6c..014377f359 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -20628,3 +20628,20 @@ roadmap: labels: - kind:docs notes: 'evidence/release/0.69.4/refs-qwen35-9b/README.md' +- id: PMAT-4269 + github_issue: 4269 + item_type: task + title: 'one-engine workstream M: port qwen3_moe run, ST CUDA run, APR CPU serve, APR Q4K scheduler, ST serve and ST chat onto realizar::session::Session (child of #4263)' + status: in_progress + priority: high + assigned_to: aprender-cb + created: 2026-09-24T11:30:00Z + updated: 2026-09-24T11:30:00Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Roadmap id authorized by the 0.69.3 captain (aprender-36). Targets release/0.69.3 via the #4263 one-engine branch; #4265 (APR CPU stop/top_p) closed from this workstream.' From 83bb714f61539abac5ca6f21f77046d902193efe Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 22:39:49 +0200 Subject: [PATCH 08/68] fix(serve): SafeTensors chat/generate stop at EOS/<|im_end|> and honour top_p (#4334) chat.rs and simple.rs built their own GenerateConfig with stop_tokens: vec![] and top_p: 0.9, so every SafeTensors reply ran to max_tokens and ignored the request's top_p (the #4265 defect on a sibling path). Both now go through apr_cpu_generate_config with the tokenizer's EOS + chat-turn-end ids (tokenizer_info_stop_tokens, now shared with the APR CPU path), and strip the stop id the loop ended on from the reply. Refs #4265 #4249 Co-Authored-By: Claude Opus 5.5 --- crates/apr-cli/src/commands/serve/chat.rs | 25 ++-- crates/apr-cli/src/commands/serve/handlers.rs | 39 ++++-- .../apr-cli/src/commands/serve/safetensors.rs | 4 + crates/apr-cli/src/commands/serve/simple.rs | 25 ++-- .../serve/tests_safetensors_stop_4334.rs | 119 ++++++++++++++++++ 5 files changed, 183 insertions(+), 29 deletions(-) create mode 100644 crates/apr-cli/src/commands/serve/tests_safetensors_stop_4334.rs diff --git a/crates/apr-cli/src/commands/serve/chat.rs b/crates/apr-cli/src/commands/serve/chat.rs index f8190817df..c64aa90cdf 100644 --- a/crates/apr-cli/src/commands/serve/chat.rs +++ b/crates/apr-cli/src/commands/serve/chat.rs @@ -267,16 +267,19 @@ pub(crate) async fn safetensors_chat_completions_handler( .get("temperature") .and_then(|t| t.as_f64()) .unwrap_or(0.0) as f32; - let gen_config = realizar::apr_transformer::GenerateConfig { - max_tokens, - temperature, - top_p: 0.9, - top_k: 0, - repetition_penalty: 1.0, - trace: false, - stop_tokens: vec![], - cancel: realizar::generate::CancelToken::never(), - }; + // #4334: stop at the tokenizer's EOS / chat-turn end and honour the request's + // top_p, through the same config builder as the APR CPU path (#4265). + let top_p = request + .get("top_p") + .and_then(serde_json::Value::as_f64) + .map(|p| p as f32); + let stop_tokens = state + .tokenizer_info + .as_ref() + .map(super::handlers::tokenizer_info_stop_tokens) + .unwrap_or_default(); + let gen_config = + super::handlers::apr_cpu_generate_config(max_tokens, temperature, top_p, stop_tokens); let output_ids = { // PMAT-189: Handle transformer lock poisoning gracefully let t = match transformer.lock() { @@ -305,7 +308,7 @@ pub(crate) async fn safetensors_chat_completions_handler( let elapsed = start.elapsed(); // Decode output using BPE tokenizer (PMAT-093) - let new_tokens = &output_ids[input_ids.len()..]; + let new_tokens = super::handlers::generated_reply_tokens(&output_ids, input_ids.len(), &gen_config); let output_text = if let Some(ref tok_info) = state.tokenizer_info { match tok_info.tokenizer.decode(new_tokens) { Ok(text) => text, diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index a1868d8985..56f762b93f 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -1177,18 +1177,43 @@ fn apr_cpu_stop_tokens(state: &AprServerState) -> Vec { ); } if let Some(tok) = &state.tokenizer { - stop.extend(tok.eos_token_id); - stop.extend(tok.vocab.iter().enumerate().filter_map(|(id, t)| { - APR_CPU_TURN_END_TOKENS - .contains(&t.as_str()) - .then_some(id as u32) - })); + stop.extend(tokenizer_info_stop_tokens(tok)); } stop.sort_unstable(); stop.dedup(); stop } +/// #4334: the stop ids a tokenizer.json-backed tokenizer declares: its EOS plus +/// the chat-turn terminators its vocab has. Shared by the APR CPU path and the +/// SafeTensors handlers (`chat.rs`, `simple.rs`), which hardcoded an empty set. +#[cfg(feature = "inference")] +pub(crate) fn tokenizer_info_stop_tokens(tok: &SafeTensorsTokenizerInfo) -> Vec { + let mut stop: Vec = tok.eos_token_id.into_iter().collect(); + stop.extend(tok.vocab.iter().enumerate().filter_map(|(id, t)| { + APR_CPU_TURN_END_TOKENS + .contains(&t.as_str()) + .then_some(id as u32) + })); + stop.sort_unstable(); + stop.dedup(); + stop +} + +/// #4334: the reply tokens of a `generate_with_cache` result: everything past the +/// prompt, minus the stop id the loop ended on (its stop set is +/// `gen_config.stop_tokens` plus token 0). +#[cfg(feature = "inference")] +pub(crate) fn generated_reply_tokens<'a>( + output_ids: &'a [u32], + prompt_len: usize, + gen_config: &realizar::apr_transformer::GenerateConfig, +) -> &'a [u32] { + let mut stop_ids = gen_config.stop_tokens.clone(); + stop_ids.push(0); + apr_cpu_reply_tokens(output_ids.get(prompt_len..).unwrap_or(&[]), &stop_ids) +} + /// The reply text's tokens: `new_tokens` without the stop id the loop ended on. /// `stop_ids` must be the loop's whole stop set, token 0 included — the loop /// ends on 0 even when it is not in `stop_tokens` (#4265). @@ -1202,7 +1227,7 @@ fn apr_cpu_reply_tokens<'a>(new_tokens: &'a [u32], stop_ids: &[u32]) -> &'a [u32 /// #4265: the one `GenerateConfig` every APR CPU path (blocking, SSE, NDJSON) builds. #[cfg(feature = "inference")] -fn apr_cpu_generate_config( +pub(crate) fn apr_cpu_generate_config( max_tokens: usize, temperature: f32, top_p: Option, diff --git a/crates/apr-cli/src/commands/serve/safetensors.rs b/crates/apr-cli/src/commands/serve/safetensors.rs index 60bcde2c78..a289c03edf 100644 --- a/crates/apr-cli/src/commands/serve/safetensors.rs +++ b/crates/apr-cli/src/commands/serve/safetensors.rs @@ -460,3 +460,7 @@ fn classify_bos_eos(content: &str) -> (bool, bool) { include!("chat.rs"); include!("simple.rs"); + +#[cfg(all(test, feature = "inference"))] +#[path = "tests_safetensors_stop_4334.rs"] +mod tests_safetensors_stop_4334; diff --git a/crates/apr-cli/src/commands/serve/simple.rs b/crates/apr-cli/src/commands/serve/simple.rs index d0f74d0106..262ada4961 100644 --- a/crates/apr-cli/src/commands/serve/simple.rs +++ b/crates/apr-cli/src/commands/serve/simple.rs @@ -40,16 +40,19 @@ pub(crate) async fn safetensors_generate_handler( .get("temperature") .and_then(|t| t.as_f64()) .unwrap_or(0.0) as f32; - let gen_config = realizar::apr_transformer::GenerateConfig { - max_tokens, - temperature, - top_p: 0.9, - top_k: 0, - repetition_penalty: 1.0, - trace: false, - stop_tokens: vec![], - cancel: realizar::generate::CancelToken::never(), - }; + // #4334: stop at the tokenizer's EOS / chat-turn end and honour the request's + // top_p, through the same config builder as the APR CPU path (#4265). + let top_p = request + .get("top_p") + .and_then(serde_json::Value::as_f64) + .map(|p| p as f32); + let stop_tokens = state + .tokenizer_info + .as_ref() + .map(super::handlers::tokenizer_info_stop_tokens) + .unwrap_or_default(); + let gen_config = + super::handlers::apr_cpu_generate_config(max_tokens, temperature, top_p, stop_tokens); let output_ids = { // PMAT-189: Handle transformer lock poisoning gracefully let t = match transformer.lock() { @@ -78,7 +81,7 @@ pub(crate) async fn safetensors_generate_handler( let elapsed = start.elapsed(); // Decode using BPE tokenizer (PMAT-093) - let new_tokens = &output_ids[input_ids.len()..]; + let new_tokens = super::handlers::generated_reply_tokens(&output_ids, input_ids.len(), &gen_config); let output_text = if let Some(ref tok_info) = state.tokenizer_info { match tok_info.tokenizer.decode(new_tokens) { Ok(text) => text, diff --git a/crates/apr-cli/src/commands/serve/tests_safetensors_stop_4334.rs b/crates/apr-cli/src/commands/serve/tests_safetensors_stop_4334.rs new file mode 100644 index 0000000000..11daccb0b9 --- /dev/null +++ b/crates/apr-cli/src/commands/serve/tests_safetensors_stop_4334.rs @@ -0,0 +1,119 @@ +//! #4334: the SafeTensors serve handlers (`chat.rs`, `simple.rs`) stop at the +//! tokenizer's EOS / chat-turn end, like the APR CPU path does since #4265. +//! They hardcoded `stop_tokens: vec![]`, so every reply ran to `max_tokens`. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use super::*; +use axum::body::to_bytes; +use realizar::apr_transformer::{AprTransformer, AprTransformerConfig}; + +const VOCAB: usize = 100; +const IM_END: u32 = 7; +const EOS: u32 = 9; +const MAX_TOKENS: u64 = 8; + +/// A vocab with `<|im_end|>` at `IM_END`; every other entry is an ordinary piece. +fn vocab() -> Vec { + (0..VOCAB) + .map(|i| match i { + 0 => "".to_string(), + 7 => "<|im_end|>".to_string(), + _ => format!("t{i}"), + }) + .collect() +} + +fn tokenizer_info(eos: Option) -> SafeTensorsTokenizerInfo { + let vocab = vocab(); + SafeTensorsTokenizerInfo { + tokenizer: std::sync::Arc::new( + realizar::tokenizer::BPETokenizer::new(vocab.clone(), vec![], "") + .expect("tiny tokenizer"), + ), + vocab, + bos_token_id: None, + eos_token_id: eos, + } +} + +/// A tiny zero-weight transformer whose LM-head bias makes greedy decoding emit +/// `always` at every step, so the only way generation ends early is a stop id. +fn state(always: u32, eos: Option) -> SafeTensorsState { + let mut t = AprTransformer::new(AprTransformerConfig { + architecture: "test".to_string(), + hidden_dim: 16, + num_layers: 1, + num_heads: 2, + num_kv_heads: 2, + vocab_size: VOCAB, + intermediate_dim: 32, + context_length: 128, + rope_theta: 10000.0, + eps: 1e-5, + eos_token_id: None, + ..Default::default() + }); + let mut bias = vec![0.0; VOCAB]; + bias[always as usize] = 10.0; + t.lm_head_bias = Some(bias); + SafeTensorsState { + transformer: Some(std::sync::Arc::new(std::sync::Mutex::new(t))), + tokenizer_info: Some(tokenizer_info(eos)), + model_path: "test".to_string(), + } +} + +async fn json(resp: axum::response::Response) -> serde_json::Value { + let bytes = to_bytes(resp.into_body(), 64 * 1024).await.expect("body"); + serde_json::from_slice(&bytes).expect("json") +} + +#[test] +fn stop_set_holds_eos_and_turn_end_from_vocab() { + assert_eq!( + super::super::handlers::tokenizer_info_stop_tokens(&tokenizer_info(Some(EOS))), + vec![IM_END, EOS] + ); + assert_eq!( + super::super::handlers::tokenizer_info_stop_tokens(&tokenizer_info(None)), + vec![IM_END] + ); +} + +#[tokio::test] +async fn generate_handler_stops_at_eos() { + let resp = safetensors_generate_handler( + axum::extract::State(state(EOS, Some(EOS))), + axum::Json(serde_json::json!({"prompt": "t1", "max_tokens": MAX_TOKENS})), + ) + .await; + let v = json(resp).await; + // One step: the model emits EOS, the loop stops, and the stop id is not reply text. + assert_eq!(v["tokens_generated"], 0, "{v}"); +} + +#[tokio::test] +async fn generate_handler_runs_to_max_without_a_stop_id() { + // Positive control: the same model emitting an ordinary token runs to max_tokens, + // so the test above measures the stop set, not a broken generation loop. + let resp = safetensors_generate_handler( + axum::extract::State(state(11, Some(EOS))), + axum::Json(serde_json::json!({"prompt": "t1", "max_tokens": MAX_TOKENS})), + ) + .await; + assert_eq!(json(resp).await["tokens_generated"], MAX_TOKENS); +} + +#[tokio::test] +async fn chat_handler_stops_at_im_end() { + let resp = safetensors_chat_completions_handler( + axum::extract::State(state(IM_END, None)), + axum::Json(serde_json::json!({ + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": MAX_TOKENS, + })), + ) + .await; + let v = json(resp).await; + assert_eq!(v["usage"]["completion_tokens"], 0, "{v}"); +} From cc47da2f6ebf8e7523c327e74a2181e45249e298 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 25 Sep 2026 00:13:16 +0200 Subject: [PATCH 09/68] =?UTF-8?q?fix(audit):=20re-audit=20GET=20/tensors?= =?UTF-8?q?=20row=20=E2=80=94=20safetensors.rs:164=20was=20the=20/health?= =?UTF-8?q?=20handler;=20the=20route=20is=20:170=20(0.69.4=20dogfood=20G2.?= =?UTF-8?q?1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This branch changes serve/safetensors.rs, which the ledger cites, so G2.1 required the row to be re-audited. On inspection the row was already stale on main: :164 is inside /health, and /tensors is at :170 on main and here. The single-server /tensors row is corrected; check_dogfood_coverage: run PASS, self-test PASS. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/audits/surface_audit.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/audits/surface_audit.csv b/docs/audits/surface_audit.csv index eb554f829c..c918f4edfb 100644 --- a/docs/audits/surface_audit.csv +++ b/docs/audits/surface_audit.csv @@ -46,7 +46,7 @@ apr,GET /metrics/dispatch,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/a apr,GET /models,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:95,high apr,GET /ready,6,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:93,high apr,GET /realize/model,4,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:105,high -apr,GET /tensors,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/serve/safetensors.rs:164,low +apr,GET /tensors,6,x86_64-linux,llama.cpp,no,10,apr-core-commands,crates/apr-cli/src/commands/serve/safetensors.rs:170,low apr,GET /v1/audit/:request_id,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:146,medium apr,GET /v1/effective-config,7,x86_64-linux,llama.cpp,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:161,high apr,GET /v1/gpu/status,7,x86_64-linux,vllm,yes,3,http-apr-serve,crates/aprender-serve/src/api/router.rs:149,medium From 7b69014070a15186c2198a1f91d12dff1725da32 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 25 Sep 2026 00:19:06 +0200 Subject: [PATCH 10/68] =?UTF-8?q?fix(claims):=200.69.4=20guard-cargo=20red?= =?UTF-8?q?=20=E2=80=94=20handler=5Fgpu=5Fcompletion=20speed=20literals=20?= =?UTF-8?q?removed=20(#4085)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI run 36057175909 on batch/0.69.4-epic-4249 failed guard-cargo on handler_gpu_completion.rs:364-365 ("100+ tok/s CPU, 500+ tok/s GPU", "800+ tok/s (2.8x Ollama)"). These are not new claims: #4265's forward-port pushed the lines down 2 from their baselined :362/:363. Per #4085 they are removed, not re-keyed; the comment points at docs/BEATS.md, where the measured Ollama comparison lives. Baselines only shrink: claim_literal 4 removed, perf_claim_citation 7 removed (stale entries, --update with 0 new hits; diffs are removals only). Both guards rc 0, check_baseline_ratchets PASS, cargo fmt -p apr-cli clean. Co-Authored-By: Claude Opus 5.5 --- .../apr-cli/src/commands/serve/handler_gpu_completion.rs | 4 ++-- scripts/claim_literal_baseline.txt | 4 ---- scripts/perf_claim_citation_baseline.txt | 7 ------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 66faaca34d..5d2afbd1d2 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -361,8 +361,8 @@ fn print_gpu_server_banner(bind_addr: &str) { /// Start GGUF model inference server with Ollama-parity performance /// /// Uses realizar's full inference API for text generation, streaming, and batch inference. -/// Achieves Ollama-parity: 100+ tok/s CPU, 500+ tok/s GPU. -/// With --gpu --batch flags: 800+ tok/s (2.8x Ollama) via batched GPU inference. +/// `--gpu --batch` serves through batched GPU inference. What that measures against +/// Ollama is on the beat scoreboard (docs/BEATS.md), not in this comment (#4085). #[cfg(feature = "inference")] fn start_gguf_server(model_path: &Path, config: &ServerConfig) -> Result<()> { use realizar::gguf::{MappedGGUFModel, OwnedQuantizedModel}; diff --git a/scripts/claim_literal_baseline.txt b/scripts/claim_literal_baseline.txt index f4484429a5..3fcf928b51 100644 --- a/scripts/claim_literal_baseline.txt +++ b/scripts/claim_literal_baseline.txt @@ -186,10 +186,6 @@ crates/apr-cli/src/commands/benchmark.rs:438 crates/apr-cli/src/commands/gguf_generate_result.rs:371 crates/apr-cli/src/commands/gguf_generate_result.rs:372 crates/apr-cli/src/commands/qa.rs:147 -crates/apr-cli/src/commands/run_tests_layer_trace.rs:55 -crates/apr-cli/src/commands/run_tests_layer_trace.rs:56 -crates/apr-cli/src/commands/serve/handler_gpu_completion.rs:362 -crates/apr-cli/src/commands/serve/handler_gpu_completion.rs:363 crates/apr-cli/src/commands/serve/handlers_include_01.rs:4 crates/apr-cli/src/commands/serve/mod.rs:209 crates/apr-cli/src/commands/speedup.rs:11 diff --git a/scripts/perf_claim_citation_baseline.txt b/scripts/perf_claim_citation_baseline.txt index f49364c3cf..d13adf4be8 100644 --- a/scripts/perf_claim_citation_baseline.txt +++ b/scripts/perf_claim_citation_baseline.txt @@ -83,7 +83,6 @@ book/src/ml-fundamentals/feature-scaling.md:585 book/src/ml-fundamentals/graph-algorithms.md:649 book/src/ml-fundamentals/graph-components-traversal.md:512 book/src/ml-fundamentals/graph-pathfinding.md:396 -crates/apr-cli/src/commands/eval/inference.rs:516 crates/apr-cli/src/commands/kernel.rs:346 crates/apr-cli/src/commands/ollama.rs:24 crates/apr-cli/src/commands/qa.rs:152 @@ -120,9 +119,6 @@ crates/aprender-profile/src/cli.rs:249 crates/aprender-profile/src/hpu.rs:10 crates/aprender-profile/src/model_persistence.rs:4 crates/aprender-qa-gen/src/kernel_coverage.rs:122 -crates/aprender-serve/src/api/mod.rs:161 -crates/aprender-serve/src/api/mod.rs:164 -crates/aprender-serve/src/api/mod_app_state_new.rs:517 crates/aprender-serve/src/apr/mapped_apr_model.rs:63 crates/aprender-serve/src/apr/special_tokens.rs:63 crates/aprender-serve/src/brick/brick.rs:316 @@ -154,9 +150,6 @@ crates/aprender-serve/src/quantize/fused_k.rs:266 crates/aprender-serve/src/quantize/fused_q4_0_q8_0.rs:5 crates/aprender-serve/src/quantize/fused_q5k_q6k.rs:135 crates/aprender-serve/src/quantize/fused_q8_0_q8_0.rs:230 -crates/aprender-serve/src/quantize/mod.rs:179 -crates/aprender-serve/src/quantize/mod.rs:292 -crates/aprender-serve/src/quantize/mod.rs:318 crates/aprender-serve/src/quantize/parallel_dequant.rs:144 crates/aprender-serve/src/quantize/parallel_dequant.rs:145 crates/aprender-serve/src/quantize/parallel_dequant.rs:35 From cb2d2cdb3cc30b0be1ca8b491aac1b40ee6856a8 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 25 Sep 2026 01:12:06 +0200 Subject: [PATCH 11/68] fix(paths): 0.69.4 qwen35-9b harness model path via ${APR_MODELS:-$HOME/models}, not a user home check_hardcoded_paths.sh (guard-tree, CI 36066752831) flagged 3 shipped /home/noah/models paths added by 9e7225663. Same file resolves for the original runner; APR_MODELS overrides on other hosts. Co-Authored-By: Claude Opus 5.5 (1M context) --- evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh | 2 +- evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh | 2 +- evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh index f5ebf993ed..7ef1390e50 100644 --- a/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/margin-only.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -u cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 -M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +M=${APR_MODELS:-$HOME/models}/Qwen3.5-9B-Q4_K_M.gguf gpu-q --prio 1 -- bash -c ' nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/llama.smi-inlock 2>&1 llama-server -m '"$M"' -ngl 99 -c 40960 --jinja --port 18261 -np 1 > out/llama-server-margin.log 2>&1 & sp=$! diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh index 34f478bf69..931672630c 100644 --- a/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/oracle-only.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -u cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 -M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +M=${APR_MODELS:-$HOME/models}/Qwen3.5-9B-Q4_K_M.gguf gpu-q --prio 1 -- bash -c ' nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader > out/llama.smi-inlock 2>&1 llama-server -m '"$M"' -ngl 99 -c 40960 --jinja --port 18261 -np 1 > out/llama-server.log 2>&1 & sp=$! diff --git a/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh index 90213bad14..519caefdcc 100755 --- a/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh +++ b/evidence/release/0.69.4/refs-qwen35-9b/harness/run.sh @@ -5,7 +5,7 @@ export TMPDIR=/mnt/nvme-raid0/tmp/g1-9b/tmp mkdir -p "$TMPDIR" cd /mnt/nvme-raid0/tmp/g1-9b || exit 2 B=/mnt/nvme-raid0/agent-wt/f5-bins -M=/home/noah/models/Qwen3.5-9B-Q4_K_M.gguf +M=${APR_MODELS:-$HOME/models}/Qwen3.5-9B-Q4_K_M.gguf declare -A BIN=([rc]=$B/apr-0.69.3-rc-7ff50ec2a [base]=$B/apr-0.69.1-base-eed4a959a) one() { local k="$1-9b-$2" From d585fe0f87d883517ba7ecce1b49a6d3823ff1a2 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 15:59:12 +0200 Subject: [PATCH 12/68] perf(serve): derive the APR CPU stop set once at load, not per request (#4295) apr_cpu_stop_tokens walked the whole tokenizer.json vocab on every request. The set is now derived when AprServerState is built (derive_apr_cpu_stop_tokens) and held as Arc<[u32]>; a request reads it. a_request_reads_the_stop_set_derived_at_load changes the tokenizer after the state is built and requires the load-time set. Ported onto batch/0.69.4-epic-4249 by aprender-x4233: the one conflict is the vocab scan, which #4334 factored into tokenizer_info_stop_tokens there. derive_apr_cpu_stop_tokens now calls it on the load-time tokenizer. Pmat-Ticket: PMAT-4295 Refs #4295 (cherry picked from commit deb9ff43a) Co-Authored-By: Claude Opus 5.5 --- crates/apr-cli/src/commands/serve/handlers.rs | 30 +++++++++--- .../serve/tests_apr_cpu_gen_config_4265.rs | 49 +++++++++++++------ 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/crates/apr-cli/src/commands/serve/handlers.rs b/crates/apr-cli/src/commands/serve/handlers.rs index 56f762b93f..c73a9d46e1 100644 --- a/crates/apr-cli/src/commands/serve/handlers.rs +++ b/crates/apr-cli/src/commands/serve/handlers.rs @@ -1062,6 +1062,9 @@ struct AprServerState { /// always leaves this `None`; only `build_demo_apr_cpu_router_for_test` sets /// it, so the streaming falsifier can observe real multi-chunk NDJSON. demo_scripted_tokens: Option>, + /// #4295: the stop set, derived once from the tokenizers at load + /// ([`derive_apr_cpu_stop_tokens`]), never rescanned per request. + stop_tokens: Arc<[u32]>, } /// Output from a successful APR inference run @@ -1162,13 +1165,23 @@ fn run_apr_cpu_inference( const APR_CPU_TURN_END_TOKENS: &[&str] = &["<|im_end|>", "<|endoftext|>", "<|eot_id|>", ""]; -/// #4265: the token ids that end an APR CPU generation: every EOS the loaded -/// tokenizer declares plus the chat-turn terminators it has an id for. The -/// generation loop already stops at id 0 on its own (`is_eos_token`). +/// #4265: the token ids that end an APR CPU generation, as derived at load. #[cfg(feature = "inference")] fn apr_cpu_stop_tokens(state: &AprServerState) -> Vec { + state.stop_tokens.to_vec() +} + +/// #4265: every EOS the loaded tokenizers declare plus the chat-turn +/// terminators they have an id for. The generation loop already stops at id 0 +/// on its own (`is_eos_token`). It walks the whole tokenizer.json vocab, so it +/// runs once, when the state is built (#4295). +#[cfg(feature = "inference")] +fn derive_apr_cpu_stop_tokens( + embedded_tokenizer: Option<&realizar::apr::BpeTokenizer>, + tokenizer: Option<&SafeTensorsTokenizerInfo>, +) -> Arc<[u32]> { let mut stop = Vec::new(); - if let Some(tok) = &state.embedded_tokenizer { + if let Some(tok) = embedded_tokenizer { stop.extend(tok.eos_id); stop.extend( APR_CPU_TURN_END_TOKENS @@ -1176,12 +1189,12 @@ fn apr_cpu_stop_tokens(state: &AprServerState) -> Vec { .filter_map(|t| tok.special_tokens.get(*t).copied()), ); } - if let Some(tok) = &state.tokenizer { + if let Some(tok) = tokenizer { stop.extend(tokenizer_info_stop_tokens(tok)); } stop.sort_unstable(); stop.dedup(); - stop + stop.into() } /// #4334: the stop ids a tokenizer.json-backed tokenizer declares: its EOS plus @@ -1362,6 +1375,8 @@ fn load_apr_model_state(model_path: &Path, config: &ServerConfig) -> Result Result axum::Router { embedded_tokenizer: None, model_name: "apr".to_string(), demo_scripted_tokens: None, + stop_tokens: Arc::from([]), }; build_apr_cpu_router(state, super::auth::AuthGate::disabled()) } @@ -1729,6 +1746,7 @@ pub fn build_demo_streaming_apr_cpu_router_for_test() -> axum::Router { "world".to_string(), "!".to_string(), ]), + stop_tokens: Arc::from([]), }; build_apr_cpu_router(state, super::auth::AuthGate::disabled()) } diff --git a/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs b/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs index c59302aba6..731fb9e1a6 100644 --- a/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs +++ b/crates/apr-cli/src/commands/serve/tests_apr_cpu_gen_config_4265.rs @@ -9,23 +9,31 @@ fn state_with( embedded: Option, eos_from_tokenizer_json: Option, ) -> AprServerState { + let tokenizer = eos_from_tokenizer_json.map(tokenizer_json); + let stop_tokens = derive_apr_cpu_stop_tokens(embedded.as_ref(), tokenizer.as_ref()); AprServerState { transformer: None, model_type: "test".to_string(), architecture: "test".to_string(), is_transformer: false, - tokenizer: eos_from_tokenizer_json.map(|eos| SafeTensorsTokenizerInfo { - tokenizer: std::sync::Arc::new( - realizar::tokenizer::BPETokenizer::new(vec!["".to_string()], vec![], "") - .expect("tiny tokenizer"), - ), - vocab: vec![], - bos_token_id: None, - eos_token_id: Some(eos), - }), + tokenizer, embedded_tokenizer: embedded, model_name: "apr".to_string(), demo_scripted_tokens: None, + stop_tokens, + } +} + +/// A sibling tokenizer.json with `eos` and an empty vocab. +fn tokenizer_json(eos: u32) -> SafeTensorsTokenizerInfo { + SafeTensorsTokenizerInfo { + tokenizer: std::sync::Arc::new( + realizar::tokenizer::BPETokenizer::new(vec!["".to_string()], vec![], "") + .expect("tiny tokenizer"), + ), + vocab: vec![], + bos_token_id: None, + eos_token_id: Some(eos), } } @@ -113,13 +121,24 @@ fn reply_keeps_a_budget_cut_and_an_inner_stop_id() { #[test] fn stop_set_holds_sibling_tokenizer_json_turn_end_by_vocab_index() { + let mut tok = tokenizer_json(2); + tok.vocab = ["a", "<|im_start|>", "b", "<|im_end|>"] + .iter() + .map(|t| (*t).to_string()) + .collect(); + // <|im_end|> sits at index 3; <|im_start|> (index 1) is not a stop. + assert_eq!(&*derive_apr_cpu_stop_tokens(None, Some(&tok)), &[2, 3]); +} + +/// #4295: a request reads the set derived at load; it never walks the vocab +/// again. The tokenizer changes after the state is built and the request's +/// stop set does not. +#[test] +fn a_request_reads_the_stop_set_derived_at_load() { let mut s = state_with(None, Some(2)); if let Some(tok) = s.tokenizer.as_mut() { - tok.vocab = ["a", "<|im_start|>", "b", "<|im_end|>"] - .iter() - .map(|t| (*t).to_string()) - .collect(); + tok.vocab = vec!["<|im_end|>".to_string()]; + tok.eos_token_id = Some(7); } - // <|im_end|> sits at index 3; <|im_start|> (index 1) is not a stop. - assert_eq!(apr_cpu_stop_tokens(&s), vec![2, 3]); + assert_eq!(apr_cpu_stop_tokens(&s), vec![2]); } From 4c12a0b645275d19d62a28ae0b38df8d168fd465 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 13:02:57 +0200 Subject: [PATCH 13/68] =?UTF-8?q?test(#4214):=20RED=20=E2=80=94=20a=20repe?= =?UTF-8?q?ated=20or=20re-rendered=20prompt=20resumes=20from=20the=20last?= =?UTF-8?q?=20turn's=20checkpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scripted forward: an identical prompt, a re-rendered turn 2 and turn 3 must resume at the checkpoint (3 RED); a dropped state, a lost checkpoint, a prompt diverging before it and a score must not reuse it. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 664d1cfcb46c57e5e92c61204cb67038cba4cc17) (cherry picked from commit fd2112bcdb45c6367fb6537a8ac1d1de07ac47fc) --- crates/aprender-serve/src/session.rs | 27 ++++ crates/aprender-serve/src/session_tests.rs | 140 +++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index 9f6892ddc3..4b08a668b4 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -79,6 +79,33 @@ pub trait ArchForward { Ok(()) } + /// Where to keep a copy of the state while prefilling `prompt` (#4214): + /// the state after `prompt[..k]` is what a later prompt most likely + /// repeats — a chat's history before its generation header. `None` (the + /// default): this forward keeps no copies, and a prompt that does not + /// extend what the state holds is prefilled from 0. + fn checkpoint_at(&self, _prompt: &[u32]) -> Option { + None + } + + /// Keep a copy of the state as it holds now, replacing any earlier copy. + /// + /// # Errors + /// The copy could not be made; the session then keeps none. + fn save_checkpoint(&mut self) -> Result<()> { + Ok(()) + } + + /// Return the state to the copy [`ArchForward::save_checkpoint`] kept. + /// `false` when there is none to return to (never saved, or dropped by a + /// reallocation or a fallback): the state is then unchanged. + /// + /// # Errors + /// The copy could not be put back; what the state holds is then unknown. + fn restore_checkpoint(&mut self) -> Result { + Ok(false) + } + /// The state holds `tokens[..start]` (`start == 0`: reset it). Advance it /// to hold all of `tokens` and return the logits after the last one. /// diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index 59b84cb463..c8f7b4679f 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -11,6 +11,15 @@ struct Scripted { calls: Vec<(usize, usize)>, reserves: usize, drop_on_reserve: bool, + /// Checkpoint before the last position holding this token (#4214). + marker: Option, + /// Positions the state holds, as the forward itself tracks them. + held: usize, + /// Positions the saved checkpoint holds. + saved: Option, + restores: usize, + /// The forward loses its checkpoint (a reallocation, a fallback). + lose_checkpoint: bool, } impl Scripted { @@ -21,6 +30,18 @@ impl Scripted { calls: Vec::new(), reserves: 0, drop_on_reserve: false, + marker: None, + held: 0, + saved: None, + restores: 0, + lose_checkpoint: false, + } + } + + fn checkpointing(next: u32, context: usize, marker: u32) -> Self { + Self { + marker: Some(marker), + ..Self::new(next, context) } } } @@ -45,8 +66,35 @@ impl ArchForward for Scripted { self.reserves += 1; Ok(self.drop_on_reserve) } + fn checkpoint_at(&self, prompt: &[u32]) -> Option { + let marker = self.marker?; + prompt.iter().rposition(|&t| t == marker) + } + fn save_checkpoint(&mut self) -> Result<()> { + assert!(self.marker.is_some(), "saved on a forward that keeps none"); + self.saved = Some(self.held); + Ok(()) + } + fn restore_checkpoint(&mut self) -> Result { + if self.lose_checkpoint { + self.saved = None; + } + let Some(saved) = self.saved else { + return Ok(false); + }; + self.held = saved; + self.restores += 1; + Ok(true) + } fn forward(&mut self, tokens: &[u32], start: usize) -> Result> { + // The session may only claim positions the state holds. + assert!( + start == 0 || start == self.held, + "forward from {start}, but the state holds {}", + self.held + ); self.calls.push((tokens.len(), start)); + self.held = tokens.len(); let mut logits = vec![0.0; 8]; logits[self.next as usize] = 1.0; Ok(logits) @@ -234,3 +282,95 @@ fn plain_greedy_takes_the_device_argmax_and_a_penalty_or_sampling_does_not() { "a repeat penalty needs the logits" ); } + +// #4214 / #4274: a prompt that repeats, or re-renders, an earlier turn's history +// resumes from the checkpoint the earlier turn left, not from position 0. + +const MARK: u32 = 7; + +#[test] +fn an_identical_prompt_again_restores_the_checkpoint_before_its_last_marker() { + let mut s = Session::new(Scripted::checkpointing(3, 100, MARK)); + let prompt = [7801, 7802, MARK, 7803]; + let t1 = s + .generate(&prompt, &greedy(1), &mut |_| true) + .expect("t1"); + assert_eq!(t1.reused, 0); + // Prefilled in two spans: up to the checkpoint, then the rest. + assert_eq!(s.engine().calls, vec![(2, 0), (4, 2)]); + let t2 = s + .generate(&prompt, &greedy(1), &mut |_| true) + .expect("t2"); + assert_eq!(t2.reused, 2, "the repeat prefilled only from the checkpoint"); + assert_eq!(t2.tokens, t1.tokens); + assert_eq!(s.engine().calls.last(), Some(&(4, 2))); + assert_eq!(s.engine().restores, 1); +} + +#[test] +fn a_re_rendered_history_resumes_from_the_last_turns_checkpoint() { + let mut s = Session::new(Scripted::checkpointing(3, 100, MARK)); + let t1 = s + .generate(&[7811, MARK, 7812], &greedy(2), &mut |_| true) + .expect("t1"); + assert_eq!(t1.tokens, vec![7811, MARK, 7812, 3, 3]); + // Turn 2 re-renders turn 1's reply (7899, not the 3 it generated), so it + // does not extend what the state holds. + let p2 = [7811, MARK, 7812, 7899, MARK, 7813]; + let before = s.engine().calls.len(); + let t2 = s.generate(&p2, &greedy(1), &mut |_| true).expect("t2"); + assert_eq!(t2.reused, 1, "resumed at turn 1's checkpoint, not at 0"); + assert_eq!(s.engine().calls[before..], [(5, 1), (6, 5)]); + // And turn 3 resumes at turn 2's. + let p3 = [7811, MARK, 7812, 7899, MARK, 7813, 7898, MARK, 7814]; + let t3 = s.generate(&p3, &greedy(1), &mut |_| true).expect("t3"); + assert_eq!(t3.reused, 5); +} + +#[test] +fn a_prompt_that_diverges_before_the_checkpoint_starts_over() { + let mut s = Session::new(Scripted::checkpointing(3, 100, MARK)); + s.generate(&[7821, 7822, MARK, 7823], &greedy(1), &mut |_| true) + .expect("t1"); + let t2 = s + .generate(&[7829, 7822, MARK, 7823], &greedy(1), &mut |_| true) + .expect("t2"); + assert_eq!(t2.reused, 0); + assert_eq!(s.engine().restores, 0); +} + +#[test] +fn a_checkpoint_does_not_survive_a_dropped_state() { + let mut f = Scripted::checkpointing(3, 100, MARK); + f.drop_on_reserve = true; + let mut s = Session::new(f); + let prompt = [7831, 7832, MARK, 7833]; + s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); + let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); + assert_eq!(t2.reused, 0, "reserve said the state was dropped"); + assert_eq!(s.engine().restores, 0); +} + +#[test] +fn a_checkpoint_the_forward_lost_is_not_reused() { + let mut f = Scripted::checkpointing(3, 100, MARK); + f.lose_checkpoint = true; + let mut s = Session::new(f); + let prompt = [7841, 7842, MARK, 7843]; + s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); + let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); + assert_eq!(t2.reused, 0); + assert_eq!(s.engine().calls.last(), Some(&(4, 2)), "re-checkpointed from 0"); + assert_eq!(s.engine().calls[s.engine().calls.len() - 2], (2, 0)); +} + +#[test] +fn scoring_rewrites_the_state_so_the_checkpoint_is_dropped() { + let mut s = Session::new(Scripted::checkpointing(3, 100, MARK)); + let prompt = [7851, 7852, MARK, 7853]; + s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); + s.score(&[7859, 7858], &mut |_, _| true).expect("score"); + let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); + assert_eq!(t2.reused, 0, "the positions under the checkpoint were rewritten"); + assert_eq!(s.engine().restores, 0); +} From 71d34774ad677a4ce5907322a60c1c2485871de5 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 13:28:12 +0200 Subject: [PATCH 14/68] fix(session): a repeated or re-rendered prompt resumes from a recurrent-state checkpoint (#4214, #4274) Qwen3.5's Gated DeltaNet state cannot be rolled back, so a prompt that did not strictly extend the state (the same prompt again, #4214; serve's re-rendered chat history, #4274) prefilled from position 0. The prompt is now prefilled in two spans, and the forward keeps a copy of the state between them, before the prompt's last <|im_start|> (else before its last token). A later prompt that repeats the copied tokens restores the conv/ssm state (CPU clone; device D2D on the compute stream), truncates KV to the copied rows and prefills from there. The copy is dropped whenever the positions under it are rewritten: a reallocation, a GPU fallback, a forward from below it (score, a diverging prompt) or a failed forward. CPU: 21/21 session tests, incl. the 0.8B model: identical repeat resumes at the generation header and a re-rendered turn 2 at turn 1's, token-identical to the one-shot at temp 0. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 629ef71c1285c185414e300a2c39a90e6a8baa8a) (cherry picked from commit 7d1b3af6522f7d29fea2d8c36e241fcddf0bfb6e) --- .../src/gguf/cuda/forward_qwen35_cuda.rs | 106 ++++++++++++++++++ crates/aprender-serve/src/gguf/cuda/mod.rs | 2 +- .../gguf/inference/forward/forward_qwen35.rs | 44 ++++++++ .../gguf/inference/forward/qwen35_session.rs | 77 +++++++++++++ .../inference/forward/qwen35_session_tests.rs | 87 ++++++++++++++ crates/aprender-serve/src/gguf/runtime.rs | 6 + crates/aprender-serve/src/session.rs | 74 +++++++++++- crates/aprender-serve/src/session_tests.rs | 9 +- 8 files changed, 398 insertions(+), 7 deletions(-) diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs index b7ed0513ba..f709b3a3d3 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -118,6 +118,16 @@ pub struct Qwen35CudaState { kv_len: usize, } +/// A copy of a [`Qwen35CudaState`]'s recurrent half at one position (#4214), +/// on the device: each Gated `DeltaNet` layer's conv window and recurrent +/// state, plus how many KV rows the state held. The KV rows stay in the +/// state; a later position writes only rows past them. +pub struct Qwen35CudaCheckpoint { + /// `(layer, conv, ssm)` for every `DeltaNet` layer. + layers: Vec<(usize, GpuBuffer, GpuBuffer)>, + kv_len: usize, +} + impl Qwen35CudaState { /// Elements in one layer's causal-conv window. #[must_use] @@ -797,6 +807,102 @@ impl<'a> Qwen35CudaModel<'a> { Ok(()) } + /// Copy `state`'s recurrent half into `into`, reusing its buffers when it + /// already holds a copy of this model's shape (#4214). + /// + /// # Errors + /// Any CUDA allocation, copy or synchronization failure. + pub fn save_checkpoint( + &self, + state: &Qwen35CudaState, + into: &mut Option, + ) -> Result<()> { + let stream = self.executor.compute_stream(); + // The copy runs on the compute stream, after every launch that wrote the state. + let deltanet: Vec = (0..state.kv.len()) + .filter(|&i| state.kv[i].is_none()) + .collect(); + let reusable = into.as_ref().is_some_and(|c| { + c.layers.len() == deltanet.len() + && c.layers.iter().zip(&deltanet).all(|((l, conv, ssm), &i)| { + *l == i && conv.len() == state.conv_len && ssm.len() == state.ssm_len + }) + }); + if !reusable { + *into = None; + let mut layers = Vec::with_capacity(deltanet.len()); + for &i in &deltanet { + layers.push(( + i, + Self::zeros(&self.executor, state.conv_len)?, + Self::zeros(&self.executor, state.ssm_len)?, + )); + } + *into = Some(Qwen35CudaCheckpoint { layers, kv_len: 0 }); + } + let ckpt = into + .as_mut() + .ok_or_else(|| RealizarError::InvalidShape { + reason: "qwen35_cuda: the checkpoint was never allocated".to_string(), + })?; + for (i, conv, ssm) in &mut ckpt.layers { + // SAFETY: source and destination are live device buffers of equal + // length (checked by the copy) owned by `state` and `ckpt`, which + // both outlive the synchronize below. + unsafe { + conv.copy_from_buffer_async(&state.conv[*i], stream) + .map_err(|e| gpu_err("qwen35_cuda_checkpoint", &e))?; + ssm.copy_from_buffer_async(&state.ssm[*i], stream) + .map_err(|e| gpu_err("qwen35_cuda_checkpoint", &e))?; + } + } + stream + .synchronize() + .map_err(|e| gpu_err("qwen35_cuda_checkpoint", &e))?; + ckpt.kv_len = state.kv_len; + Ok(()) + } + + /// Return `state` to `ckpt`. `false` (the state unchanged) when the state + /// no longer holds the KV rows the copy was taken over, or is shaped + /// differently. + /// + /// # Errors + /// Any CUDA copy or synchronization failure; the state is then unknown. + pub fn restore_checkpoint( + &self, + ckpt: &Qwen35CudaCheckpoint, + state: &mut Qwen35CudaState, + ) -> Result { + let fits = state.kv_len >= ckpt.kv_len + && ckpt.kv_len <= state.max_seq_len + && ckpt.layers.iter().all(|(i, conv, ssm)| { + state.kv.get(*i).is_some_and(Option::is_none) + && conv.len() == state.conv_len + && ssm.len() == state.ssm_len + }); + if !fits { + return Ok(false); + } + let stream = self.executor.compute_stream(); + for (i, conv, ssm) in &ckpt.layers { + // SAFETY: as in `save_checkpoint`, with the roles swapped. + unsafe { + state.conv[*i] + .copy_from_buffer_async(conv, stream) + .map_err(|e| gpu_err("qwen35_cuda_restore", &e))?; + state.ssm[*i] + .copy_from_buffer_async(ssm, stream) + .map_err(|e| gpu_err("qwen35_cuda_restore", &e))?; + } + } + stream + .synchronize() + .map_err(|e| gpu_err("qwen35_cuda_restore", &e))?; + state.kv_len = ckpt.kv_len; + Ok(true) + } + /// Run `f` with the model's own state detached. /// /// The layer bodies take `&mut Qwen35CudaState` so a caller can drive an diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index 14f99f7b43..9b4b4d40b8 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -47,7 +47,7 @@ mod weights; pub use backend::CudaBackend; // PMAT-3477 (#3090): the Gated DeltaNet GPU model and its device state. pub use forward_qwen35_cuda::{ - PrefillAttention, Qwen35CudaModel, Qwen35CudaState, PREFILL_MAX_CHUNK_ROWS, + PrefillAttention, Qwen35CudaCheckpoint, Qwen35CudaModel, Qwen35CudaState, PREFILL_MAX_CHUNK_ROWS, UNIFIED_PREFILL_CHUNK_ROWS, }; // #3714: the Qwen3-MoE GPU model, its device state, and the MoE shape it is built for. diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs index 6db6560018..6c7285684d 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs @@ -479,6 +479,50 @@ impl Qwen35State { } } +/// A copy of a [`Qwen35State`]'s recurrent half at one position (#4214): the +/// conv windows and recurrent states, which cannot be rolled back, plus how +/// many KV rows it held — the rows themselves stay in the state, where a later +/// position never overwrites them. +#[derive(Debug, Clone)] +pub struct Qwen35Checkpoint { + conv_states: Vec>, + ssm_states: Vec>, + kv_len: usize, +} + +impl Qwen35State { + /// Copy the state as it holds now. + #[must_use] + pub fn checkpoint(&self) -> Qwen35Checkpoint { + Qwen35Checkpoint { + conv_states: self.conv_states.clone(), + ssm_states: self.ssm_states.clone(), + kv_len: self.kv_cache.len(), + } + } + + /// Return to `checkpoint`. `false` (the state unchanged) when the state no + /// longer holds the KV rows the copy was taken over. + pub fn restore(&mut self, checkpoint: &Qwen35Checkpoint) -> bool { + if self.kv_cache.len() < checkpoint.kv_len + || self.conv_states.len() != checkpoint.conv_states.len() + || self.ssm_states.len() != checkpoint.ssm_states.len() + { + return false; + } + for (dst, src) in self + .conv_states + .iter_mut() + .zip(&checkpoint.conv_states) + .chain(self.ssm_states.iter_mut().zip(&checkpoint.ssm_states)) + { + dst.copy_from_slice(src); + } + self.kv_cache.truncate(checkpoint.kv_len); + true + } +} + pub(crate) struct Qwen35OwnedDeltaNetLayer { pub(crate) attn_norm: Vec, pub(crate) attn_qkv: OwnedQuantizedTensor, diff --git a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session.rs b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session.rs index eb3c018a36..921e2fc4f5 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session.rs @@ -211,6 +211,14 @@ fn plan_capacity( } } +/// The copy of the decode state a session resumes from (#4214), on the backend +/// that took it. +enum Checkpoint { + #[cfg(feature = "cuda")] + Gpu(crate::gguf::cuda::Qwen35CudaCheckpoint), + Cpu(crate::gguf::forward_qwen35::Qwen35Checkpoint), +} + /// Where a session runs its forward. enum Backend { #[cfg(feature = "cuda")] @@ -295,6 +303,12 @@ pub struct Qwen35Forward { /// Prompts the GPU prefilled in one batched call — the evidence that `apr /// serve` took the batched path, not a speed that merely looks like it. batched_prefills: usize, + /// The saved copy of the decode state (#4214); dropped whenever the state + /// it was taken from is reallocated or the backend changes. + checkpoint: Option, + /// The `<|im_start|>` token, when the vocabulary has one: a chat prompt's + /// checkpoint goes just before its last one (the generation header). + im_start: Option, } impl Qwen35Forward { @@ -379,6 +393,11 @@ impl Qwen35Forward { format!("[qwen35] {QWEN35_SESSION_PREFILL_ENV}=per-token: prompts prefill one token at a time, not batched"), ); } + let im_start = mapped + .model + .vocabulary() + .and_then(|v| v.iter().position(|t| t == "<|im_start|>")) + .and_then(|i| u32::try_from(i).ok()); let route = qwen35_route(no_gpu, cfg!(feature = "cuda")); if let Some(notice) = qwen35_route_notice(route) { say(&mut notices, notice.to_string()); @@ -416,6 +435,8 @@ impl Qwen35Forward { notices, per_token_prefill, batched_prefills: 0, + checkpoint: None, + im_start, }) } @@ -612,6 +633,7 @@ impl Qwen35Forward { .min(self.context_length) .max(positions); self.allocations += 1; + self.checkpoint = None; let qwen = self.qwen; match &mut self.backend { #[cfg(feature = "cuda")] @@ -635,6 +657,7 @@ impl Qwen35Forward { fn fall_back_to_cpu(&mut self, reason: &str) -> Result<()> { say(&mut self.notices, fallback_line(reason)); self.backend = Backend::Cpu(None); + self.checkpoint = None; self.capacity = 0; match self.ensure_capacity() { Ok(()) => Ok(()), @@ -680,6 +703,60 @@ impl crate::session::ArchForward for Qwen35Forward { Ok(self.allocations != before) } + /// Before the prompt's last `<|im_start|>` — a chat's history, up to the + /// generation header — else before its last token. + fn checkpoint_at(&self, prompt: &[u32]) -> Option { + self.im_start + .and_then(|id| prompt.iter().rposition(|&t| t == id)) + .filter(|&k| k > 0) + .or_else(|| prompt.len().checked_sub(1)) + } + + fn save_checkpoint(&mut self) -> Result<()> { + let saved = match &mut self.backend { + #[cfg(feature = "cuda")] + Backend::Gpu(gpu) => { + let Some(state) = gpu.state.as_ref() else { + self.checkpoint = None; + return Ok(()); + }; + let mut slot = match self.checkpoint.take() { + Some(Checkpoint::Gpu(c)) => Some(c), + _ => None, + }; + gpu.model.save_checkpoint(state, &mut slot)?; + slot.map(Checkpoint::Gpu) + }, + Backend::Cpu(state) => state + .as_ref() + .map(|state| Checkpoint::Cpu(state.checkpoint())), + }; + self.checkpoint = saved; + Ok(()) + } + + /// A device copy that fails is a GPU failure: the session moves to the CPU, + /// loudly, and the turn prefills from 0 there. + fn restore_checkpoint(&mut self) -> Result { + let restored = match (&mut self.backend, &self.checkpoint) { + #[cfg(feature = "cuda")] + (Backend::Gpu(gpu), Some(Checkpoint::Gpu(c))) => match gpu.state.as_mut() { + Some(state) => gpu.model.restore_checkpoint(c, state), + None => Ok(false), + }, + (Backend::Cpu(Some(state)), Some(Checkpoint::Cpu(c))) => Ok(state.restore(c)), + _ => Ok(false), + }; + match restored { + Ok(restored) => Ok(restored), + Err(e) if self.on_gpu() => { + self.fall_back_to_cpu(&format!("the device checkpoint would not restore: {e}"))?; + Ok(false) + }, + Err(e) => Err(e), + } + } + fn validate(&mut self, probe: &[u32]) -> Result<()> { #[cfg(feature = "cuda")] match self.validate_gpu_once(probe) { diff --git a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs index 87297fbec4..e65226e97b 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs @@ -179,6 +179,60 @@ fn on_token_returning_false_ends_the_turn_after_that_token() { ); } +/// Where a chat prompt's checkpoint goes: before its last `<|im_start|>`. +fn header_at(mapped: &MappedGGUFModel, prompt: &[u32]) -> usize { + let im_start = encode(mapped, "<|im_start|>"); + assert_eq!(im_start.len(), 1, "<|im_start|> is one token"); + prompt + .iter() + .rposition(|&t| t == im_start[0]) + .expect("a chat prompt holds <|im_start|>") +} + +/// Turn 2 as a chat server renders it: turn 1's reply re-rendered from text +/// (here, not the tokens turn 1 generated), then the new user turn. +fn re_rendered_turn_two(mapped: &MappedGGUFModel, p1: &[u32]) -> Vec { + let mut p2 = p1.to_vec(); + p2.extend(encode( + mapped, + &format!("The capital of Peru is Lima.<|im_end|>\n{}", user_turn("And of Chile?")), + )); + p2 +} + +#[test] +fn cpu_an_identical_prompt_again_resumes_at_its_generation_header_and_matches_one_shot() { + // #4214: the same prompt twice cost the whole prefill twice. + let mapped = mapped_or_skip!(); + let mut session = Qwen35Session::load(&mapped, true).expect("load"); + let config = greedy(6); + let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); + let t1 = session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + let t2 = session.generate(&p1, &config, &mut |_| true).expect("turn 2"); + assert_eq!(t2.reused, header_at(&mapped, &p1), "resumed at the checkpoint"); + assert!(t2.reused > 0); + assert_eq!(t2.tokens, t1.tokens); + assert_eq!(t2.tokens, one_shot_cpu(&mapped, &p1, &config)); +} + +#[test] +fn cpu_a_re_rendered_turn_two_resumes_at_turn_ones_header_and_matches_one_shot() { + // #4274: serve re-renders the history, so turn 2 never extended turn 1. + let mapped = mapped_or_skip!(); + let mut session = Qwen35Session::load(&mapped, true).expect("load"); + let config = greedy(6); + let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); + session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + let p2 = re_rendered_turn_two(&mapped, &p1); + let t2 = session.generate(&p2, &config, &mut |_| true).expect("turn 2"); + assert_eq!(t2.reused, header_at(&mapped, &p1), "resumed at turn 1's checkpoint"); + assert_eq!( + t2.tokens, + one_shot_cpu(&mapped, &p2, &config), + "a resumed state must decode exactly what a fresh one does" + ); +} + #[cfg(feature = "cuda")] mod gpu { use super::*; @@ -378,6 +432,39 @@ mod gpu { assert_eq!(t2.reused, 0); assert_eq!(t2.tokens, one_shot_gpu(&mapped, &p2, &config)); } + + #[test] + fn gpu_an_identical_prompt_again_resumes_at_its_generation_header_and_matches_one_shot() { + let mapped = mapped_or_skip!(); + let Some(mut session) = gpu_session_or_skip(&mapped) else { + return; + }; + let config = greedy(6); + let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); + let t1 = session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + let t2 = session.generate(&p1, &config, &mut |_| true).expect("turn 2"); + assert!(t1.used_gpu && t2.used_gpu, "no fallback"); + assert_eq!(t2.reused, header_at(&mapped, &p1)); + assert!(t2.reused > 0); + assert_eq!(t2.tokens, t1.tokens); + assert_eq!(t2.tokens, one_shot_gpu(&mapped, &p1, &config)); + } + + #[test] + fn gpu_a_re_rendered_turn_two_resumes_at_turn_ones_header_and_matches_one_shot() { + let mapped = mapped_or_skip!(); + let Some(mut session) = gpu_session_or_skip(&mapped) else { + return; + }; + let config = greedy(6); + let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); + session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + let p2 = re_rendered_turn_two(&mapped, &p1); + let t2 = session.generate(&p2, &config, &mut |_| true).expect("turn 2"); + assert!(t2.used_gpu, "no fallback"); + assert_eq!(t2.reused, header_at(&mapped, &p1)); + assert_eq!(t2.tokens, one_shot_gpu(&mapped, &p2, &config)); + } } // #4255: the two #4247 branches no GPU test reaches — the prefill that does not diff --git a/crates/aprender-serve/src/gguf/runtime.rs b/crates/aprender-serve/src/gguf/runtime.rs index a63dbb708c..6d70e09ae2 100644 --- a/crates/aprender-serve/src/gguf/runtime.rs +++ b/crates/aprender-serve/src/gguf/runtime.rs @@ -249,6 +249,12 @@ impl OwnedQuantizedKVCache { self.seq_len = (self.seq_len + n).min(self.max_seq_len); } + /// Keep only the first `new_len` positions, by the row width the cache was + /// built with (#4214: a session resuming from a checkpoint). + pub fn truncate(&mut self, new_len: usize) { + self.rollback_to(new_len, self._hidden_dim); + } + /// PAR-098: Rollback cache to a previous position (for speculative decode rejection) /// /// When draft tokens are rejected, we need to remove their K/V entries. diff --git a/crates/aprender-serve/src/session.rs b/crates/aprender-serve/src/session.rs index 4b08a668b4..af45637287 100644 --- a/crates/aprender-serve/src/session.rs +++ b/crates/aprender-serve/src/session.rs @@ -156,6 +156,9 @@ pub struct Session { /// call entered, so a guard can ask "did THIS verb's session serve it?" /// without re-deriving the verb's prompt tokens. id: u64, + /// The tokens whose forward the forward's saved copy holds (#4214), or + /// `None` when it keeps no copy the session trusts. + checkpoint: Option>, } static NEXT_SESSION_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); @@ -167,6 +170,7 @@ impl Session { forward, processed: Vec::new(), id: NEXT_SESSION_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), + checkpoint: None, } } @@ -238,15 +242,75 @@ impl Session { } else { 0 }; + Ok((self.step(tokens, start)?, start)) + } + + /// Bring the state up to a strict prefix of `prompt` that + /// [`Self::advance_and_choose`] then extends with the rest: resume from the + /// checkpoint an earlier turn left when `prompt` does not extend the state + /// but repeats what the checkpoint holds (#4214: the same prompt again; + /// #4274: a chat history re-rendered with the last reply changed), and + /// prefill up to a new checkpoint where [`ArchForward::checkpoint_at`] says. + /// Returns how many leading tokens of `prompt` were already held. + fn prepare_prompt(&mut self, prompt: &[u32]) -> Result { + let mut start = if self.extends(prompt) { + self.processed.len() + } else { + 0 + }; + if start == 0 { + if let Some(held) = self.checkpoint.take() { + if prompt.len() > held.len() && prompt.starts_with(&held) { + match self.forward.restore_checkpoint() { + Ok(true) => { + start = held.len(); + self.processed.clone_from(&held); + self.checkpoint = Some(held); + }, + Ok(false) => {}, + Err(e) => { + self.processed.clear(); + return Err(e); + }, + } + } else { + self.checkpoint = Some(held); + } + } + } + let reused = start; + if let Some(k) = self + .forward + .checkpoint_at(prompt) + .filter(|&k| k > start && k < prompt.len()) + { + self.step(&prompt[..k], start)?; + self.checkpoint = match self.forward.save_checkpoint() { + Ok(()) => Some(prompt[..k].to_vec()), + Err(_) => None, + }; + } + Ok(reused) + } + + /// Forward `tokens` from `start` (the state holds `tokens[..start]`) and + /// record what the state then holds. + fn step(&mut self, tokens: &[u32], start: usize) -> Result> { + // Positions under the checkpoint are about to be rewritten: the copy + // no longer matches the attention rows it would resume on. + if self.checkpoint.as_ref().is_some_and(|c| start < c.len()) { + self.checkpoint = None; + } match self.forward.forward(tokens, start) { Ok(logits) => { self.processed.truncate(start); self.processed.extend_from_slice(&tokens[start..]); - Ok((logits, start)) + Ok(logits) }, Err(e) => { // What the state holds after a failed forward is unknown. self.processed.clear(); + self.checkpoint = None; Err(e) }, } @@ -268,6 +332,9 @@ impl Session { } else { 0 }; + if self.checkpoint.as_ref().is_some_and(|c| start < c.len()) { + self.checkpoint = None; + } match self.forward.forward_greedy(tokens, start) { Ok(Some(next)) => { self.processed.truncate(start); @@ -277,6 +344,7 @@ impl Session { Ok(None) => {}, Err(e) => { self.processed.clear(); + self.checkpoint = None; return Err(e); }, } @@ -294,6 +362,7 @@ impl Session { fn reserve(&mut self, positions: usize) -> Result<()> { if self.forward.reserve(positions)? { self.processed.clear(); + self.checkpoint = None; } Ok(()) } @@ -348,8 +417,9 @@ impl Session { turn_budget(prompt.len(), config.max_tokens, context_length); self.reserve(prompt.len() + budget)?; + let reused = self.prepare_prompt(prompt)?; let mut rng = rand::rngs::StdRng::seed_from_u64(config.seed); - let (mut next, reused) = self.advance_and_choose(prompt, config, &mut rng)?; + let (mut next, _) = self.advance_and_choose(prompt, config, &mut rng)?; let mut tokens = prompt.to_vec(); let mut context_capped = false; for generated in 1..=budget { diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index c8f7b4679f..fe51d82501 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -318,13 +318,14 @@ fn a_re_rendered_history_resumes_from_the_last_turns_checkpoint() { // does not extend what the state holds. let p2 = [7811, MARK, 7812, 7899, MARK, 7813]; let before = s.engine().calls.len(); - let t2 = s.generate(&p2, &greedy(1), &mut |_| true).expect("t2"); + let t2 = s.generate(&p2, &greedy(2), &mut |_| true).expect("t2"); assert_eq!(t2.reused, 1, "resumed at turn 1's checkpoint, not at 0"); - assert_eq!(s.engine().calls[before..], [(5, 1), (6, 5)]); - // And turn 3 resumes at turn 2's. + // From the checkpoint to turn 2's own (its last marker), then the rest. + assert_eq!(s.engine().calls[before..before + 2], [(4, 1), (6, 4)]); + // Turn 3 re-renders turn 2's reply too, and resumes at turn 2's checkpoint. let p3 = [7811, MARK, 7812, 7899, MARK, 7813, 7898, MARK, 7814]; let t3 = s.generate(&p3, &greedy(1), &mut |_| true).expect("t3"); - assert_eq!(t3.reused, 5); + assert_eq!(t3.reused, 4); } #[test] From e6004d7178e5a0e0d1017e006f12ef41ac79cc70 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 14:33:34 +0200 Subject: [PATCH 15/68] test(#4214): a prompt prefills in two batched spans, split at its generation header The checkpoint is taken at the last <|im_start|>, so turn 1 is two batched prefills and turn 2's suffix two more. Measured on lambda RTX 4090 under gpu-q: the counts were 2 and 4, and the tokens are still identical to the one-token path and to the one-shot decode. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 082586fef1223d3447da024c77c1c6fa8e5abc9a) (cherry picked from commit 00108667d19594c08e75d68b7a300d6fee807cf9) --- .../src/gguf/inference/forward/qwen35_session_tests.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs index e65226e97b..cbe754a71d 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs @@ -371,10 +371,12 @@ mod gpu { let o1 = one_token .generate(&p1, &config, &mut |_| true) .expect("turn 1"); + // #4214: a prompt is prefilled in two spans, split at its generation header so the + // checkpoint can be taken there; each span goes through the batched prefill. assert_eq!( batched.batched_prefills(), - 1, - "turn 1's prompt went through the batched prefill" + 2, + "both spans of turn 1's prompt went through the batched prefill" ); assert_eq!( one_token.batched_prefills(), @@ -398,8 +400,8 @@ mod gpu { assert_eq!(b2.reused, b1.tokens.len() - 1, "turn 2 extended the state"); assert_eq!( batched.batched_prefills(), - 2, - "turn 2's new suffix went through the batched prefill, from a nonzero position" + 4, + "both spans of turn 2's new suffix went through the batched prefill, from a nonzero position" ); assert!(b2.used_gpu && o2.used_gpu, "neither fell back to the CPU"); assert_eq!(b2.tokens, o2.tokens, "turn 2: batched == one-token"); From cbc571aaa7f31974cc77d2d4d58e53dce35f7e9c Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 16:18:02 +0200 Subject: [PATCH 16/68] fix(serve): /v1/effective-config agrees with /health on the Qwen3.5 hybrid (#4254) On the v0.69.1 CPU asset serving Qwen3.5-4B, /health said compute_mode "cpu" while /v1/effective-config said compute_class "unknown", backend_loaded [], build_features [..., "gpu"], quantization null, parameter_count null and build_commit null. Each field now reads what the process actually has: - backend_loaded / compute_class count the resident Qwen3.5 session (cuda when it is on the GPU, cpu otherwise), the same predicate /health reads. - build_features drops "gpu"/"cuda" when the CLI that launched the server was built without wgpu/cuda: realizar compiles wgpu by default, but apr serve cannot dispatch to it, and --list-devices already said so. - the hybrid's model source carries a quantization (dominant_quantization's rule, read off the GGUF tensor table: modal qtype over attn_output/FFN projections) and a parameter count (sum of tensor elements). The dense CPU and CUDA paths carry the parameter count too. - build.rs reads HEAD from .git directly (worktrees, packed-refs) when `git rev-parse` is unavailable, before falling back to .git-sha. Tests: dispatchable_build_features rows; a real-model Qwen3.5-0.8B row that both routes report cpu on one state (mutating the session read out of backend_loaded reproduces the #4254 body exactly and fails it); the qwen35 serve source carries Q4_K and a 0.5-1.5B parameter count. Refs #4254, #4252 (PMAT-4252) Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 144c5e9fb372bbd5a272dbf9b39cd4bebd046c77) (cherry picked from commit 1203d0371edec02a64c85473c02d6bb7f29fb25a) --- crates/apr-cli/build.rs | 67 ++++++++++++++ .../commands/serve/handler_gpu_completion.rs | 7 +- crates/apr-cli/src/commands/serve/server.rs | 79 ++++++++++++++++- .../src/api/effective_config.rs | 87 +++++++++++++++++-- .../api/tests/effective_config_route_pp2.rs | 39 +++++++++ 5 files changed, 271 insertions(+), 8 deletions(-) diff --git a/crates/apr-cli/build.rs b/crates/apr-cli/build.rs index 4667107f37..a37b3e00d7 100644 --- a/crates/apr-cli/build.rs +++ b/crates/apr-cli/build.rs @@ -2,6 +2,9 @@ // Resolve APR_GIT_SHA with a multi-source fallback hierarchy: // 1. APR_GIT_SHA_OVERRIDE env var (CI/release hook) // 2. git rev-parse --short HEAD (dev builds from worktree or primary checkout) +// 2b. the SHA read straight from `.git` — no git binary (#4254: the release asset +// said "v0.69.1+no-git" because `git` fails inside the sibling build container +// over a bind-mounted checkout, while the checkout's `.git` is right there) // 3. committed .git-sha file (crates.io installs) // 4. "v{CARGO_PKG_VERSION}+no-git" (informative fallback — never bare "unknown") fn main() { @@ -63,6 +66,11 @@ fn resolve_git_sha() -> String { return sha; } + // 2b. Read the checkout's own `.git` when the git binary cannot answer (#4254) + if let Some(sha) = read_head_sha_from_dot_git() { + return sha; + } + // 3. Committed .git-sha fallback (updated by release script at publish time) if let Ok(contents) = std::fs::read_to_string(".git-sha") { let s = contents.trim().to_string(); @@ -75,3 +83,62 @@ fn resolve_git_sha() -> String { let version = std::env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "0.0.0".to_string()); format!("v{version}+no-git") } + +/// #4254: the commit HEAD names, read from `.git` without the git binary. +/// +/// Walks up from `CARGO_MANIFEST_DIR` to the first `.git`, which is either the git +/// directory or, in a worktree, a file `gitdir: ` (whose `commondir` holds the +/// shared refs). HEAD is a bare SHA when detached (a tag checkout) or `ref: `, +/// resolved as a loose ref, then through `packed-refs`. Short form, 9 hex digits. +fn read_head_sha_from_dot_git() -> Option { + use std::path::{Path, PathBuf}; + let start = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").ok()?); + let dot_git = start + .ancestors() + .map(|d| d.join(".git")) + .find(|p| p.exists())?; + let git_dir: PathBuf = if dot_git.is_file() { + let text = std::fs::read_to_string(&dot_git).ok()?; + let rel = text.trim().strip_prefix("gitdir:")?.trim(); + let p = Path::new(rel); + if p.is_absolute() { + p.to_path_buf() + } else { + dot_git.parent()?.join(p) + } + } else { + dot_git + }; + let common_dir = match std::fs::read_to_string(git_dir.join("commondir")) { + Ok(c) => { + let p = Path::new(c.trim()); + if p.is_absolute() { + p.to_path_buf() + } else { + git_dir.join(p) + } + } + Err(_) => git_dir.clone(), + }; + let head = std::fs::read_to_string(git_dir.join("HEAD")).ok()?; + let head = head.trim(); + let full = match head.strip_prefix("ref:") { + None => head.to_string(), + Some(name) => { + let name = name.trim(); + [&git_dir, &common_dir] + .iter() + .find_map(|d| std::fs::read_to_string(d.join(name)).ok()) + .map(|s| s.trim().to_string()) + .or_else(|| { + let packed = std::fs::read_to_string(common_dir.join("packed-refs")).ok()?; + packed.lines().find_map(|l| { + let (sha, r) = l.split_once(' ')?; + (r.trim() == name).then(|| sha.to_string()) + }) + })? + } + }; + let is_sha = full.len() >= 40 && full.bytes().all(|b| b.is_ascii_hexdigit()); + is_sha.then(|| full[..9].to_string()) +} diff --git a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs index 8e83f6b5b9..a5d45927f4 100644 --- a/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs +++ b/crates/apr-cli/src/commands/serve/handler_gpu_completion.rs @@ -609,7 +609,7 @@ fn start_gguf_server_cuda( "{}", "qwen3moe: serving through the MoE CUDA dispatch (#3714, #3987)".cyan() ); - let model_source = measured_model_source(&quantized_model, config); + let model_source = measured_model_source(&quantized_model, Some(&mapped_model), config); let state = AppState::with_quantized_model_and_vocab(quantized_model, vocab) .map_err(|e| CliError::InferenceFailed(format!("Failed to create state: {e}")))? .with_model_source(model_source) @@ -632,6 +632,10 @@ fn start_gguf_server_cuda( ); println!(" Max sequence length: {max_seq_len}"); + // #4254: measured before the model moves into CUDA. Without it `/v1/effective-config` + // on a CUDA server reported quantization, path and parameter count all null. + let model_source = measured_model_source(&quantized_model, Some(&mapped_model), config); + match OwnedQuantizedModelCuda::with_max_seq_len(quantized_model, 0, max_seq_len) { Ok(mut cuda_model) => { preload_gpu_weights(&mut cuda_model); @@ -649,6 +653,7 @@ fn start_gguf_server_cuda( let state = AppState::with_cuda_model_and_vocab(cuda_model, vocab) .map_err(|e| CliError::InferenceFailed(format!("Failed to create state: {e}")))? .with_mapped_gguf_model(mapped_model.clone()) + .with_model_source(model_source) // PP-14/PP-15/§9 #8: the resolved offload AND this binary's // `cfg!` feature list (including `cuda-batch`) reach the served // process, so a receipt records what the build was rather than diff --git a/crates/apr-cli/src/commands/serve/server.rs b/crates/apr-cli/src/commands/serve/server.rs index 480b9ebd94..90366ef3bf 100644 --- a/crates/apr-cli/src/commands/serve/server.rs +++ b/crates/apr-cli/src/commands/serve/server.rs @@ -37,6 +37,7 @@ fn dominant_quantization(model: &realizar::gguf::OwnedQuantizedModel) -> Option< #[cfg(feature = "inference")] fn measured_model_source( model: &realizar::gguf::OwnedQuantizedModel, + mapped: Option<&realizar::gguf::MappedGGUFModel>, config: &ServerConfig, ) -> realizar::api::ModelSourceInfo { let base = config @@ -52,9 +53,50 @@ fn measured_model_source( if let Some(quantization) = dominant_quantization(model) { source = source.with_quantization(quantization); } + // #4254: counted from the file's own tensor table, which is the whole model. The + // owned model is not: the Qwen3.5 hybrid reaches the servers as its BASE (#3571). + if let Some(mapped) = mapped { + source = source.with_parameter_count(gguf_parameter_count(&mapped.model)); + } source } +/// #4254: [`dominant_quantization`]'s rule, read off the GGUF's own tensor table: the +/// modal qtype across every layer's attention-output and FFN projections. The Qwen3.5 +/// hybrid has no owned model to ask, so the tensor table is the only measurement it has. +/// The embedding is left out on both paths — on a 0.8B model its Q6_K would outweigh +/// every Q4_K projection, and it is not what "the model's quantization" names. +#[cfg(feature = "inference")] +fn gguf_dominant_quantization(model: &realizar::gguf::GGUFModel) -> Option<&'static str> { + use std::collections::HashMap; + + const PROJECTIONS: [&str; 4] = [ + ".attn_output.weight", + ".ffn_up.weight", + ".ffn_down.weight", + ".ffn_gate.weight", + ]; + let mut counts: HashMap = HashMap::new(); + for t in model + .tensors + .iter() + .filter(|t| t.name.starts_with("blk.") && PROJECTIONS.iter().any(|p| t.name.ends_with(p))) + { + *counts.entry(t.qtype).or_insert(0) += 1; + } + counts + .into_iter() + .filter_map(|(qtype, n)| realizar::api::gguf_qtype_name(qtype).map(|name| (name, n))) + .max_by_key(|&(_, n)| n) + .map(|(name, _)| name) +} + +/// #4254: every element of every tensor the GGUF declares. +#[cfg(feature = "inference")] +fn gguf_parameter_count(model: &realizar::gguf::GGUFModel) -> u64 { + model.tensors.iter().map(|t| t.dims.iter().product::()).sum() +} + /// Run the CPU inference server /// /// `mapped_model` is `None` for non-GGUF formats (APR / SafeTensors). For @@ -76,7 +118,7 @@ fn run_cpu_server( // Measure the model BEFORE it is moved into AppState. Anything not // measurable here stays absent — `/realize/model` no longer substitutes // `size_bytes: 0` / `context_length: 4096` / `quantization: "Q4_K_M"`. - let model_source = measured_model_source(&quantized_model, config); + let model_source = measured_model_source(&quantized_model, mapped_model.as_deref(), config); let mut state = AppState::with_quantized_model_and_vocab(quantized_model, vocab) .map_err(|e| CliError::InferenceFailed(format!("Failed to create app state: {e}")))? @@ -180,7 +222,13 @@ fn build_qwen35_state( .unwrap_or_default() .with_architecture("qwen35") .with_model_max_context_length(context_length) - .with_context_length(config.context_length); + .with_context_length(config.context_length) + .with_parameter_count(gguf_parameter_count(&mapped_model.model)); + // #4254: the v0.69.1 CPU asset served this model with `quantization: null`. + let model_source = match gguf_dominant_quantization(&mapped_model.model) { + Some(q) => model_source.with_quantization(q), + None => model_source, + }; let state = AppState::with_qwen35_session(session, mapped_model, vocab) .map_err(|e| CliError::InferenceFailed(format!("Failed to create app state: {e}")))? .with_model_source(model_source) @@ -247,6 +295,33 @@ mod qwen35_serve_route_tests { assert!(layers > 0, "the session holds the hybrid's layers: {layers}"); } + /// #4254: the v0.69.1 CPU asset served Qwen3.5 with `quantization: null` and + /// `parameter_count: null` in `/v1/effective-config` — the hybrid path built its + /// source from the path alone. Both are now read off the GGUF's own tensor table. + #[test] + fn qwen35_serve_source_carries_quantization_and_parameter_count_4254() { + if !Path::new(MODEL).exists() { + eprintln!("SKIP: {MODEL} is absent"); + return; + } + let mapped = std::sync::Arc::new( + realizar::gguf::MappedGGUFModel::from_path(MODEL).expect("map the GGUF"), + ); + let config = ServerConfig { + no_gpu: true, + ..ServerConfig::default() + }; + let state = build_qwen35_state(mapped, &config).expect("qwen35 serve passes"); + let body = realizar::api::effective_config(&state); + // A Q4_K_M file holds most of its weight elements in Q4_K (some + // projections are Q6_K); "Q4_K_M" is a file label, not a tensor type. + assert_eq!(body.model.quantization.as_deref(), Some("Q4_K")); + // Qwen3.5-0.8B: a count, not a label — anything off by 10x is a wrong sum. + let n = body.model.parameter_count.expect("parameter_count is measured"); + assert!((500_000_000..1_500_000_000).contains(&n), "parameter_count {n}"); + assert_eq!(body.compute_class, "cpu"); + } + /// The startup line's thinking mode is read off what the template renders. #[test] fn the_thinking_mode_is_read_off_the_rendered_template() { diff --git a/crates/aprender-serve/src/api/effective_config.rs b/crates/aprender-serve/src/api/effective_config.rs index 81513d2a87..7498f1a15a 100644 --- a/crates/aprender-serve/src/api/effective_config.rs +++ b/crates/aprender-serve/src/api/effective_config.rs @@ -512,7 +512,8 @@ pub struct EffectiveConfigResponse { pub server: ServerReport, /// The dispatch path this process will take, from residency (PP-2). pub compute_class: &'static str, - /// `realizar`'s own `cfg!` feature list. + /// `realizar`'s own `cfg!` feature list, less any GPU backend the launching CLI + /// cannot dispatch to (#4254) — so it agrees with `apr serve run --list-devices`. pub build_features: Vec<&'static str>, /// The launching CLI's `cfg!` feature list, when it supplied one (§9 #8). pub build_features_cli: Option>, @@ -556,6 +557,30 @@ pub fn build_features() -> Vec<&'static str> { features } +/// #4254: `realizar` compiles wgpu by default (`gpu`), but `apr serve` routes to it only +/// when the CLI was built with its own `wgpu` feature. The v0.69.1 `-cpu` asset reported +/// `gpu` here while `--list-devices` said "NO accelerator compiled in" and +/// `--backend wgpu` refused. A backend the launcher cannot dispatch to is not reported. +/// With no CLI list (a library embedder), the library's own list stands. +#[must_use] +pub fn dispatchable_build_features( + library: Vec<&'static str>, + cli: Option<&[String]>, +) -> Vec<&'static str> { + let Some(cli) = cli else { + return library; + }; + let has = |f: &str| cli.iter().any(|c| c == f); + library + .into_iter() + .filter(|f| match *f { + "gpu" => has("wgpu"), + "cuda" => has("cuda"), + _ => true, + }) + .collect() +} + /// Backends resident in this `AppState`, in dispatch order. /// /// Derived from what is actually loaded. `has_cuda_model()` and friends are the @@ -564,11 +589,22 @@ pub fn build_features() -> Vec<&'static str> { #[must_use] pub fn backend_loaded(state: &AppState) -> Vec<&'static str> { let mut loaded: Vec<&'static str> = Vec::new(); + // #4254: the #3571 hybrid (Qwen3.5) session is resident on its own — neither the + // quantized model nor a CUDA executor slot is set — so it reported `[]`/`unknown` + // on both the CPU and the CUDA build while `/health` (which reads the session) + // said `cpu`/`gpu`. Its only GPU path is CUDA (#4271: no wgpu path). + let qwen35_on_gpu = state + .qwen35_session() + .map(|s| s.on_gpu.load(std::sync::atomic::Ordering::Relaxed)); + if qwen35_on_gpu == Some(true) { + loaded.push("cuda"); + } #[cfg(feature = "cuda")] { - if state.has_cuda_model() + if (state.has_cuda_model() || state.safetensors_cuda_model().is_some() - || state.apr_q4k_tx().is_some() + || state.apr_q4k_tx().is_some()) + && !loaded.contains(&"cuda") { loaded.push("cuda"); } @@ -577,7 +613,10 @@ pub fn backend_loaded(state: &AppState) -> Vec<&'static str> { if state.has_gpu_model() || state.has_cached_model() { loaded.push("wgpu"); } - if state.quantized_model().is_some() || state.apr_transformer().is_some() { + if state.quantized_model().is_some() + || state.apr_transformer().is_some() + || qwen35_on_gpu == Some(false) + { loaded.push("cpu"); } loaded @@ -671,7 +710,13 @@ pub fn effective_config(state: &AppState) -> EffectiveConfigResponse { server }, compute_class: compute_class_from_residency(state), - build_features: build_features(), + build_features: dispatchable_build_features( + build_features(), + effective + .offload + .as_ref() + .map(|o| o.build_features.as_slice()), + ), build_features_cli: effective.offload.as_ref().map(|o| o.build_features.clone()), backend_loaded: backend_loaded(state), model: ModelReport::from_state(state), @@ -804,6 +849,38 @@ impl Default for EffectiveConfigState { mod effective_config_tests { use super::*; + fn cli(features: &[&str]) -> Vec { + features.iter().map(|s| (*s).to_string()).collect() + } + + /// #4254: the v0.69.1 CPU asset advertised `build_features: [..., "gpu"]` + /// because aprender-serve's `gpu` feature is on by default, while the + /// binary had no wgpu dispatch. The CLI's own list decides what dispatches. + #[test] + fn dispatchable_build_features_drops_what_the_cli_cannot_dispatch_4254() { + let lib = vec!["cpu", "gpu", "cuda"]; + // The CPU asset: the CLI was built with neither `wgpu` nor `cuda`. + assert_eq!( + dispatchable_build_features(lib.clone(), Some(&cli(&["inference"]))), + vec!["cpu"] + ); + assert_eq!( + dispatchable_build_features(lib.clone(), Some(&cli(&["inference", "cuda"]))), + vec!["cpu", "cuda"] + ); + assert_eq!( + dispatchable_build_features(lib.clone(), Some(&cli(&["wgpu"]))), + vec!["cpu", "gpu"] + ); + // No CLI list (a library embedder): the library's list stands. + assert_eq!(dispatchable_build_features(lib.clone(), None), lib); + // The filter only removes, never invents a feature the library lacks. + assert_eq!( + dispatchable_build_features(vec!["cpu"], Some(&cli(&["cuda", "wgpu"]))), + vec!["cpu"] + ); + } + fn report(autofit: &[&str], explicit: &[&str]) -> OffloadReport { OffloadReport { gpu_layers_requested: "all".to_string(), diff --git a/crates/aprender-serve/src/api/tests/effective_config_route_pp2.rs b/crates/aprender-serve/src/api/tests/effective_config_route_pp2.rs index 7b8d99347f..505e60613d 100644 --- a/crates/aprender-serve/src/api/tests/effective_config_route_pp2.rs +++ b/crates/aprender-serve/src/api/tests/effective_config_route_pp2.rs @@ -504,3 +504,42 @@ async fn an_uninstrumented_scheduler_reports_null_not_zero() { assert!(body["scheduler"]["peak_in_flight"].is_null(), "{body}"); assert_eq!(body["scheduler"]["prefill_chunk_size"].as_u64(), Some(256)); } + +/// #4254: on the v0.69.1 CPU asset serving Qwen3.5-4B, `/health` said +/// `compute_mode: "cpu"` while `/v1/effective-config` said +/// `compute_class: "unknown"` and `backend_loaded: []` — the hybrid session +/// was the one resident backend `backend_loaded` never asked about. Both +/// routes now read the same session, so they must agree on the same state. +#[tokio::test(flavor = "multi_thread")] +async fn qwen35_cpu_session_reports_cpu_on_both_routes_4254() { + use crate::gguf::qwen35_session::Qwen35Session; + use crate::gguf::MappedGGUFModel; + use std::sync::Arc; + + const MODEL_PATH: &str = "/home/noah/models/Qwen3.5-0.8B-Q4_K_M.gguf"; + if !std::path::Path::new(MODEL_PATH).exists() { + eprintln!("SKIP: {MODEL_PATH} is absent"); + return; + } + let mapped = Arc::new(MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF")); + let vocab = mapped.model.vocabulary().expect("vocabulary"); + // `no_gpu = true`: the CPU path, which is what the v0.69.1 CPU asset ran. + let session = Qwen35Session::load(&mapped, true).expect("load the hybrid"); + let state = AppState::with_qwen35_session(session, mapped, vocab).expect("app state"); + + let (status, cfg) = get_json(state.clone(), "/v1/effective-config").await; + assert_eq!(status, StatusCode::OK); + let (status, health) = get_json(state, "/health").await; + assert_eq!(status, StatusCode::OK); + + assert_eq!( + cfg["backend_loaded"], + serde_json::json!(["cpu"]), + "the CPU hybrid session is a resident CPU backend: {cfg}" + ); + assert_eq!(cfg["compute_class"].as_str(), Some("cpu")); + assert_eq!( + cfg["compute_class"], health["compute_mode"], + "/v1/effective-config and /health disagree on one state" + ); +} From 2b0c860e6358562423a0ecc984f5c58f04417a60 Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Fri, 25 Sep 2026 05:19:42 +0200 Subject: [PATCH 17/68] test(#4274): a re-rendered chat turn resumes from turn 1's checkpoint; the device argmax drops a checkpoint it writes under Reconciles cf20f6f52 (#4274, explicit `keep` from the chat backend) onto the #4214 automatic checkpoint (7d1b3af65), which already covers #4274: it splits every prompt before its last <|im_start|>, so a re-rendered history resumes without the backend predicting a boundary. Keeping both would have left two checkpoint systems, so cf20f6f52's code is not taken; its real-template test is. - qwen35_chat_backend_tests: second_chat_turn_resumes_from_the_first_turns_checkpoint_4274 (0.8B, thinking on and off, official template): turn 2 does not extend turn 1, reuses > 0 tokens, and answers what a fresh full re-prefill answers. - session: advance_and_choose's forward_greedy branch (release-line #4263) now drops the checkpoint when it prefills from under it, like step() does. Falsifier a_device_argmax_from_below_the_checkpoint_drops_it; mutant (condition -> false) turns it RED. - runtime: truncate carries the same used_underscore_binding allow as its sibling (clippy -D warnings). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9eb442ebc29c7b0eb68e9917ae75321d2d678a2e) --- .../src/api/qwen35_chat_backend_tests.rs | 89 +++++++++++++++++++ .../src/gguf/cuda/forward_qwen35_cuda.rs | 8 +- crates/aprender-serve/src/gguf/cuda/mod.rs | 4 +- .../inference/forward/qwen35_session_tests.rs | 49 +++++++--- crates/aprender-serve/src/gguf/runtime.rs | 1 + crates/aprender-serve/src/session_tests.rs | 55 ++++++++++-- 6 files changed, 179 insertions(+), 27 deletions(-) diff --git a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs index c56a1b0274..167d57df3e 100644 --- a/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs +++ b/crates/aprender-serve/src/api/qwen35_chat_backend_tests.rs @@ -836,3 +836,92 @@ async fn a_streamed_completion_arrives_token_by_token_and_ends_with_usage() { let usage = usage.expect("the terminal chunk carries usage (#4272)"); assert_eq!(usage, plain["usage"], "the stream's usage is the body's"); } + +// #4274: a chat's second turn, re-rendered through the official template (which +// drops turn 1's think block), resumed from the checkpoint turn 1 left, and +// answered exactly what a full re-prefill answers. + +fn msg_4274(role: &str, content: &str) -> ChatMessage { + ChatMessage { + role: role.to_string(), + content: content.to_string(), + ..Default::default() + } +} + +/// The rendered and encoded prompt for `messages`, and the greedy turn. +fn turn_4274( + session: &mut Qwen35Session, + mapped: &MappedGGUFModel, + messages: &[ChatMessage], + thinking: Option, +) -> (Vec, crate::session::Turn) { + let m = &mapped.model; + let text = crate::api::realize_handlers::format_chat_messages_official_thinking( + Some(m), + messages, + Some("qwen35"), + thinking, + ) + .expect("render"); + let ids = m.encode(&text).expect("encode"); + let config = QuantizedGenerateConfig { + max_tokens: 24, + temperature: 0.0, + top_k: 1, + stop_tokens: m.eos_token_id().into_iter().collect(), + ..QuantizedGenerateConfig::default() + }; + let turn = session + .generate(&ids, &config, &mut |_| true) + .expect("generate"); + (ids, turn) +} + +#[test] +fn second_chat_turn_resumes_from_the_first_turns_checkpoint_4274() { + if !std::path::Path::new(MODEL_PATH).exists() { + eprintln!("SKIP: {MODEL_PATH} is absent"); + return; + } + let mapped = MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let m = &mapped.model; + for thinking in [Some(false), Some(true)] { + let mut session = Qwen35Session::load(&mapped, true).expect("load the hybrid"); + let first = [msg_4274("user", "Name three colors.")]; + let (ids1, turn1) = turn_4274(&mut session, &mapped, &first, thinking); + + let mut reply = turn1.tokens[ids1.len()..].to_vec(); + if reply.last().is_some_and(|t| m.eos_token_id() == Some(*t)) { + reply.pop(); + } + let reply = crate::api::realize_handlers::clean_chat_output(&m.decode(&reply)); + let second = [ + first[0].clone(), + msg_4274("assistant", &reply), + msg_4274("user", "Two more."), + ]; + let (ids2, turn2) = turn_4274(&mut session, &mapped, &second, thinking); + let shared = ids1.iter().zip(&ids2).take_while(|(a, b)| a == b).count(); + assert!( + shared < ids1.len(), + "thinking {thinking:?}: turn 2 must NOT extend turn 1 (else this is not #4274)" + ); + // The #4274 body: turn 2 re-prefilled from 0 (reused 0). + assert!( + turn2.reused > 0 && turn2.reused <= shared, + "thinking {thinking:?}: turn 2 must resume from turn 1's checkpoint \ + (reused {}, shared prefix {shared})", + turn2.reused + ); + + let mut fresh = Qwen35Session::load(&mapped, true).expect("load the hybrid"); + let (_, full) = turn_4274(&mut fresh, &mapped, &second, thinking); + assert_eq!(full.reused, 0); + assert_eq!( + m.decode(&turn2.tokens[ids2.len()..]), + m.decode(&full.tokens[ids2.len()..]), + "thinking {thinking:?}: the resumed turn must answer what a full re-prefill answers" + ); + } +} diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs index f709b3a3d3..917bffc402 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -840,11 +840,9 @@ impl<'a> Qwen35CudaModel<'a> { } *into = Some(Qwen35CudaCheckpoint { layers, kv_len: 0 }); } - let ckpt = into - .as_mut() - .ok_or_else(|| RealizarError::InvalidShape { - reason: "qwen35_cuda: the checkpoint was never allocated".to_string(), - })?; + let ckpt = into.as_mut().ok_or_else(|| RealizarError::InvalidShape { + reason: "qwen35_cuda: the checkpoint was never allocated".to_string(), + })?; for (i, conv, ssm) in &mut ckpt.layers { // SAFETY: source and destination are live device buffers of equal // length (checked by the copy) owned by `state` and `ckpt`, which diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index 9b4b4d40b8..add810fbf5 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -47,8 +47,8 @@ mod weights; pub use backend::CudaBackend; // PMAT-3477 (#3090): the Gated DeltaNet GPU model and its device state. pub use forward_qwen35_cuda::{ - PrefillAttention, Qwen35CudaCheckpoint, Qwen35CudaModel, Qwen35CudaState, PREFILL_MAX_CHUNK_ROWS, - UNIFIED_PREFILL_CHUNK_ROWS, + PrefillAttention, Qwen35CudaCheckpoint, Qwen35CudaModel, Qwen35CudaState, + PREFILL_MAX_CHUNK_ROWS, UNIFIED_PREFILL_CHUNK_ROWS, }; // #3714: the Qwen3-MoE GPU model, its device state, and the MoE shape it is built for. pub use forward_qwen3_moe_resident::{Qwen3MoeCudaModel, Qwen3MoeCudaState, Qwen3MoeShape}; diff --git a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs index cbe754a71d..71649be8b7 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/qwen35_session_tests.rs @@ -195,7 +195,10 @@ fn re_rendered_turn_two(mapped: &MappedGGUFModel, p1: &[u32]) -> Vec { let mut p2 = p1.to_vec(); p2.extend(encode( mapped, - &format!("The capital of Peru is Lima.<|im_end|>\n{}", user_turn("And of Chile?")), + &format!( + "The capital of Peru is Lima.<|im_end|>\n{}", + user_turn("And of Chile?") + ), )); p2 } @@ -207,9 +210,17 @@ fn cpu_an_identical_prompt_again_resumes_at_its_generation_header_and_matches_on let mut session = Qwen35Session::load(&mapped, true).expect("load"); let config = greedy(6); let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); - let t1 = session.generate(&p1, &config, &mut |_| true).expect("turn 1"); - let t2 = session.generate(&p1, &config, &mut |_| true).expect("turn 2"); - assert_eq!(t2.reused, header_at(&mapped, &p1), "resumed at the checkpoint"); + let t1 = session + .generate(&p1, &config, &mut |_| true) + .expect("turn 1"); + let t2 = session + .generate(&p1, &config, &mut |_| true) + .expect("turn 2"); + assert_eq!( + t2.reused, + header_at(&mapped, &p1), + "resumed at the checkpoint" + ); assert!(t2.reused > 0); assert_eq!(t2.tokens, t1.tokens); assert_eq!(t2.tokens, one_shot_cpu(&mapped, &p1, &config)); @@ -222,10 +233,18 @@ fn cpu_a_re_rendered_turn_two_resumes_at_turn_ones_header_and_matches_one_shot() let mut session = Qwen35Session::load(&mapped, true).expect("load"); let config = greedy(6); let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); - session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + session + .generate(&p1, &config, &mut |_| true) + .expect("turn 1"); let p2 = re_rendered_turn_two(&mapped, &p1); - let t2 = session.generate(&p2, &config, &mut |_| true).expect("turn 2"); - assert_eq!(t2.reused, header_at(&mapped, &p1), "resumed at turn 1's checkpoint"); + let t2 = session + .generate(&p2, &config, &mut |_| true) + .expect("turn 2"); + assert_eq!( + t2.reused, + header_at(&mapped, &p1), + "resumed at turn 1's checkpoint" + ); assert_eq!( t2.tokens, one_shot_cpu(&mapped, &p2, &config), @@ -443,8 +462,12 @@ mod gpu { }; let config = greedy(6); let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); - let t1 = session.generate(&p1, &config, &mut |_| true).expect("turn 1"); - let t2 = session.generate(&p1, &config, &mut |_| true).expect("turn 2"); + let t1 = session + .generate(&p1, &config, &mut |_| true) + .expect("turn 1"); + let t2 = session + .generate(&p1, &config, &mut |_| true) + .expect("turn 2"); assert!(t1.used_gpu && t2.used_gpu, "no fallback"); assert_eq!(t2.reused, header_at(&mapped, &p1)); assert!(t2.reused > 0); @@ -460,9 +483,13 @@ mod gpu { }; let config = greedy(6); let p1 = encode(&mapped, &user_turn("Name the capital of Peru.")); - session.generate(&p1, &config, &mut |_| true).expect("turn 1"); + session + .generate(&p1, &config, &mut |_| true) + .expect("turn 1"); let p2 = re_rendered_turn_two(&mapped, &p1); - let t2 = session.generate(&p2, &config, &mut |_| true).expect("turn 2"); + let t2 = session + .generate(&p2, &config, &mut |_| true) + .expect("turn 2"); assert!(t2.used_gpu, "no fallback"); assert_eq!(t2.reused, header_at(&mapped, &p1)); assert_eq!(t2.tokens, one_shot_gpu(&mapped, &p2, &config)); diff --git a/crates/aprender-serve/src/gguf/runtime.rs b/crates/aprender-serve/src/gguf/runtime.rs index 6d70e09ae2..c6d29543c9 100644 --- a/crates/aprender-serve/src/gguf/runtime.rs +++ b/crates/aprender-serve/src/gguf/runtime.rs @@ -251,6 +251,7 @@ impl OwnedQuantizedKVCache { /// Keep only the first `new_len` positions, by the row width the cache was /// built with (#4214: a session resuming from a checkpoint). + #[allow(clippy::used_underscore_binding)] // `_hidden_dim` holds the kv_dim pub fn truncate(&mut self, new_len: usize) { self.rollback_to(new_len, self._hidden_dim); } diff --git a/crates/aprender-serve/src/session_tests.rs b/crates/aprender-serve/src/session_tests.rs index fe51d82501..f3fc7bd474 100644 --- a/crates/aprender-serve/src/session_tests.rs +++ b/crates/aprender-serve/src/session_tests.rs @@ -245,8 +245,18 @@ impl ArchForward for DeviceGreedy { fn forward(&mut self, tokens: &[u32], start: usize) -> Result> { self.inner.forward(tokens, start) } + fn checkpoint_at(&self, prompt: &[u32]) -> Option { + self.inner.checkpoint_at(prompt) + } + fn save_checkpoint(&mut self) -> Result<()> { + self.inner.save_checkpoint() + } + fn restore_checkpoint(&mut self) -> Result { + self.inner.restore_checkpoint() + } fn forward_greedy(&mut self, tokens: &[u32], start: usize) -> Result> { self.inner.calls.push((tokens.len(), start)); + self.inner.held = tokens.len(); self.greedy_calls += 1; Ok(Some(self.greedy_answer)) } @@ -292,16 +302,15 @@ const MARK: u32 = 7; fn an_identical_prompt_again_restores_the_checkpoint_before_its_last_marker() { let mut s = Session::new(Scripted::checkpointing(3, 100, MARK)); let prompt = [7801, 7802, MARK, 7803]; - let t1 = s - .generate(&prompt, &greedy(1), &mut |_| true) - .expect("t1"); + let t1 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); assert_eq!(t1.reused, 0); // Prefilled in two spans: up to the checkpoint, then the rest. assert_eq!(s.engine().calls, vec![(2, 0), (4, 2)]); - let t2 = s - .generate(&prompt, &greedy(1), &mut |_| true) - .expect("t2"); - assert_eq!(t2.reused, 2, "the repeat prefilled only from the checkpoint"); + let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); + assert_eq!( + t2.reused, 2, + "the repeat prefilled only from the checkpoint" + ); assert_eq!(t2.tokens, t1.tokens); assert_eq!(s.engine().calls.last(), Some(&(4, 2))); assert_eq!(s.engine().restores, 1); @@ -361,7 +370,11 @@ fn a_checkpoint_the_forward_lost_is_not_reused() { s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); assert_eq!(t2.reused, 0); - assert_eq!(s.engine().calls.last(), Some(&(4, 2)), "re-checkpointed from 0"); + assert_eq!( + s.engine().calls.last(), + Some(&(4, 2)), + "re-checkpointed from 0" + ); assert_eq!(s.engine().calls[s.engine().calls.len() - 2], (2, 0)); } @@ -372,6 +385,30 @@ fn scoring_rewrites_the_state_so_the_checkpoint_is_dropped() { s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); s.score(&[7859, 7858], &mut |_, _| true).expect("score"); let t2 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t2"); - assert_eq!(t2.reused, 0, "the positions under the checkpoint were rewritten"); + assert_eq!( + t2.reused, 0, + "the positions under the checkpoint were rewritten" + ); assert_eq!(s.engine().restores, 0); } + +#[test] +fn a_device_argmax_from_below_the_checkpoint_drops_it() { + // The device-argmax path rewrites positions too: a prompt it prefills from + // under the checkpoint leaves a copy whose KV rows no longer match. + let mut s = Session::new(DeviceGreedy { + inner: Scripted::checkpointing(3, 100, MARK), + greedy_answer: 5, + greedy_calls: 0, + }); + let prompt = [7861, 7862, MARK, 7863]; + s.generate(&prompt, &greedy(1), &mut |_| true).expect("t1"); + s.generate(&[7869], &greedy(1), &mut |_| true).expect("t2"); + assert!(s.engine().greedy_calls > 0, "t2 took the device argmax"); + let t3 = s.generate(&prompt, &greedy(1), &mut |_| true).expect("t3"); + assert_eq!( + t3.reused, 0, + "position 0 was rewritten under the checkpoint" + ); + assert_eq!(s.engine().inner.restores, 0); +} From c1a89cb706f29561affd1e2b3460ec03bca8b4fe Mon Sep 17 00:00:00 2001 From: Noah Gift Date: Thu, 24 Sep 2026 00:04:51 +0200 Subject: [PATCH 18/68] fix(dogfood): the declared ladder gate reads a release's RECORDED scope, and says SCOPED (#4086) At 0.69.1 the release gate judged the operator's recorded emergency scope (`--scope crux-smoke`, satisfied on both hosts) while the dogfood ran the same check_model_ladder.sh with no arguments -- the full ladder -- and went RED: two gates, one release, opposite verdicts, shipped as known-red #4086. check_model_ladder.sh, called with no --scope, now applies the scope RECORDED for the release being judged (crux_smoke_scope.recorded_scope over ladder.emergency_scopes). It is never inferred: no entry for this version = the full ladder; two entries or a nameless one = RED, judging neither. It is SAID: a `SCOPED: -- ...` line precedes the scope's own verdict, and `--scope none` forces the full ladder on a scoped release (the nightly). MODEL_LADDER_CRUX_DIR is the receipts seam for a caller that cannot pass --crux. dogfood.sh's classify_declared carries a gate's `SCOPED:` line into its row note, green or red (the scope NAME only: mark() keeps 200 chars and a red row's reason must survive), so a scoped pass never reads as a bare `exit=0`. Must-RED: with NO recorded scope, CRUX smoke receipts that would satisfy a scope do not turn the gate green -- the full ladder is judged and is RED (e2e row). Tables: select (6 rows, 4 module mutants), e2e dispatch (7 rows, 4 script mutants), dogfood note (4 rows, 3 mutants); all mutants killed. Real data: the no-argument call at cut d8a6df53a on the published 0.69.1 receipts is SCOPED and green; at this branch's HEAD it is RED (receipts are not from this binary). Contract: FALSIFY-MCL-017. Co-Authored-By: Claude Opus 5.5 (1M context) (cherry picked from commit 51222808aac6ac2c3e4754368a2d367b42e96249) --- contracts/model-capability-ladder-v1.yaml | 5 + scripts/check_dogfood_no_defer.sh | 21 +++- scripts/check_model_ladder.sh | 129 ++++++++++++++++++++++ scripts/dogfood.sh | 14 ++- scripts/lib/crux_smoke_scope.py | 20 ++++ 5 files changed, 183 insertions(+), 6 deletions(-) diff --git a/contracts/model-capability-ladder-v1.yaml b/contracts/model-capability-ladder-v1.yaml index f3c13618ce..db4d6e0e4e 100644 --- a/contracts/model-capability-ladder-v1.yaml +++ b/contracts/model-capability-ladder-v1.yaml @@ -458,6 +458,11 @@ falsification_tests: prediction: "check_model_ladder.sh --self-test: green-cells passes (20 owed on lambda, 12 pass, 8 honest refusals); a missing cell, a duplicate row, a fallback, a thinking block that never closed, a prompt under its rung, an empty answer, a co-tenant refusal, an OOM instead of a refusal, a rung that passes nowhere, thinking_modes that disagree with the template, a missing context_length (#3733), no cells, a consumer without a number, a missing context-rungs file, a dropped cells block or verb, a rung id or consumer dropped from context-rungs.json vs origin/main, an arch with no held representative, and a pass where the declared arithmetic says the cell cannot fit (the arithmetic decides what is owed, so it must be right) are each RED naming it; 11 module mutants are each killed by their case" test: "bash scripts/check_model_ladder.sh --self-test" if_fails: "a release dogfoods one verb at one short prompt with thinking off and calls the model family working" + - id: FALSIFY-MCL-017 + rule: "A recorded emergency scope applies to the gate called with no --scope, and is SAID; none recorded is the full ladder (#4086)" + prediction: "check_model_ladder.sh --self-test: the select table (no scope / another release / a prefix of the release -> the full ladder; one scope -> it applies; two scopes or a nameless one -> RED) lands on crux_smoke_scope.recorded_scope, and 4 module mutants are each killed; end to end, NO recorded scope judges the full ladder beside CRUX smoke receipts that would satisfy a scope (RED, no SCOPED: line), a recorded scope prints `SCOPED: ` and returns the SCOPE's verdict (green and red), --scope none and a scope for another release judge the full ladder, two scopes are RED judging neither, and the receipts are found through MODEL_LADDER_CRUX_DIR; 4 dispatch mutants are each killed. check_dogfood_no_defer.sh: a declared gate's `SCOPED:` line is carried into its dogfood row, green or red, and a row with none stays plain (3 mutants)" + test: "bash scripts/check_model_ladder.sh --self-test && bash scripts/check_dogfood_no_defer.sh --self-test" + if_fails: "the dogfood's declared ladder gate judges the full ladder while the release gate judges the recorded scope -- two gates, one release, opposite verdicts (0.69.1, #4086) -- or a scoped pass reads as `exit=0`, indistinguishable from every rung green" qa_gate: id: F-MCL-001 diff --git a/scripts/check_dogfood_no_defer.sh b/scripts/check_dogfood_no_defer.sh index 405bf6f6a3..4a54985237 100755 --- a/scripts/check_dogfood_no_defer.sh +++ b/scripts/check_dogfood_no_defer.sh @@ -53,16 +53,24 @@ lift() { # lift