diff --git a/crates/aprender-serve/src/cuda/executor/core.rs b/crates/aprender-serve/src/cuda/executor/core.rs index 58aff3d9bd..a31ecc3648 100644 --- a/crates/aprender-serve/src/cuda/executor/core.rs +++ b/crates/aprender-serve/src/cuda/executor/core.rs @@ -186,6 +186,7 @@ impl CudaExecutor { num_sms: context.multiprocessor_count().unwrap_or(8) as u32, // PMAT-027: Q8 activation cache starts invalid q8_activation_valid: false, + q8_activation_src: (0, 0), fp8_activation_cache_key: None, fp8_weight_scales: HashMap::new(), fp8_act_scale_buf: None, diff --git a/crates/aprender-serve/src/cuda/executor/executor_api.rs b/crates/aprender-serve/src/cuda/executor/executor_api.rs index e835c5a4d9..6147516ced 100644 --- a/crates/aprender-serve/src/cuda/executor/executor_api.rs +++ b/crates/aprender-serve/src/cuda/executor/executor_api.rs @@ -493,6 +493,27 @@ impl CudaExecutor { self.stream.synchronize() } + /// Overwrite a resident device buffer from the host, queued on the + /// execution stream: no allocation and no host wait. The copy lands after + /// every kernel already queued and before every kernel queued later, so a + /// buffer reused token after token is never overwritten while a previous + /// token's kernels still read it (#4316). + /// + /// # Safety + /// `src` must stay valid until the execution stream is synchronized. + /// + /// # Errors + /// `src` and `dst` differ in length, or the transfer fails. + pub unsafe fn upload_into_async( + &self, + dst: &mut GpuBuffer, + src: &[f32], + ) -> Result<(), GpuError> { + // SAFETY: the caller guarantees `src` outlives the next sync of + // `self.stream`, which is the stream the copy is queued on. + unsafe { dst.copy_from_host_async(src, &self.stream) } + } + /// Get memory pool statistics (IMP-900d) #[must_use] pub fn pool_stats(&self) -> PoolStats { diff --git a/crates/aprender-serve/src/cuda/executor/gdn_ops.rs b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs index 5446ac9d6b..7e508a7dce 100644 --- a/crates/aprender-serve/src/cuda/executor/gdn_ops.rs +++ b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs @@ -50,6 +50,9 @@ impl CudaExecutor { ) -> Result<(), GpuError> { for (i, &p) in ptrs.iter().enumerate() { validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + // #4258: any pointer argument may be an output (several GDN kernels + // write in place), so any of them can stale the Q8 activation. + self.q8_activation_written(p); } let mut args: Vec = ptrs.to_vec(); let mut raw: Vec<*mut std::ffi::c_void> = args @@ -103,6 +106,9 @@ impl CudaExecutor { ) -> Result<(), GpuError> { for (i, &p) in ptrs.iter().enumerate() { validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + // #4258: any pointer argument may be an output (several GDN kernels + // write in place), so any of them can stale the Q8 activation. + self.q8_activation_written(p); } let mut args: Vec = ptrs.to_vec(); args.extend_from_slice(scalars); diff --git a/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs b/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs index 0164695b2e..e3898813d9 100644 --- a/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs +++ b/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs @@ -150,6 +150,7 @@ impl CudaExecutor { hidden_size: u32, epsilon: f32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 // GH-559 DIAGNOSTIC: CPU RMSNorm bypass for Blackwell static CPU_RMSNORM: std::sync::OnceLock = std::sync::OnceLock::new(); let use_cpu_rmsnorm = *CPU_RMSNORM.get_or_init(|| { @@ -304,6 +305,7 @@ impl CudaExecutor { num_heads: u32, epsilon: f32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::PerHeadRmsNorm { head_dim, num_heads, diff --git a/crates/aprender-serve/src/cuda/executor/mod.rs b/crates/aprender-serve/src/cuda/executor/mod.rs index ba746ce993..dd7ec79f67 100644 --- a/crates/aprender-serve/src/cuda/executor/mod.rs +++ b/crates/aprender-serve/src/cuda/executor/mod.rs @@ -299,6 +299,8 @@ mod poison_trace_test; // FALSIFY-QDOT-008 (#3111): the Q5_K GEMV against gguf-py's values of a llama.cpp block #[cfg(test)] mod tests_q5k_ggml; +#[cfg(test)] +mod tests_q8_activation_staleness; // COV-003 through COV-006 (layer preload, kv_cache, attention, quantized) #[cfg(test)] @@ -635,6 +637,11 @@ pub struct CudaExecutor { // Set to true after q8_quantize_into; callers invalidate (set false) // when the input buffer content changes (e.g. after RMSNorm write). q8_activation_valid: bool, + // #4258: which activation the Q8 cache holds — (source buffer ptr, element + // count). `ensure_q8_activation` re-quantizes when a GEMV's input is not this + // buffer, and every in-place writer clears `q8_activation_valid` when it writes + // this buffer, so a DP4A GEMV never reads another activation's Q8_1 bytes. + q8_activation_src: (u64, u32), // PMAT-084: FP8 activation cache — skip redundant absmax+convert when // multiple FP8 GEMMs share the same input (QKV phase, FFN gate+up). // Saves 84 kernel pairs per prefill (3 per layer × 28 layers). diff --git a/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs b/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs index c481080c9c..82b6084c8f 100644 --- a/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs +++ b/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs @@ -126,10 +126,7 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch DP4A GEMV kernel let num_warps = self.gpu_profile.mwv_warps; @@ -231,10 +228,7 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch half-warp DP4A GEMV kernel let num_warps = self.gpu_profile.mwv_warps; @@ -475,10 +469,7 @@ impl CudaExecutor { // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::FusedGateUpSwigluHwDp4aQ4KGemv { k, n }; @@ -617,10 +608,7 @@ impl CudaExecutor { // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::FusedKVHwDp4aQ4KGemv { k, n }; @@ -726,11 +714,7 @@ impl CudaExecutor { // Skip Q8 quantize when the same input buffer was already quantized // (e.g., K/V projections share input with Q, up shares with gate). // Saves 3 Q8 launches per layer × 28 layers = 84 launches per step. - if !self.q8_activation_valid { - let total_elements = m * k; - self.q8_quantize_into(input, &q8_buf, total_elements)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, m * k)?; // Step 2: Launch batched HW DP4A kernel let num_warps = self.gpu_profile.mwv_warps; diff --git a/crates/aprender-serve/src/cuda/executor/residual.rs b/crates/aprender-serve/src/cuda/executor/residual.rs index 31ded422d6..59ea26648f 100644 --- a/crates/aprender-serve/src/cuda/executor/residual.rs +++ b/crates/aprender-serve/src/cuda/executor/residual.rs @@ -309,6 +309,7 @@ impl CudaExecutor { output: &GpuBuffer, n: u32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::ResidualAdd { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); // GH-129: PTX is n-independent (n is a runtime param), so use constant cache key. diff --git a/crates/aprender-serve/src/cuda/executor/silu.rs b/crates/aprender-serve/src/cuda/executor/silu.rs index db1bce1953..1844d87194 100644 --- a/crates/aprender-serve/src/cuda/executor/silu.rs +++ b/crates/aprender-serve/src/cuda/executor/silu.rs @@ -235,6 +235,7 @@ impl CudaExecutor { output: &GpuBuffer, n: u32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::FusedSwiglu { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); let cache_key = format!("fused_swiglu_{}", n); diff --git a/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs b/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs new file mode 100644 index 0000000000..b6bed92563 --- /dev/null +++ b/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs @@ -0,0 +1,188 @@ +//! #4258: the DP4A GEMVs cache their Q8_1 activation behind `q8_activation_valid`. Before +//! #4258 nothing on the Qwen3.5 path cleared that flag, so every DP4A GEMV after the first +//! reused the FIRST quantized activation — across projections, layers and tokens. +//! +//! Each test runs a DP4A Q4_K GEMV whose input is NOT the activation last quantized, and +//! judges it against the float (Mwv) GEMV of the same input. A stale activation reproduces +//! the previous GEMV's output exactly; a fresh one lands within Q8_1 rounding of the float. + +use super::CudaExecutor; +use crate::cuda::gpu_profile::Q4kVariant; +use serial_test::serial; +use trueno_gpu::driver::GpuBuffer; + +const N: u32 = 32; +const K: u32 = 256; +/// Q8_1 activation rounding on these inputs is well under 1% relative L2. +const FRESH_TOL: f32 = 0.05; +/// The two inputs are built so their float outputs sit far apart; below this the test +/// could not tell a stale activation from a fresh one. +const DISTINCT_FLOOR: f32 = 0.2; + +/// `N` rows of one Q4_K super-block each: d = 1.0, dmin = 0, every 6-bit scale 1, +/// pseudo-random nibbles. +fn q4k_weights() -> Vec { + let mut seed = 0x2545_f491_u32; + let mut out = Vec::with_capacity(N as usize * 144); + for _ in 0..N { + out.extend_from_slice(&[0x00, 0x3C, 0x00, 0x00]); + out.extend_from_slice(&[0x01; 12]); + for _ in 0..128 { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + out.push((seed >> 16) as u8); + } + } + out +} + +fn input_a() -> Vec { + (0..K).map(|i| (i as f32 * 0.37).sin()).collect() +} + +fn input_b() -> Vec { + (0..K) + .map(|i| (i as f32 * 0.11).cos() * 2.0 - 0.5) + .collect() +} + +fn rel_l2(got: &[f32], want: &[f32]) -> f32 { + let num: f32 = got.iter().zip(want).map(|(g, w)| (g - w).powi(2)).sum(); + let den: f32 = want.iter().map(|w| w * w).sum(); + (num / den).sqrt() +} + +struct Rig { + ex: CudaExecutor, + w: u64, + out: GpuBuffer, +} + +impl Rig { + fn new(mut ex: CudaExecutor) -> Self { + ex.init_workspace(K as usize, K as usize) + .expect("init_workspace"); + ex.load_quantized_weights("q8_staleness_4258", &q4k_weights()) + .expect("load Q4_K"); + let w = ex + .get_quantized_weight_ptr("q8_staleness_4258") + .expect("weight ptr"); + let out = GpuBuffer::new(ex.context(), N as usize).expect("out"); + Self { ex, w, out } + } + + fn buf(&self, host: &[f32]) -> GpuBuffer { + GpuBuffer::from_host(self.ex.context(), host).expect("buffer") + } + + fn gemv(&mut self, variant: Q4kVariant, input: &GpuBuffer) -> Vec { + self.ex.gpu_profile.q4k = variant; + self.ex + .q4k_gemv_into(self.w, input, &self.out, N, K) + .expect("gemv"); + self.ex.synchronize().expect("sync"); + let mut host = vec![0.0f32; N as usize]; + self.out.copy_to_host(&mut host).expect("copy out"); + host + } + + /// The float GEMV of whatever `input` holds right now. + fn float_ref(&mut self, input: &GpuBuffer) -> Vec { + self.ex.synchronize().expect("sync"); + let mut host = vec![0.0f32; K as usize]; + input.copy_to_host(&mut host).expect("copy in"); + let fresh = self.buf(&host); + self.gemv(Q4kVariant::Mwv, &fresh) + } +} + +const DP4A: [Q4kVariant; 2] = [Q4kVariant::HwDp4a, Q4kVariant::MwvDp4a]; + +/// Two DP4A GEMVs on two DIFFERENT buffers: the second must quantize its own input. +#[test] +#[serial] +fn a_dp4a_gemv_on_a_different_buffer_quantizes_that_buffer() { + let ex = crate::cuda_executor_or_skip!(0); + let mut rig = Rig::new(ex); + let (a, b) = (rig.buf(&input_a()), rig.buf(&input_b())); + let (ref_a, ref_b) = (rig.float_ref(&a), rig.float_ref(&b)); + assert!(rel_l2(&ref_a, &ref_b) > DISTINCT_FLOOR, "inputs too alike"); + + for variant in DP4A { + let got_b = rig.gemv(variant, &b); + let got_a = rig.gemv(variant, &a); + let (err_b, err_a) = (rel_l2(&got_b, &ref_b), rel_l2(&got_a, &ref_a)); + eprintln!("[4258] {variant:?}: first input {err_b:.3e}, second input {err_a:.3e}"); + assert!( + err_b < FRESH_TOL, + "{variant:?}: the first DP4A GEMV is off ({err_b})" + ); + assert!( + err_a < FRESH_TOL, + "{variant:?}: a DP4A GEMV on a second buffer is {err_a} from the float GEMV of \ + that buffer — it reused the first buffer's Q8_1 activation (#4258)" + ); + } +} + +/// One buffer, rewritten in place by each Qwen3.5 decode primitive that feeds a GEMV: the +/// DP4A GEMV after the write must quantize the NEW contents. +#[test] +#[serial] +fn a_dp4a_gemv_after_a_qwen35_writer_rewrites_its_input_requantizes() { + let ex = crate::cuda_executor_or_skip!(0); + let mut rig = Rig::new(ex); + let (a_host, b_host) = (input_a(), input_b()); + let (a, b) = (rig.buf(&a_host), rig.buf(&b_host)); + let ones = rig.buf(&vec![1.0f32; K as usize]); + + type Writer = + fn(&mut CudaExecutor, &GpuBuffer, &GpuBuffer, &GpuBuffer, &GpuBuffer); + let writers: [(&str, Writer); 5] = [ + ("rmsnorm_into", |ex, _a, b, one, x| { + ex.rmsnorm_into(b, one, x, K, 1e-6).expect("rmsnorm_into"); + }), + ("per_head_rmsnorm_into", |ex, _a, b, one, x| { + // `one` is K ones; per-head gamma reads only its first 64. + ex.per_head_rmsnorm_into(b, one, x, 64, K / 64, 1e-6) + .expect("per_head_rmsnorm_into"); + }), + ("residual_add_into", |ex, a, b, _one, x| { + ex.residual_add_into(a, b, x, K).expect("residual_add_into"); + }), + ("fused_swiglu_into", |ex, a, b, _one, x| { + ex.fused_swiglu_into(b, a, x, K).expect("fused_swiglu_into"); + }), + ("gdn_sigmoid_gate_into", |ex, _a, b, _one, x| { + ex.gdn_sigmoid_gate_into(x, b, K) + .expect("gdn_sigmoid_gate_into"); + }), + ]; + + // Every `x` stays alive for the whole test: a freed buffer's address is handed + // straight back to the next allocation, and a host upload into it is not a write + // the executor sees (see `ensure_q8_activation`). + let xs: Vec> = (0..DP4A.len() * writers.len()) + .map(|_| rig.buf(&a_host)) + .collect(); + let mut xs = xs.iter(); + for variant in DP4A { + for (name, write) in writers { + let x = xs.next().expect("one x per case"); + let before = rig.gemv(variant, x); + write(&mut rig.ex, &a, &b, &ones, x); + let want = rig.float_ref(x); + assert!( + rel_l2(&before, &want) > DISTINCT_FLOOR, + "{name}: write changed too little" + ); + let got = rig.gemv(variant, x); + let err = rel_l2(&got, &want); + eprintln!("[4258] {variant:?} after {name}: {err:.3e}"); + assert!( + err < FRESH_TOL, + "{variant:?}: the DP4A GEMV after `{name}` rewrote its input is {err} from the \ + float GEMV of the new contents — it reused the stale Q8_1 activation (#4258)" + ); + } + } +} diff --git a/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs b/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs index 5f0f362390..ab70dbbb98 100644 --- a/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs +++ b/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs @@ -333,6 +333,38 @@ impl CudaExecutor { Ok(buf_output) } + /// #4258: quantize the first `n` elements of `input` into `q8_buf` unless the + /// Q8 activation cache already holds exactly that. The cache is keyed on the + /// source buffer, so a GEMV over a different buffer quantizes its own input + /// instead of reusing the last one (PMAT-027 keyed only on a bool). + /// + /// The executor only sees writes made by its own kernels, which invalidate + /// through [`Self::q8_activation_written`]. A host upload into a buffer, or a + /// freed buffer whose address is reallocated, is invisible to it: the caller + /// must clear `q8_activation_valid` before a DP4A GEMV reads such a buffer. + pub(crate) fn ensure_q8_activation( + &mut self, + input: &GpuBuffer, + q8_buf: &GpuBuffer, + n: u32, + ) -> Result<(), GpuError> { + let src = (input.as_ptr(), n); + if !self.q8_activation_valid || self.q8_activation_src != src { + self.q8_quantize_into(input, q8_buf, n)?; + self.q8_activation_valid = true; + self.q8_activation_src = src; + } + Ok(()) + } + + /// #4258: a kernel is about to write device buffer `dst`; if that buffer is + /// the one the Q8 activation cache was quantized from, the cache is stale. + pub(crate) fn q8_activation_written(&mut self, dst: u64) { + if dst == self.q8_activation_src.0 { + self.q8_activation_valid = false; + } + } + /// PAR-PERF-DP4A: Q8 quantize into PRE-ALLOCATED buffer (zero allocation) /// /// Five-Whys root cause (2026-02-09): diff --git a/crates/aprender-serve/src/cuda/executor/weight.rs b/crates/aprender-serve/src/cuda/executor/weight.rs index c84c9b5cd4..c82826c562 100644 --- a/crates/aprender-serve/src/cuda/executor/weight.rs +++ b/crates/aprender-serve/src/cuda/executor/weight.rs @@ -271,10 +271,7 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch DP4A Q6K GEMV kernel let num_warps = self.gpu_profile.mwv_warps; @@ -365,10 +362,7 @@ impl CudaExecutor { // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::HwDp4aQ6KGemv { k, n, num_warps }; diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs index 8ff1f85732..d53728dfa4 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -257,6 +257,11 @@ pub struct Qwen35CudaModel<'a> { out_normed: GpuBuffer, /// `[vocab_size]`, the logits — the ONE buffer a token's forward downloads. logits_buf: GpuBuffer, + /// `[hidden_dim]`, the residual stream: each token's embedding row is + /// uploaded into it in stream order and every layer updates it in place. + /// Allocated once, so its device pointer is the same on every token (#4316); + /// `None` only while `forward_single` holds it. + residual: Option>, dims: Qwen35CudaDims, /// Positions the device KV caches hold. max_seq_len: usize, @@ -519,21 +524,16 @@ impl<'a> Qwen35CudaModel<'a> { // PRODUCTION DEFAULT, not a test affordance (PMAT-3477 / #3090): this // architecture runs the FLOAT Q4_K/Q6_K GEMV kernels, never the DP4A - // ones `GpuProfile::detect` picks for a dense decode. The DP4A kernels - // quantize the ACTIVATION to int8, and Qwen3.5 feeds its projections - // straight into a recurrence, which compounds that error instead of - // absorbing it. Measured on the real 0.8B file: with the float variants - // pinned, a whole DeltaNet layer's output is 0.000 relative from a - // second float run and inside the layer budget against the CPU; with - // `HwDp4a` the DeltaNet-only path lands **1.656 relative** away, and the - // end-to-end argmax is garbage — a wrong token at position 0, not a - // rounding difference. The falsifier lives in the tests file - // (`qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence`). + // ones `GpuProfile::detect` picks for a dense decode. // - // Recovering the DP4A throughput for this architecture (a higher- - // precision activation quantization, or DP4A only on the layers that do - // not feed the recurrence) is the DP4A-through-recurrence ticket, - // 0.69.0. Until it lands, correctness is not optional here. + // The pin's original reason, "DP4A is catastrophic through the + // recurrence" (1.656 relative, a wrong argmax at position 0), was + // #4258: no qwen35 writer kernel cleared `q8_activation_valid`, so + // every DP4A GEMV reused the first Q8_1 activation. With that fixed, + // DP4A holds the parity contract on 0.8B + // (`qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence`). The pin + // stays until DP4A is re-measured on 2B/4B (#4030). Lifting it is a + // perf decision to make on that evidence, not a side effect of this fix. Self::pin_float_gemv(&mut executor.gpu_profile); let mut layers = Vec::with_capacity(model.layers.len()); @@ -589,6 +589,7 @@ impl<'a> Qwen35CudaModel<'a> { let attn_scratch = Self::build_attn_scratch(&executor, dims)?; let out_normed = Self::zeros(&executor, dims.hidden_dim as usize)?; let logits_buf = Self::zeros(&executor, dims.vocab_size as usize)?; + let residual = Some(Self::zeros(&executor, dims.hidden_dim as usize)?); let state = Self::build_state(&executor, &layers, dims, max_seq_len)?; Ok(Self { model, @@ -601,6 +602,7 @@ impl<'a> Qwen35CudaModel<'a> { lm_head, out_normed, logits_buf, + residual, dims, max_seq_len, }) @@ -1408,15 +1410,50 @@ impl<'a> Qwen35CudaModel<'a> { }); } - let dev = GpuBuffer::from_host( - self.executor.context(), + // The resident residual leaves `self` for the length of the token, so + // the `&mut self` layer calls can borrow it, and goes back on every + // path — a refused token must not cost the next one its buffer. It is + // only missing if an earlier token panicked mid-forward; re-allocate + // then rather than fail every later token. + let mut dev = match self.residual.take() { + Some(buf) => buf, + None => Self::zeros(&self.executor, hidden_dim)?, + }; + let out = self.forward_resident( &embedding[start..start + hidden_dim], - ) - .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + &mut dev, + state, + position, + ); + self.residual = Some(dev); + out + } + + /// The resident residual's device pointer — `None` while a token holds it. + #[cfg(test)] + pub(crate) fn residual_ptr(&self) -> Option { + self.residual.as_ref().map(|b| b.as_ptr() as u64) + } + + /// `forward_single` past its checks, on the resident residual `dev`. + fn forward_resident( + &mut self, + row: &[f32], + dev: &mut GpuBuffer, + state: &mut Qwen35CudaState, + position: usize, + ) -> Result> { + // SAFETY: `row` borrows the embedding table behind `self.model` + // (`&'a`), which outlives `self` and therefore the executor's stream: + // whenever the queued copy runs, and whether or not this token reaches + // the `sync_stream` below, its source memory is still alive. + unsafe { self.executor.upload_into_async(dev, row) } + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + let dev = &*dev; for il in 0..self.layers.len() { match self.layers[il] { - CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, &dev)?, - CudaLayer::Attention(_) => self.attention_layer(state, il, &dev, position)?, + CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, dev)?, + CudaLayer::Attention(_) => self.attention_layer(state, il, dev, position)?, } } // The tail stays on the device (#3090 review). `hidden_to_logits` would @@ -1429,7 +1466,7 @@ impl<'a> Qwen35CudaModel<'a> { let d = self.dims; self.executor .rmsnorm_into( - &dev, + dev, &self.output_norm, &self.out_normed, d.hidden_dim, diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs index 2d1ea33116..55e1a6e3c3 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs @@ -1093,6 +1093,52 @@ fn qwen35_cuda_state_is_sized_from_the_config() { ); } +/// The residual stream is ONE device buffer for the model's life (#4316): the +/// same device pointer before the first token, after every token and after a +/// refused one. A per-token `GpuBuffer::from_host` — what `forward_single` did +/// before — hands out a new pointer, which no captured CUDA graph can replay +/// (#4215). That the upload into it really happens every token is +/// [`qwen35_cuda_forward_single_matches_cpu_logits_end_to_end`]'s job: a stale +/// residual is the previous token's hidden state, not this token's embedding. +#[test] +#[serial_test::serial] +fn qwen35_cuda_the_residual_is_one_buffer_across_tokens() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let mut state = gpu.new_state().expect("device state"); + let at_build = gpu + .residual_ptr() + .expect("the residual is allocated at construction"); + + for (pos, &token) in LONG_PROMPT.iter().enumerate() { + gpu.forward_single(token, &mut state, pos) + .expect("gpu forward"); + assert_eq!( + gpu.residual_ptr(), + Some(at_build), + "pos {pos}: the residual moved — a token allocated its own" + ); + } + + let vocab = u32::try_from(qwen.base.token_embedding().len() / qwen.base.config.hidden_dim) + .expect("vocab fits u32"); + let pos = LONG_PROMPT.len(); + assert!(gpu.forward_single(vocab, &mut state, pos).is_err()); + assert_eq!( + gpu.residual_ptr(), + Some(at_build), + "a refused token must hand the residual back" + ); + gpu.forward_single(LONG_PROMPT[0], &mut state, pos) + .expect("the token after a refusal runs on the same buffer"); + assert_eq!(gpu.residual_ptr(), Some(at_build)); +} + /// `forward_hidden_deltanet_only` refuses a hidden state of the wrong width /// instead of reading past the buffer. #[test] @@ -1230,21 +1276,22 @@ fn qwen35_cuda_a_fresh_model_pins_the_float_gemv_variants() { ); } -/// WHY THE PIN EXISTS (PMAT-3477 / #3090): with the DP4A GEMV kernels armed, -/// the same forward that matches the CPU token for token produces a DIFFERENT -/// token, or a direction nowhere near the CPU's. +/// DP4A through the recurrence holds the parity contract (#4258). /// -/// This is a falsifier, not a bug reproduction: it asserts the failure, so if a -/// future DP4A activation-quantization change makes the path correct, this test -/// goes RED and the pin (and the DP4A-through-recurrence ticket, 0.69.0) can be -/// reconsidered on evidence. +/// This was the falsifier `qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence` +/// (PMAT-3477 / #3090), which asserted the opposite: 1.656 relative away and a wrong +/// argmax under `HwDp4a`. That reading was not Q8_1 quantization error. No qwen35 +/// writer kernel ever cleared `q8_activation_valid`, so every DP4A GEMV after the +/// first reused the FIRST token's Q8_1 activation (#4258). With the activation cache +/// keyed to its input and invalidated by the writers, the same run passes at every +/// position. So this now guards the fix: if the stale-activation reuse comes back, +/// the argmax breaks here. /// -/// Measured on the real 0.8B file at position 0: see the printed reading. The -/// DeltaNet-only path is 1.656 relative away from the CPU under `HwDp4a` versus -/// 0.000 float-vs-float; end to end the argmax is simply wrong. +/// The float pin in `Qwen35CudaModel::with_max_seq_len` stays until DP4A is +/// re-measured on 2B/4B (#4030). This test only says the pin's stated reason is gone. #[test] #[serial_test::serial] -fn qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence() { +fn qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence() { use crate::cuda::gpu_profile::{Q4kVariant, Q6kVariant}; let executor = qwen35_cuda_fixture_or_skip!(); let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); @@ -1289,11 +1336,12 @@ fn qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence() { } } - assert!( - broken > 0, - "the DP4A GEMV path passed the parity contract at every position — it is no longer \ - catastrophic through the recurrence, so re-measure and revisit the pin in \ - Qwen35CudaModel::with_max_seq_len (DP4A-through-recurrence ticket, 0.69.0)" + assert_eq!( + broken, + 0, + "the DP4A GEMV path broke the parity contract at {broken} of {} positions: a stale \ + Q8_1 activation is being reused again (#4258)", + LONG_PROMPT.len() ); } diff --git a/docs/audits/impl-PMAT-4258-receipt.md b/docs/audits/impl-PMAT-4258-receipt.md new file mode 100644 index 0000000000..b83c030982 --- /dev/null +++ b/docs/audits/impl-PMAT-4258-receipt.md @@ -0,0 +1,33 @@ +# Implementation receipt — PMAT-4258 (#4258), carried onto batch PR #4322 + +## Provenance +The fix was written on `fix/q8-flag-qwen35-4258` (PR #4302). That round was AGREED 3/3 PASS on head 4482ab98b. The +PR was then parked by the emergency fold #4311 because of a conflict against release/0.69.3. Here it is cherry-picked +(`-x`) onto main-based `perf/0694-qwen35-persistent-embed`: + +- 263958fc3 ← 1f14aea95 (fix) +- 56c9f9112 ← e9d074b83 (test inputs) +- 4d4a7841c ← 4482ab98b (roadmap) + +The code commits applied cleanly. The roadmap conflict was resolved as the union of both sides. + +## Change carried +`ensure_q8_activation` keys the cached Q8_1 activation by `(ptr, len)`. The qwen35 writer kernels invalidate it: +`rmsnorm_into`, `per_head_rmsnorm_into`, `residual_add_into`, `fused_swiglu_into` and `gdn_sigmoid_gate_into`. + +## New on this branch (555a6774d) +`qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence` asserted DP4A breaks parity. It now fails, as its own +message intends. It becomes `qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence`, which asserts 0 broken +positions, so a return of the stale reuse is caught. The pin comment in `with_max_seq_len` now states the real cause. +The float pin itself is NOT changed: lifting it waits on a 2B/4B re-measure (#4030). + +## Gates (head 555a6774d) +- Build (`--release --features cuda --no-run`): rc=0. +- `clippy --lib --features cuda -D warnings`: rc=0. +- `fmt --check`: rc=0. +- Roadmap sorted/unique: PASS. +- GPU tests on RTX 4090 via gpu-q, with compute-apps empty at start (logs in `/mnt/nvme-raid0/tmp/q8-gpu/`): + - `qwen35_cuda*`: 17 passed, 0 failed. + - `tests_q8_activation_staleness`: 2 passed. Max |DP4A − float| is 3.119e-3 for the second buffer and 6.488e-4 + after `rmsnorm_into`. + - DP4A on 0.8B over 6 positions: argmax equal at every position, cosine 0.997423 to 0.999476. diff --git a/docs/audits/impl-PMAT-4316-receipt.md b/docs/audits/impl-PMAT-4316-receipt.md new file mode 100644 index 0000000000..120a96e2fd --- /dev/null +++ b/docs/audits/impl-PMAT-4316-receipt.md @@ -0,0 +1,39 @@ +# Implementation receipt — PMAT-4316 (#4316, part of 0.69.4 epic #4249) + +Author: claude-opus-5-5. Base: origin/main @ aa7c6ef03. + +## Claim +`Qwen35CudaModel::forward_single` (crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs) called +`GpuBuffer::from_host` on every token: a cuMemAlloc, a blocking HtoD copy and a cuMemFree per token, with +a device pointer that changed each time. No captured CUDA graph can replay that (#4215). + +## Change +- `CudaExecutor::upload_into_async` (executor_api.rs): an `unsafe fn` that queues `copy_from_host_async` + on the execution stream (`self.stream`, the stream `sync_stream` waits on and the layer kernels run on). + Its safety contract, that `src` stays valid until the stream syncs, is passed on to the caller. +- `Qwen35CudaModel.residual: Option>` (`[hidden_dim]`) is allocated once in `new`. + `forward_single` validates the input, takes the buffer out of `self` (the layer calls take `&mut self`), + runs `forward_resident`, and puts the buffer back on every return path, error paths included. It is + re-allocated only if a panic mid-token ever lost it. +- `forward_resident` = the old body after the checks, starting with an in-stream upload of the + embedding row into the resident buffer. The row borrows `self.model`'s embedding table (`&'a`), + which outlives `self`, so the async copy's source memory is alive whenever the copy runs. +- `forward_hidden_deltanet_only` (a test/diagnostic helper, not the decode path) is unchanged. + +## Test +- NEW `qwen35_cuda_the_residual_is_one_buffer_across_tokens`: the device pointer is identical after + construction, after each of 6 tokens, after a refused (out-of-vocab) token, and after the token that + follows the refusal. +- Unchanged parity oracle `qwen35_cuda_forward_single_matches_cpu_logits_end_to_end`. It catches a missed + or stale upload, because a stale residual is the previous token's hidden state, not this token's + embedding. + +## Gates +- `cargo test --release -p aprender-serve --lib --features cuda --no-run`: rc=0. +- `cargo clippy -p aprender-serve --lib --features cuda -- -D warnings`: rc=0. + (`--tests` clippy has 1284 pre-existing errors, all in `quantize/iq*` test tables and none in these files.) +- `cargo fmt --all -- --check`: rc=0. Roadmap guards (sorted, ids unique): PASS. +- GPU run of `qwen35_cuda*` on RTX 4090 through gpu-q, with compute-apps empty at start (the test binary built from + this code, commit 02729ec9a; the later commits are docs only): **17 passed, 0 failed, 0 skipped**, in 24.6 s. + e2e logits worst cosine 0.998348 / rel L-inf 6.721e-2 over 6 positions. 4B argmax end-to-end passes. + `qwen35_cuda_the_residual_is_one_buffer_across_tokens` ok. Log: /mnt/nvme-raid0/tmp/embed-gpu/test-bba6b0924.log diff --git a/docs/audits/quorum-PMAT-4258-receipt.md b/docs/audits/quorum-PMAT-4258-receipt.md new file mode 100644 index 0000000000..7c5a55b5c9 --- /dev/null +++ b/docs/audits/quorum-PMAT-4258-receipt.md @@ -0,0 +1,25 @@ +# Quorum receipt — batch #4322, delta PMAT-4258 (#4258 stale-Q8 carry + DP4A parity guard) + +Author: claude-opus-5-5. Base: 86498a9db, the last head of this branch that a quorum or the GPU had judged; the +PMAT-4316 part is covered by `quorum-PMAT-4316-receipt.md`. Shape: sonnet-5 + 1 agy gemini-3.1-pro-high + haiku-4-5 +(operator 2026-09-24). + +Why a delta round: the full-batch brief was 52 KB. Everything below 86498a9db was already judged 3/3, and the carried +#4302 code was agreed 3/3 on 4482ab98b. This round judges the cherry-picks as landed here, plus the new 555a6774d. + +## Round 1 — AGREED 3/3 PASS on head 36f1ea707 +Every lane got the same brief: 33300 B, sha256 aa728804…, diff_sha256 7f8fff34f272…. + +| lane | model | verdict | +|------|-------|---------| +| agy | gemini-3.1-pro-high (measured) | PASS. Its 3 findings cite the acceptance criteria as met. Artifact: `quorum-PMAT-4258.json` | +| claude | claude-sonnet-5 | PASS. It traced every qwen35 GEMV input. The residual that the host upload writes feeds only rmsnorm/residual_add, never a GEMV, so the (ptr,len) key's host-upload limit does not apply. The dense float path never calls `ensure_q8_activation`. 1 finding: stale test name in `evidence/parity/thresholds.yaml:95` | +| claude | claude-haiku-4-5 | PASS, 0 findings. It checked that the receipt matches the GPU logs | + +## After the round +The one post-round commit touches only a comment: it annotates the stale test name in `evidence/parity/thresholds.yaml`, +as sonnet's finding asked. It changes no code or thresholds, and the YAML still parses. + +## GPU evidence +GREEN on 555a6774d: `qwen35_cuda*` passed 17/17 and the q8 staleness tests passed 2/2. See +`impl-PMAT-4258-receipt.md` §Gates. diff --git a/docs/audits/quorum-PMAT-4258.json b/docs/audits/quorum-PMAT-4258.json new file mode 100644 index 0000000000..d9e67315ff --- /dev/null +++ b/docs/audits/quorum-PMAT-4258.json @@ -0,0 +1,157 @@ +{ + "ticket": "PMAT-4258", + "base": "86498a9db", + "base_resolved": "86498a9db", + "base_note": "no origin/86498a9db exists; judged against the local ref", + "head": "36f1ea707d785d49f9bac0f989306b8f3ab0f457", + "diff_sha256": "7f8fff34f272251144f739a18252213231e23a294d9fc2c708ea3d8c2990faf6", + "width": 1, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 33300, + "author": { + "model": "claude-opus-5-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements the required cache invalidation logic for Q8_1 activations. It keys the cached activation by `(ptr, len)` and invalidates it across all requested qwen35 writer kernels (`rmsnorm_into`, `per_head_rmsnorm_into`, `residual_add_into`, `fused_swiglu_into`, `gdn_sigmoid_gate_into`). The AC is fully met and accompanied by the requested tests, and the existing falsifier is properly updated to reflect that DP4A now holds parity through the recurrence.", + "findings": [ + { + "claim": "A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation", + "file": "crates/aprender-serve/src/cuda/executor/tiled_gemv.rs", + "grounding": "cited" + }, + { + "claim": "or a qwen35 writer kernel rewrote it", + "file": "crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs", + "grounding": "cited" + }, + { + "claim": "a red-first test covers both", + "file": "crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs", + "grounding": "cited" + } + ], + "raw_bytes": 3179, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "or a qwen35 writer kernel rewrote it" + ] + }, + { + "file": "crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "a red-first test covers both" + ] + }, + { + "file": "crates/aprender-serve/src/cuda/executor/tiled_gemv.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=1 author=claude-opus-5-5/claude" + } +} diff --git a/docs/audits/quorum-PMAT-4316-receipt.md b/docs/audits/quorum-PMAT-4316-receipt.md new file mode 100644 index 0000000000..93a51abc48 --- /dev/null +++ b/docs/audits/quorum-PMAT-4316-receipt.md @@ -0,0 +1,17 @@ +# Quorum receipt — PMAT-4316 (#4316, qwen35 CUDA resident residual) + +Author: claude-opus-5-5. Base: origin/main @ aa7c6ef03. Shape: sonnet-5 + 1 agy gemini-3.1-pro-high + haiku-4-5 (operator 2026-09-24). + +## Round 1 — AGREED 3/3 PASS on head bba6b0924 +Every lane got the same brief: 16294 B, sha256 217e98df…76b121de. The agy artifact records prompt_bytes 16294 and diff_sha256 2b6d7a02…. + +| lane | model | verdict | +|------|-------|---------| +| agy | gemini-3.1-pro-high (measured) | PASS. Its 2 findings confirm claims and carry `fix: none`. Artifact: `quorum-PMAT-4316.json` | +| claude | claude-sonnet-5 | PASS, 0 findings. It grep-verified that every layer launch and `sync_stream` use `self.stream` | +| claude | claude-haiku-4-5 | PASS, 0 findings | + +Earlier attempts did not count. The first agy round (head 635bd6761) was aborted when a concurrent `--brief-only` run shared its lanes dir. The Claude lanes were refused by kind-gate until `kind:code` was added, in bba6b0924. + +## GPU evidence +GREEN: `qwen35_cuda*` passed 17/17 on RTX 4090. See `impl-PMAT-4316-receipt.md` §Gates. diff --git a/docs/audits/quorum-PMAT-4316.json b/docs/audits/quorum-PMAT-4316.json new file mode 100644 index 0000000000..bea6761e60 --- /dev/null +++ b/docs/audits/quorum-PMAT-4316.json @@ -0,0 +1,146 @@ +{ + "ticket": "PMAT-4316", + "base": "origin/main", + "base_resolved": "origin/main", + "base_note": "no origin/origin/main exists; judged against the local ref (fetch failed: origin/origin/main may itself be stale)", + "head": "bba6b092429ac361bd47db645b9ac2b7635c9bbf", + "diff_sha256": "2b6d7a02371945e66299d84ef4db04bf25322d8159fae5505c61c51f4c25574d", + "width": 1, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 16294, + "author": { + "model": "claude-opus-5-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements the resident residual buffer and in-stream embedding upload as claimed in the receipt for PMAT-4316. The `GpuBuffer::from_host` is replaced with an `upload_into_async` call, eliminating per-token allocations. The buffer is safely checked out and returned even on error paths. The new test qwen35_cuda_the_residual_is_one_buffer_across_tokens verifies this. No refutations found.", + "findings": [ + { + "claim": "puts the buffer back on every return path, error paths included", + "file": "crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs", + "fix": "none", + "grounding": "cited", + "line": 1415 + }, + { + "claim": "upload_into_async is an unsafe fn that queues copy_from_host_async on the execution stream", + "file": "crates/aprender-serve/src/cuda/executor/executor_api.rs", + "fix": "none", + "grounding": "cited", + "line": 493 + } + ], + "raw_bytes": 3036, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-serve/src/cuda/executor/executor_api.rs", + "line": 493, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "upload_into_async is an unsafe fn that queues copy_from_host_async on the execution stream" + ] + }, + { + "file": "crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs", + "line": 1415, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "puts the buffer back on every return path, error paths included" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=1 author=claude-opus-5-5/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-4258.yaml b/docs/roadmaps/entries/PMAT-4258.yaml new file mode 100644 index 0000000000..02fabb1eeb --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4258.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4258 + github_issue: 4258 + item_type: task + title: 'Qwen3.5 CUDA: q8_activation_valid is never cleared on the qwen35 path, so DP4A reuses a stale Q8_1 activation' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T10:25:13Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation or a qwen35 writer kernel (rmsnorm_into, per_head_rmsnorm_into, residual_add_into, fused_swiglu_into, gdn_sigmoid_gate_into) rewrote it; a red-first test covers both.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found in /mnt/nvme-raid0/tmp/alt0693/REPORT.md "Extra finding". The DP4A-catastrophic falsifier now passes parity at every position on 0.8B; re-measure the #4030 pin on 2B/4B.' diff --git a/docs/roadmaps/entries/PMAT-4316.yaml b/docs/roadmaps/entries/PMAT-4316.yaml new file mode 100644 index 0000000000..8f12882137 --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4316.yaml @@ -0,0 +1,17 @@ +- id: PMAT-4316 + github_issue: 4316 + item_type: task + title: 'perf(qwen35-cuda): resident residual buffer + in-stream embedding upload (0.69.4)' + status: in_progress + priority: high + assigned_to: aprender-36 + created: 2026-09-24T14:58:07Z + updated: 2026-09-24T14:58:07Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index 90671942ca..4c7e6b502b 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -20610,3 +20610,38 @@ roadmap: labels: - kind:code notes: null +- id: PMAT-4258 + github_issue: 4258 + item_type: task + title: 'Qwen3.5 CUDA: q8_activation_valid is never cleared on the qwen35 path, so DP4A reuses a stale Q8_1 activation' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T10:25:13Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation or a qwen35 writer kernel (rmsnorm_into, per_head_rmsnorm_into, residual_add_into, fused_swiglu_into, gdn_sigmoid_gate_into) rewrote it; a red-first test covers both.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found in /mnt/nvme-raid0/tmp/alt0693/REPORT.md "Extra finding". The DP4A-catastrophic falsifier now passes parity at every position on 0.8B; re-measure the #4030 pin on 2B/4B.' +- id: PMAT-4316 + github_issue: 4316 + item_type: task + title: 'perf(qwen35-cuda): resident residual buffer + in-stream embedding upload (0.69.4)' + status: in_progress + priority: high + assigned_to: aprender-36 + created: 2026-09-24T14:58:07Z + updated: 2026-09-24T14:58:07Z + spec: null + acceptance_criteria: [] + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: null diff --git a/evidence/parity/thresholds.yaml b/evidence/parity/thresholds.yaml index b182d28a4a..57114f5912 100644 --- a/evidence/parity/thresholds.yaml +++ b/evidence/parity/thresholds.yaml @@ -96,7 +96,11 @@ models: # on lambda: cosine 0.466 down to -0.246; on gx10 (peer session, same # head, 16:03Z): +0.179 down to -0.765 with the GPU argmax stuck at one # token at every position (the model pins the float GEMV for exactly - # this reason; the test is the standing falsifier on both hosts) + # this reason; the test is the standing falsifier on both hosts). + # #4258: that reading was a stale Q8_1 activation, not DP4A error. The + # test is now `qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence` + # (0.8B, lambda: cosine 0.997423..0.999476, argmax 6/6). The numbers above + # stay as a known-bad record of a real stale-activation defect. # margin: 0.998971 - 0.99 = 0.008971 under the lower good floor; 0.99 - 0.466 = 0.524 over the higher bad ceiling qwen3.5-0.8b: compute_class: gpu