diff --git a/crates/aprender-gpu/src/driver/memory/buffer.rs b/crates/aprender-gpu/src/driver/memory/buffer.rs index 2590cfbf61..01b67e648a 100644 --- a/crates/aprender-gpu/src/driver/memory/buffer.rs +++ b/crates/aprender-gpu/src/driver/memory/buffer.rs @@ -90,6 +90,31 @@ thread_local! { /// Device bytes this thread has allocated through `GpuBuffer` and not yet /// freed. static OUTSTANDING_DEVICE_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Device allocations this thread has made through `GpuBuffer`, ever (#4215). + static DEVICE_ALLOCS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Bytes this thread has copied device→host through `GpuBuffer`, ever (#4215). + static DEVICE_TO_HOST_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Device allocations the current thread has made through `GpuBuffer` since it +/// started — a monotonic count, so a caller measures a region by differencing +/// two readings (#4215: a steady-state decode token must allocate nothing). +#[must_use] +pub fn device_allocs_total() -> u64 { + DEVICE_ALLOCS.with(std::cell::Cell::get) +} + +/// Bytes the current thread has copied device→host through `GpuBuffer` since +/// it started, monotonic like [`device_allocs_total`] (#4215: a greedy decode +/// token downloads the 4-byte token id, not the logits). +#[must_use] +pub fn device_to_host_bytes_total() -> u64 { + DEVICE_TO_HOST_BYTES.with(std::cell::Cell::get) +} + +/// Record a device→host copy against the current thread. +pub(super) fn record_device_to_host(bytes: usize) { + DEVICE_TO_HOST_BYTES.with(|c| c.set(c.get().saturating_add(bytes as u64))); } /// Device bytes allocated by the current thread through `GpuBuffer` and not @@ -124,6 +149,7 @@ pub fn device_bytes_outstanding() -> u64 { /// Record a successful device allocation against the current thread. fn record_device_alloc(bytes: usize) { OUTSTANDING_DEVICE_BYTES.with(|c| c.set(c.get().saturating_add(bytes as u64))); + DEVICE_ALLOCS.with(|c| c.set(c.get().saturating_add(1))); } /// Record a device free against the current thread. diff --git a/crates/aprender-gpu/src/driver/memory/mod.rs b/crates/aprender-gpu/src/driver/memory/mod.rs index 8aae9bbbeb..ebf9a3c3b1 100644 --- a/crates/aprender-gpu/src/driver/memory/mod.rs +++ b/crates/aprender-gpu/src/driver/memory/mod.rs @@ -18,8 +18,8 @@ mod transfer; #[allow(unused_imports, unreachable_pub)] // Re-exported by driver/mod.rs pub use buffer::{ - classify_device_memory, device_bytes_outstanding, device_memory_exclusive, DeviceMemoryClass, - DeviceMemoryExclusive, GpuBuffer, GpuBufferView, + classify_device_memory, device_allocs_total, device_bytes_outstanding, device_memory_exclusive, + device_to_host_bytes_total, DeviceMemoryClass, DeviceMemoryExclusive, GpuBuffer, GpuBufferView, }; // ============================================================================ diff --git a/crates/aprender-gpu/src/driver/memory/transfer.rs b/crates/aprender-gpu/src/driver/memory/transfer.rs index 111d6cbc9c..7cd6a9f18e 100644 --- a/crates/aprender-gpu/src/driver/memory/transfer.rs +++ b/crates/aprender-gpu/src/driver/memory/transfer.rs @@ -15,7 +15,7 @@ use crate::GpuError; // zeros when the CUDA context is not current on the calling thread // (see paiml/trueno#232). -use super::buffer::GpuBuffer; +use super::buffer::{record_device_to_host, GpuBuffer}; // ============================================================================ // Blocking host-to-device primitive @@ -203,6 +203,7 @@ impl GpuBuffer { // SAFETY: data is valid for size bytes, ptr is valid device pointer let result = unsafe { (driver.cuMemcpyDtoH)(data.as_mut_ptr() as *mut c_void, self.ptr, size) }; + record_device_to_host(size); CudaDriver::check(result).map_err(|e| GpuError::Transfer(e.to_string())) } @@ -294,6 +295,7 @@ impl GpuBuffer { stream.raw(), ) }; + record_device_to_host(size); CudaDriver::check(result).map_err(|e| GpuError::Transfer(e.to_string())) } @@ -382,6 +384,7 @@ impl GpuBuffer { // SAFETY: bounds checked above, data and ptr are valid let result = unsafe { (driver.cuMemcpyDtoH)(data.as_mut_ptr() as *mut c_void, src_ptr, size) }; + record_device_to_host(size); CudaDriver::check(result).map_err(|e| GpuError::Transfer(e.to_string())) } diff --git a/crates/aprender-gpu/src/driver/mod.rs b/crates/aprender-gpu/src/driver/mod.rs index 7470f0acbf..8ee6fb649c 100644 --- a/crates/aprender-gpu/src/driver/mod.rs +++ b/crates/aprender-gpu/src/driver/mod.rs @@ -128,8 +128,8 @@ pub use cublaslt::CublasLtHandle; pub use graph::{CaptureMode, CudaGraph, CudaGraphExec}; #[cfg(feature = "cuda")] pub use memory::{ - classify_device_memory, device_bytes_outstanding, device_memory_exclusive, DeviceMemoryClass, - DeviceMemoryExclusive, GpuBuffer, + classify_device_memory, device_allocs_total, device_bytes_outstanding, device_memory_exclusive, + device_to_host_bytes_total, DeviceMemoryClass, DeviceMemoryExclusive, GpuBuffer, }; pub use module::CudaModule; #[cfg(feature = "cuda")] diff --git a/crates/aprender-gpu/src/kernels/argmax.rs b/crates/aprender-gpu/src/kernels/argmax.rs index 57d9bd8964..29fb6a1a6d 100644 --- a/crates/aprender-gpu/src/kernels/argmax.rs +++ b/crates/aprender-gpu/src/kernels/argmax.rs @@ -19,8 +19,29 @@ //! - Expected speedup: ~1.2x overall (from ~163 tok/s to ~200 tok/s) use super::Kernel; -use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; -use crate::ptx::{PtxKernel, PtxType}; +use crate::ptx::builder::{KernelBuilder, PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxType, VirtualReg}; + +/// Does the candidate at `tid + stride` replace this thread's current best? +/// +/// Larger value wins; on an EXACT tie the lower index wins, so the reduction +/// returns the FIRST maximum — the contract of the CPU `ops::argmax` this +/// kernel replaces (#4215). A strict `>` alone keeps whichever side the tree +/// happened to put in `tid`, which after the strided load can be the later +/// index. Both comparisons are ordered, so a NaN never wins. +fn other_wins( + ctx: &mut KernelBuilder<'_>, + other_val: VirtualReg, + other_idx: VirtualReg, + my_val: VirtualReg, + my_idx: VirtualReg, +) -> VirtualReg { + let greater = ctx.setp_gt_f32(other_val, my_val); + let equal = ctx.setp_eq_f32(other_val, my_val); + let earlier = ctx.setp_lt_u32(other_idx, my_idx); + let tie_to_earlier = ctx.and_pred(equal, earlier); + ctx.or_pred(greater, tie_to_earlier) +} /// ArgMax kernel configuration /// @@ -175,7 +196,7 @@ impl Kernel for ArgMaxKernel { let my_val = ctx.ld_generic_f32(sh_val_addr); let my_idx = ctx.ld_generic_u32(sh_idx_addr); - let is_greater = ctx.setp_gt_f32(other_val, my_val); + let is_greater = other_wins(ctx, other_val, other_idx, my_val, my_idx); let new_val = ctx.selp_f32(is_greater, other_val, my_val); let new_idx = ctx.selp_u32(is_greater, other_idx, my_idx); @@ -201,7 +222,7 @@ impl Kernel for ArgMaxKernel { let my_val = ctx.ld_generic_f32(sh_val_addr); let my_idx = ctx.ld_generic_u32(sh_idx_addr); - let is_greater = ctx.setp_gt_f32(other_val, my_val); + let is_greater = other_wins(ctx, other_val, other_idx, my_val, my_idx); let new_val = ctx.selp_f32(is_greater, other_val, my_val); let new_idx = ctx.selp_u32(is_greater, other_idx, my_idx); @@ -328,7 +349,7 @@ impl Kernel for ArgMaxFinalKernel { let my_val = ctx.ld_generic_f32(sh_val_addr); let my_idx = ctx.ld_generic_u32(sh_idx_addr); - let is_greater = ctx.setp_gt_f32(other_val, my_val); + let is_greater = other_wins(ctx, other_val, other_idx, my_val, my_idx); let new_val = ctx.selp_f32(is_greater, other_val, my_val); let new_idx = ctx.selp_u32(is_greater, other_idx, my_idx); diff --git a/crates/aprender-gpu/src/ptx/builder/bitwise_ops.rs b/crates/aprender-gpu/src/ptx/builder/bitwise_ops.rs index dc2bc1bf5f..eb6a8cd9d4 100644 --- a/crates/aprender-gpu/src/ptx/builder/bitwise_ops.rs +++ b/crates/aprender-gpu/src/ptx/builder/bitwise_ops.rs @@ -184,6 +184,18 @@ impl<'a> KernelBuilder<'a> { dst } + /// OR two predicates: dst = a OR b + pub fn or_pred(&mut self, a: VirtualReg, b: VirtualReg) -> VirtualReg { + let dst = self.registers.allocate_virtual(PtxType::Pred); + self.instructions.push( + PtxInstruction::new(PtxOp::Or, PtxType::Pred) + .dst(Operand::Reg(dst)) + .src(Operand::Reg(a)) + .src(Operand::Reg(b)), + ); + dst + } + /// Get shared memory base pointer /// /// PAR-062: Returns base address of shared memory for this block diff --git a/crates/aprender-gpu/src/ptx/builder/comparison.rs b/crates/aprender-gpu/src/ptx/builder/comparison.rs index a55b861fa5..a63d46725d 100644 --- a/crates/aprender-gpu/src/ptx/builder/comparison.rs +++ b/crates/aprender-gpu/src/ptx/builder/comparison.rs @@ -118,6 +118,18 @@ pub trait PtxComparison: KernelBuilderCore { pred } + /// Set predicate if a == b (f32). Ordered: false when either is NaN. + fn setp_eq_f32(&mut self, a: VirtualReg, b: VirtualReg) -> VirtualReg { + let pred = self.registers_mut().allocate_virtual(PtxType::Pred); + let mut instr = PtxInstruction::new(PtxOp::Setp, PtxType::F32) + .dst(Operand::Reg(pred)) + .src(Operand::Reg(a)) + .src(Operand::Reg(b)); + instr.label = Some(CmpOp::Eq.to_ptx_string().to_string()); + self.instructions_mut().push(instr); + pred + } + /// Set predicate comparing u32 with immediate fn setp_ge_u32_imm(&mut self, a: VirtualReg, b: u32) -> VirtualReg { let pred = self.registers_mut().allocate_virtual(PtxType::Pred); @@ -226,9 +238,13 @@ mod tests { let _lt = builder.setp_lt_f32(a, b); let _gt = builder.setp_gt_f32(a, b); - assert_eq!(builder.instructions.len(), 2); + let _eq = builder.setp_eq_f32(a, b); + + assert_eq!(builder.instructions.len(), 3); assert_eq!(builder.instructions[0].ty, PtxType::F32); assert_eq!(builder.instructions[1].ty, PtxType::F32); + assert_eq!(builder.instructions[2].ty, PtxType::F32); + assert_eq!(builder.instructions[2].label.as_deref(), Some("eq")); } #[test] diff --git a/crates/aprender-gpu/tests/argmax_falsification_f114/main.rs b/crates/aprender-gpu/tests/argmax_falsification_f114/main.rs index d9a6035208..f08b52b2e1 100644 --- a/crates/aprender-gpu/tests/argmax_falsification_f114/main.rs +++ b/crates/aprender-gpu/tests/argmax_falsification_f114/main.rs @@ -16,3 +16,5 @@ mod barrier_bounds; mod gpu_correctness; #[cfg(feature = "cuda")] mod ptx_analysis; +#[cfg(feature = "cuda")] +mod tie_break; diff --git a/crates/aprender-gpu/tests/argmax_falsification_f114/tie_break.rs b/crates/aprender-gpu/tests/argmax_falsification_f114/tie_break.rs new file mode 100644 index 0000000000..fb7aa7d7b4 --- /dev/null +++ b/crates/aprender-gpu/tests/argmax_falsification_f114/tie_break.rs @@ -0,0 +1,149 @@ +//! F114 tie-break (#4215): the GPU argmax must return the FIRST maximum, the +//! contract of the CPU `ops::argmax` it replaces on the greedy decode path. +//! +//! The tree reduction compares thread `tid` against `tid + stride`, but after a +//! strided load `tid` can already hold the LARGER index (thread 0 owns +//! 0/256/512/768, thread 1 owns 1/257/…), so a strict `>` keeps index 768 over +//! an exactly equal value at index 1. The same holds across blocks in the final +//! pass once a subtree has absorbed a later block. On a temp-0 decode an exact +//! tie then emits a different token than the CPU path — a silent divergence. +//! +//! Each case below states the index the CPU contract picks; the pre-fix kernel +//! returns the other one. + +use std::ffi::c_void; +use trueno_gpu::driver::{CudaContext, CudaModule, CudaStream, GpuBuffer, LaunchConfig}; +use trueno_gpu::kernels::{ArgMaxFinalKernel, ArgMaxKernel, Kernel}; + +/// The CPU contract (`aprender-serve` `gguf::ops::argmax`): strict `>` in index +/// order, so the first maximum wins and NaN never does. +fn cpu_first_argmax(v: &[f32]) -> u32 { + let (mut best, mut best_val) = (0u32, f32::NEG_INFINITY); + for (i, &x) in v.iter().enumerate() { + if x > best_val { + best_val = x; + best = i as u32; + } + } + best +} + +/// Both passes, exactly as `CudaExecutor::gpu_argmax` launches them. `None` +/// when this host has no CUDA device, so the caller can say it did not measure. +fn gpu_argmax(input: &[f32]) -> Option { + let ctx = CudaContext::new(0).ok()?; + let stream = CudaStream::new(&ctx).expect("stream"); + let len = input.len() as u32; + + let block = ArgMaxKernel::new(len); + let num_blocks = block.num_blocks(); + let fin = ArgMaxFinalKernel::new(num_blocks); + let mut block_mod = CudaModule::from_ptx(&ctx, &block.emit_ptx()).expect("block ptx"); + let mut final_mod = CudaModule::from_ptx(&ctx, &fin.emit_ptx()).expect("final ptx"); + + let mut in_buf: GpuBuffer = GpuBuffer::new(&ctx, input.len()).expect("alloc"); + let mut vals: GpuBuffer = GpuBuffer::new(&ctx, num_blocks as usize).expect("alloc"); + let mut idxs: GpuBuffer = GpuBuffer::new(&ctx, num_blocks as usize).expect("alloc"); + let mut out: GpuBuffer = GpuBuffer::new(&ctx, 1).expect("alloc"); + in_buf.copy_from_host(input).expect("upload"); + + let mut args: [*mut c_void; 4] = [ + in_buf.as_kernel_arg(), + vals.as_kernel_arg(), + idxs.as_kernel_arg(), + &len as *const u32 as *mut c_void, + ]; + let cfg = LaunchConfig { + grid: (num_blocks, 1, 1), + block: (256, 1, 1), + shared_mem: 0, + }; + // SAFETY: buffers sized to the kernel's immediates; args match `.param` order. + unsafe { stream.launch_kernel(&mut block_mod, block.name(), &cfg, &mut args) } + .expect("block launch"); + + let mut fargs: [*mut c_void; 4] = [ + vals.as_kernel_arg(), + idxs.as_kernel_arg(), + out.as_kernel_arg(), + &num_blocks as *const u32 as *mut c_void, + ]; + let fcfg = LaunchConfig { + grid: (1, 1, 1), + block: (256, 1, 1), + shared_mem: 0, + }; + // SAFETY: as above. + unsafe { stream.launch_kernel(&mut final_mod, fin.name(), &fcfg, &mut fargs) } + .expect("final launch"); + stream.synchronize().expect("sync"); + + let mut host = [0u32; 1]; + out.copy_to_host(&mut host).expect("download"); + Some(host[0]) +} + +/// Qwen3.5's vocabulary: 243 blocks, all of the final pass's tree in play. +const VOCAB: usize = 248_320; + +fn case(name: &str, input: &[f32]) { + let want = cpu_first_argmax(input); + let Some(got) = gpu_argmax(input) else { + // Not a pass: nothing was measured. APR_REQUIRE_CUDA=1 (the GPU host + // that runs this falsifier) turns the absence into a failure. + assert!( + std::env::var("APR_REQUIRE_CUDA").as_deref() != Ok("1"), + "{name}: APR_REQUIRE_CUDA=1 but no CUDA device" + ); + eprintln!("{name}: NOT MEASURED (no CUDA device)"); + return; + }; + assert_eq!(got, want, "{name}: GPU argmax {got}, CPU first-max {want}"); +} + +#[test] +fn f114_tie_within_block_picks_lowest_index() { + // Thread 0 reaches 768 (its 4th strided load); thread 1 holds 1. Final + // intra-block step compares tid 0 (idx 768) with tid 1 (idx 1). + let mut v = vec![-1.0f32; VOCAB]; + v[768] = 5.0; + v[1] = 5.0; + case("within-block", &v); +} + +#[test] +fn f114_tie_across_blocks_picks_lowest_index() { + // After the final pass's stride-128 step, tid 0 holds block 128's winner; + // the last step compares it with tid 1 (block 1). Block 1's element is + // the lower index. + let mut v = vec![-1.0f32; VOCAB]; + v[128 * 1024 + 7] = 5.0; + v[1024 + 3] = 5.0; + case("across-blocks", &v); +} + +#[test] +fn f114_tie_many_equal_maxima_picks_first() { + // Every element equal: the first index is the only correct answer. + case("all-equal", &vec![0.25f32; VOCAB]); +} + +#[test] +fn f114_nan_never_wins() { + // A NaN at a lower index than the real maximum must not be chosen, with or + // without the tie-break (setp.eq and setp.gt are ordered comparisons). + let mut v = vec![-1.0f32; VOCAB]; + v[3] = f32::NAN; + v[1] = 2.0; + v[900] = 2.0; + case("nan", &v); +} + +#[test] +fn f114_no_tie_is_unchanged() { + // Regression guard: distinct values, maximum late in the vocabulary. + let v: Vec = (0..VOCAB) + .map(|i| ((i as u64 * 2_654_435_761) % 1_000_003) as f32 * 1e-6) + .collect(); + case("distinct", &v); +} diff --git a/crates/aprender-serve/src/cuda/executor/core.rs b/crates/aprender-serve/src/cuda/executor/core.rs index 7065d2cedb..3ca4ed6315 100644 --- a/crates/aprender-serve/src/cuda/executor/core.rs +++ b/crates/aprender-serve/src/cuda/executor/core.rs @@ -52,6 +52,7 @@ impl CudaExecutor { #[cfg(any(debug_assertions, test))] module_key_ledger: Default::default(), weight_cache: HashMap::new(), + module_keys: Default::default(), named_fp16_weight_cache: HashMap::new(), // GH-174: SafeTensors F16 quantized_weight_cache: HashMap::new(), // PAR-005: quantized weight cache quantized_weight_types: HashMap::new(), // PAR-058: weight quant types @@ -188,6 +189,7 @@ impl CudaExecutor { num_sms: context.multiprocessor_count().unwrap_or(8) as u32, // PMAT-027: Q8 activation cache starts invalid q8_activation_valid: false, + q8_activation_src: Default::default(), fp8_act_cache: Default::default(), fp8_weight_row_absmax: HashMap::new(), fp8_act_scale_buf: None, diff --git a/crates/aprender-serve/src/cuda/executor/device.rs b/crates/aprender-serve/src/cuda/executor/device.rs index db61f3900b..0f4bde205f 100644 --- a/crates/aprender-serve/src/cuda/executor/device.rs +++ b/crates/aprender-serve/src/cuda/executor/device.rs @@ -29,14 +29,14 @@ impl CudaExecutor { // PAR-058: Zero allocation Q4_1 GEMV for Qwen2.5-0.5B FFN down let kernel_type = KernelType::Q4_1Gemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q4_1_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q4_1_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -63,7 +63,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -102,14 +102,14 @@ impl CudaExecutor { // PAR-058: Zero allocation Q5K GEMV for mixed-quantization models let kernel_type = KernelType::Q5KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q5k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q5k_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -136,7 +136,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/cuda/executor/gdn_ops.rs b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs index 2080b3f62e..9f1acb2d4f 100644 --- a/crates/aprender-serve/src/cuda/executor/gdn_ops.rs +++ b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs @@ -16,6 +16,27 @@ use super::*; +/// The most argument slots any Gated `DeltaNet` kernel takes. +const GDN_MAX_ARGS: usize = 16; + +/// Pack `ptrs` then `scalars` into `args`; returns the slot count. +fn gdn_pack_args( + kernel_name: &str, + args: &mut [u64; GDN_MAX_ARGS], + ptrs: &[u64], + scalars: &[u64], +) -> Result { + let n = ptrs.len() + scalars.len(); + if n > GDN_MAX_ARGS { + return Err(GpuError::InvalidParameter(format!( + "{kernel_name}: {n} arguments exceed {GDN_MAX_ARGS}" + ))); + } + args[..ptrs.len()].copy_from_slice(ptrs); + args[ptrs.len()..n].copy_from_slice(scalars); + Ok(n) +} + impl CudaExecutor { /// Compile (once) and fetch the module for a Gated `DeltaNet` kernel. /// @@ -49,13 +70,19 @@ impl CudaExecutor { ptrs: &[u64], ) -> Result<(), GpuError> { for (i, &p) in ptrs.iter().enumerate() { - validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + validate_kernel_arg(p, kernel_name, i)?; + // #4258: any pointer argument may be an output (several GDN kernels + // write in place), so any of them can stale the Q8 activation. + self.q8_activation_written(p); + } + // #4215: argument slots live on the stack; the `Vec` is built only when + // a graph is being recorded. + let mut args = [0u64; GDN_MAX_ARGS]; + let n = gdn_pack_args(kernel_name, &mut args, ptrs, &[])?; + let mut raw = [std::ptr::null_mut::(); GDN_MAX_ARGS]; + for (r, a) in raw.iter_mut().zip(args.iter_mut()) { + *r = std::ptr::from_mut(a).cast::(); } - let mut args: Vec = ptrs.to_vec(); - let mut raw: Vec<*mut std::ffi::c_void> = args - .iter_mut() - .map(|a| std::ptr::from_mut(a).cast::()) - .collect(); let module = self .modules @@ -66,7 +93,7 @@ impl CudaExecutor { // argument order matches the kernel's `.param` declarations. unsafe { self.stream - .launch_kernel(module, kernel_name, &config, &mut raw)?; + .launch_kernel(module, kernel_name, &config, &mut raw[..n])?; } // trueno#243 / #3413: the manual decode graph is rebuilt ONLY from @@ -78,7 +105,7 @@ impl CudaExecutor { self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), config, - arg_data: args, + arg_data: args[..n].to_vec(), }); } Ok(()) @@ -102,14 +129,19 @@ impl CudaExecutor { scalars: &[u64], ) -> Result<(), GpuError> { for (i, &p) in ptrs.iter().enumerate() { - validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + validate_kernel_arg(p, kernel_name, i)?; + // #4258: any pointer argument may be an output (several GDN kernels + // write in place), so any of them can stale the Q8 activation. + self.q8_activation_written(p); + } + // #4215: argument slots live on the stack; the `Vec` is built only when + // a graph is being recorded. + let mut args = [0u64; GDN_MAX_ARGS]; + let n = gdn_pack_args(kernel_name, &mut args, ptrs, scalars)?; + let mut raw = [std::ptr::null_mut::(); GDN_MAX_ARGS]; + for (r, a) in raw.iter_mut().zip(args.iter_mut()) { + *r = std::ptr::from_mut(a).cast::(); } - let mut args: Vec = ptrs.to_vec(); - args.extend_from_slice(scalars); - let mut raw: Vec<*mut std::ffi::c_void> = args - .iter_mut() - .map(|a| std::ptr::from_mut(a).cast::()) - .collect(); let module = self .modules @@ -121,7 +153,7 @@ impl CudaExecutor { // kernel's `.param` declarations. unsafe { self.stream - .launch_kernel(module, kernel_name, &config, &mut raw)?; + .launch_kernel(module, kernel_name, &config, &mut raw[..n])?; } // trueno#243 / #3413: the manual decode graph is rebuilt ONLY from @@ -133,7 +165,7 @@ impl CudaExecutor { self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), config, - arg_data: args, + arg_data: args[..n].to_vec(), }); } Ok(()) @@ -161,7 +193,7 @@ impl CudaExecutor { num_heads, head_dim, }; - let cache_key = format!("gdn_split_interleaved_{num_heads}_{head_dim}"); + let cache_key = module_key!(self, "gdn_split_interleaved_{}_{}", num_heads, head_dim); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -201,9 +233,13 @@ impl CudaExecutor { head_dim, n_rot, }; - let cache_key = format!( - "gdn_partial_neox_rope_{num_heads}_{head_dim}_{n_rot}_{}", - Self::f32_bits_tag(theta_scale) + let cache_key = module_key!( + self, + "gdn_partial_neox_rope_{}_{}_{}_b{:08x}", + num_heads, + head_dim, + n_rot, + theta_scale.to_bits() ); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); @@ -249,7 +285,13 @@ impl CudaExecutor { num_kv_heads, head_dim, }; - let cache_key = format!("gdn_decode_attention_{num_heads}_{num_kv_heads}_{head_dim}"); + let cache_key = module_key!( + self, + "gdn_decode_attention_{}_{}_{}", + num_heads, + num_kv_heads, + head_dim + ); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -291,7 +333,7 @@ impl CudaExecutor { channels, kernel_size, }; - let cache_key = format!("gdn_causal_conv1d_silu_{channels}_{kernel_size}"); + let cache_key = module_key!(self, "gdn_causal_conv1d_silu_{}_{}", channels, kernel_size); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -331,7 +373,18 @@ impl CudaExecutor { epsilon, }; // epsilon is an immediate in the PTX, so it belongs in the cache key. - let cache_key = format!("gdn_per_head_l2_norm_{head_dim}_{num_heads}_{epsilon:e}"); + // `{epsilon:e}` is not an integer: key it by its bits, format it once. + let cache_key = self.module_key( + ( + "gdn_per_head_l2_norm_{}_{}_{:e}", + key_dims(&[ + u64::from(head_dim), + u64::from(num_heads), + u64::from(epsilon.to_bits()), + ]), + ), + || format!("gdn_per_head_l2_norm_{head_dim}_{num_heads}_{epsilon:e}"), + ); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -357,7 +410,7 @@ impl CudaExecutor { ) -> Result<(), GpuError> { let kernel = trueno_gpu::kernels::gdn::GdnGatesKernel::new(num_heads); let kernel_type = KernelType::GdnGates { num_heads }; - let cache_key = format!("gdn_gates_{num_heads}"); + let cache_key = module_key!(self, "gdn_gates_{}", num_heads); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -415,8 +468,14 @@ impl CudaExecutor { num_k_heads, head_k_dim, }; - let cache_key = - format!("gdn_delta_rule_{num_v_heads}_{head_v_dim}_{num_k_heads}_{head_k_dim}"); + let cache_key = module_key!( + self, + "gdn_delta_rule_{}_{}_{}_{}", + num_v_heads, + head_v_dim, + num_k_heads, + head_k_dim + ); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -460,7 +519,18 @@ impl CudaExecutor { num_heads, epsilon, }; - let cache_key = format!("gdn_gated_rmsnorm_{head_dim}_{num_heads}_{epsilon:e}"); + // `{epsilon:e}` is not an integer: key it by its bits, format it once. + let cache_key = self.module_key( + ( + "gdn_gated_rmsnorm_{}_{}_{:e}", + key_dims(&[ + u64::from(head_dim), + u64::from(num_heads), + u64::from(epsilon.to_bits()), + ]), + ), + || format!("gdn_gated_rmsnorm_{head_dim}_{num_heads}_{epsilon:e}"), + ); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); @@ -491,7 +561,7 @@ impl CudaExecutor { ) -> Result<(), GpuError> { let kernel = trueno_gpu::kernels::gdn::SigmoidGateKernel::new(n); let kernel_type = KernelType::GdnSigmoidGate { n }; - let cache_key = format!("gdn_sigmoid_gate_{n}"); + let cache_key = module_key!(self, "gdn_sigmoid_gate_{}", n); let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; let (gx, _, _) = kernel.grid(); let (bx, _, _) = kernel.block(); diff --git a/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs b/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs index e6df1edf38..59ae6fc9df 100644 --- a/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs +++ b/crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs @@ -154,6 +154,7 @@ impl CudaExecutor { hidden_size: u32, epsilon: f32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 // GH-559 DIAGNOSTIC: CPU RMSNorm bypass for Blackwell static CPU_RMSNORM: std::sync::OnceLock = std::sync::OnceLock::new(); let use_cpu_rmsnorm = *CPU_RMSNORM.get_or_init(|| { @@ -212,7 +213,7 @@ impl CudaExecutor { hidden_size, epsilon, }, - format!("rmsnorm_simple_{}_{}", hidden_size, Self::f32_bits_tag(epsilon)), + module_key!(self, "rmsnorm_simple_{}_b{:08x}", hidden_size, epsilon.to_bits()), ) } else if use_precise { ( @@ -220,7 +221,7 @@ impl CudaExecutor { hidden_size, epsilon, }, - format!("rmsnorm_precise_{}_{}", hidden_size, Self::f32_bits_tag(epsilon)), + module_key!(self, "rmsnorm_precise_{}_b{:08x}", hidden_size, epsilon.to_bits()), ) } else { ( @@ -228,24 +229,24 @@ impl CudaExecutor { hidden_size, epsilon, }, - format!("rmsnorm_vectorized_{}_{}", hidden_size, Self::f32_bits_tag(epsilon)), + module_key!(self, "rmsnorm_vectorized_{}_b{:08x}", hidden_size, epsilon.to_bits()), ) }; let kernel_name = self.kernels.kernel_name(&kernel_type); - if !self.modules.contains_key(&cache_key) { + if !self.modules.contains_key(&*cache_key) { let ptx = self.kernels.generate_ptx(&kernel_type); if use_simple { eprintln!("[GH-559] RmsNorm PTX ({} bytes)", ptx.len()); } let module = self.compile_ptx(&ptx)?; - self.modules.insert(cache_key.clone(), module); + self.modules.insert(cache_key.to_string(), module); } let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // PAR-081: 256 threads for vectorized, 32 threads for simple @@ -272,7 +273,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -308,19 +309,26 @@ impl CudaExecutor { num_heads: u32, epsilon: f32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::PerHeadRmsNorm { head_dim, num_heads, epsilon, }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("per_head_rmsnorm_{}_{}_{}", head_dim, num_heads, Self::f32_bits_tag(epsilon)); + let cache_key = module_key!( + self, + "per_head_rmsnorm_{}_{}_b{:08x}", + head_dim, + num_heads, + epsilon.to_bits() + ); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One warp (32 threads) per head, one block per head @@ -350,7 +358,7 @@ impl CudaExecutor { // diverged at position 1 on sm_89 and GB10 alike (position 0 is immune: // attention over one key is V, and RoPE(0) is the identity). if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/cuda/executor/layers/par-062.rs b/crates/aprender-serve/src/cuda/executor/layers/par-062.rs index cd5e148db4..bfd23e3ef9 100644 --- a/crates/aprender-serve/src/cuda/executor/layers/par-062.rs +++ b/crates/aprender-serve/src/cuda/executor/layers/par-062.rs @@ -141,6 +141,24 @@ impl CudaExecutor { self.argmax_num_blocks = num_blocks; } + // #4215: load both modules (cached after first use) before borrowing the argmax buffers. + let argmax_kernel_type = KernelType::ArgMax { length: vocab_size }; + let argmax_key = module_key!(self, "argmax_{}", vocab_size); + if !self.modules.contains_key(&*argmax_key) { + let ptx = self.kernels.generate_ptx(&argmax_kernel_type); + let module = self.compile_ptx(&ptx)?; + self.modules.insert(argmax_key.to_string(), module); + } + + // Load second-pass kernel module (cached after first use) + let final_kernel_type = KernelType::ArgMaxFinal { num_blocks }; + let final_key = module_key!(self, "argmax_final_{}", num_blocks); + if !self.modules.contains_key(&*final_key) { + let ptx = self.kernels.generate_ptx(&final_kernel_type); + let module = self.compile_ptx(&ptx)?; + self.modules.insert(final_key.to_string(), module); + } + let block_max_vals = self .argmax_block_vals .as_ref() @@ -154,24 +172,6 @@ impl CudaExecutor { .as_ref() .expect("argmax_result must be initialized"); - // Load first-pass kernel module (cached after first use) - let argmax_kernel_type = KernelType::ArgMax { length: vocab_size }; - let argmax_key = format!("argmax_{}", vocab_size); - if !self.modules.contains_key(&argmax_key) { - let ptx = self.kernels.generate_ptx(&argmax_kernel_type); - let module = self.compile_ptx(&ptx)?; - self.modules.insert(argmax_key.clone(), module); - } - - // Load second-pass kernel module (cached after first use) - let final_kernel_type = KernelType::ArgMaxFinal { num_blocks }; - let final_key = format!("argmax_final_{}", num_blocks); - if !self.modules.contains_key(&final_key) { - let ptx = self.kernels.generate_ptx(&final_kernel_type); - let module = self.compile_ptx(&ptx)?; - self.modules.insert(final_key.clone(), module); - } - // Prepare kernel arguments let kernel_name = self.kernels.kernel_name(&argmax_kernel_type); // PAR-068-FIX: Do NOT use .with_shared_mem() - PTX declares static shared memory via .shared directive @@ -187,7 +187,7 @@ impl CudaExecutor { unsafe { let module = self .modules - .get_mut(&argmax_key) + .get_mut(&*argmax_key) .expect("argmax module just inserted"); self.stream.launch_kernel( module, @@ -216,7 +216,7 @@ impl CudaExecutor { unsafe { let final_module = self .modules - .get_mut(&final_key) + .get_mut(&*final_key) .expect("argmax_final module just inserted"); self.stream.launch_kernel( final_module, diff --git a/crates/aprender-serve/src/cuda/executor/mod.rs b/crates/aprender-serve/src/cuda/executor/mod.rs index fe7e644b62..45ddc66e38 100644 --- a/crates/aprender-serve/src/cuda/executor/mod.rs +++ b/crates/aprender-serve/src/cuda/executor/mod.rs @@ -247,6 +247,55 @@ use crate::cuda::types::{ /// can recover — only a process restart fixes it. /// /// This check prevents kernel launch and returns a clean error instead. +/// #4215: the `modules` cache key `format!($fmt, $args…)`, formatted the first +/// time these arguments are seen and served from `module_keys` afterwards, so a +/// hot launch path neither allocates nor formats. Every argument must convert +/// to `u64` losslessly (pass an `f32` as `.to_bits()`, and format it yourself if +/// the key spells it differently); at most four. +macro_rules! module_key { + ($ex:expr, $fmt:literal $(, $arg:expr)* $(,)?) => {{ + let dims = $crate::cuda::executor::key_dims(&[$(u64::from($arg)),*]); + $ex.module_key(($fmt, dims), || format!($fmt $(, $arg)*)) + }}; +} +pub(crate) use module_key; + +/// Pad a cache key's integer arguments to the fixed width `module_keys` stores. +#[inline] +pub(crate) fn key_dims(args: &[u64]) -> [u64; 4] { + assert!(args.len() <= 4, "module_key!: at most four key arguments"); + let mut dims = [0u64; 4]; + dims[..args.len()].copy_from_slice(args); + dims +} + +impl CudaExecutor { + /// The formatted `modules` key for `id`, calling `make` only on the first + /// request for it (#4215). Use through [`module_key!`]. + pub(crate) fn module_key( + &mut self, + id: (&'static str, [u64; 4]), + make: impl FnOnce() -> String, + ) -> std::sync::Arc { + if let Some(key) = self.module_keys.get(&id) { + return std::sync::Arc::clone(key); + } + let key: std::sync::Arc = make().into(); + self.module_keys.insert(id, std::sync::Arc::clone(&key)); + key + } +} + +/// [`validate_device_ptr`] for argument `index` of `kernel`, naming it only on +/// failure — the name costs a `format!`, and the check runs on every launch. +#[inline] +fn validate_kernel_arg(ptr: u64, kernel: &str, index: usize) -> Result<(), GpuError> { + if ptr == 0 { + return validate_device_ptr(ptr, &format!("{kernel} arg {index}")); + } + Ok(()) +} + #[inline] fn validate_device_ptr(ptr: u64, name: &str) -> Result<(), GpuError> { if ptr == 0 { @@ -310,6 +359,8 @@ mod poison_trace_test; // FALSIFY-QDOT-008 (#3111): the Q5_K GEMV against gguf-py's values of a llama.cpp block #[cfg(test)] mod tests_q5k_ggml; +#[cfg(test)] +mod tests_q8_activation_staleness; // COV-003 through COV-006 (layer preload, kv_cache, attention, quantized) #[cfg(test)] @@ -367,6 +418,10 @@ pub struct CudaExecutor { /// #3759: debug and test builds record what each module key was compiled from. #[cfg(any(debug_assertions, test))] module_key_ledger: module_key_guard::ModuleKeyLedger, + /// #4215: `modules` keys already formatted, by (format string, integer + /// arguments) — see [`module_key!`]. A decode token looks its keys up here + /// instead of `format!`-ing ~600 of them. + module_keys: HashMap<(&'static str, [u64; 4]), std::sync::Arc>, // Persistent weight buffers on GPU (PARITY-037) // These are loaded once at startup and reused for all forward passes weight_cache: HashMap>, @@ -645,6 +700,11 @@ pub struct CudaExecutor { // Set to true after q8_quantize_into; callers invalidate (set false) // when the input buffer content changes (e.g. after RMSNorm write). q8_activation_valid: bool, + // #4258: which activation the Q8 cache holds — (source buffer ptr, element + // count). `ensure_q8_activation` re-quantizes when a GEMV's input is not this + // buffer, and every in-place writer clears `q8_activation_valid` when it writes + // this buffer, so a DP4A GEMV never reads another activation's Q8_1 bytes. + q8_activation_src: (u64, u32), // PMAT-084: FP8 activation cache — skip redundant absmax+convert when // multiple FP8 GEMMs share the same input (QKV phase, FFN gate+up). // Saves 84 kernel pairs per prefill (3 per layer × 28 layers). diff --git a/crates/aprender-serve/src/cuda/executor/module_key_guard.rs b/crates/aprender-serve/src/cuda/executor/module_key_guard.rs index 39861fd659..8e16709f9c 100644 --- a/crates/aprender-serve/src/cuda/executor/module_key_guard.rs +++ b/crates/aprender-serve/src/cuda/executor/module_key_guard.rs @@ -24,6 +24,11 @@ use trueno_gpu::GpuError; pub(crate) struct ModuleKeyLedger { /// key -> (hash of the PTX compiled under it, `Debug` of every request proven to produce it) entries: std::collections::HashMap)>, + /// Test-only: while set, a cache hit skips the proof. The proof `format!`s the request on + /// every hit, and a test that counts a decode token's host allocations (#4215) must measure + /// the lookup a release build does, not this debug-only check. + #[cfg(test)] + suspended: bool, } #[cfg(any(debug_assertions, test))] @@ -62,6 +67,10 @@ impl CudaExecutor { #[cfg(any(debug_assertions, test))] fn prove_module_key_complete(&mut self, key: &str, kernel_type: &KernelType) { + #[cfg(test)] + if self.module_key_ledger.suspended { + return; + } let request = format!("{kernel_type:?}"); // A key compiled outside this helper has no record: nothing to compare against. let Some((_, proven)) = self.module_key_ledger.entries.get(key) else { @@ -85,4 +94,11 @@ impl CudaExecutor { ); proven.push(request); } + + /// Test-only (#4215): suspend or resume the #3759 proof on cache hits. Misses still record + /// what they compile, so a key first compiled while suspended is proven on its next hit. + #[cfg(test)] + pub(crate) fn suspend_module_key_proof(&mut self, suspended: bool) { + self.module_key_ledger.suspended = suspended; + } } diff --git a/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs b/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs index 5747d70000..6bc12bee85 100644 --- a/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs +++ b/crates/aprender-serve/src/cuda/executor/q4k_mwv_gemv.rs @@ -17,13 +17,13 @@ impl CudaExecutor { let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::MwvQ4KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("mwv_q4k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "mwv_q4k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // num_warps * 32 threads per output element, one block per output @@ -71,7 +71,7 @@ impl CudaExecutor { // was unaffected because it routes Q4K to the (recorded) DP4A variant. // Fix: record this GEMV too, mirroring mwv_dp4a_q4k_gemv_into. if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -122,22 +122,19 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch DP4A GEMV kernel let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::MwvDp4aQ4KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("mwv_dp4a_q4k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "mwv_dp4a_q4k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -174,7 +171,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction // BUG FIX (realizr#198): MWV DP4A variant was missing recording. if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -223,22 +220,19 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch half-warp DP4A GEMV kernel let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::HwDp4aQ4KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("hw_dp4a_q4k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "hw_dp4a_q4k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -269,7 +263,7 @@ impl CudaExecutor { // trueno#243: Record kernel AFTER launch (avoids borrow conflict with module) if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -307,13 +301,13 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "dp4a_q4k_gemv_into")?; let kernel_type = KernelType::Dp4aQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("dp4a_q4k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "dp4a_q4k_gemv_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One warp (32 threads) per output element @@ -387,7 +381,7 @@ impl CudaExecutor { let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One block per output element, 256 threads per block @@ -455,21 +449,18 @@ impl CudaExecutor { // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::FusedGateUpSwigluHwDp4aQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("fused_gate_up_swiglu_hw_dp4a_q4k_{}_{}", k, n); + let cache_key = module_key!(self, "fused_gate_up_swiglu_hw_dp4a_q4k_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -505,7 +496,7 @@ impl CudaExecutor { // kernels per forward pass were never captured. Graph replay used stale // ffn_act_buf from the eager pass, causing hidden_buf2 divergence (81.0). if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -616,23 +607,19 @@ impl CudaExecutor { // Skip Q8 quantize when the same input buffer was already quantized // (e.g., K/V projections share input with Q, up shares with gate). // Saves 3 Q8 launches per layer × 28 layers = 84 launches per step. - if !self.q8_activation_valid { - let total_elements = m * k; - self.q8_quantize_into(input, &q8_buf, total_elements)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, m * k)?; // Step 2: Launch batched HW DP4A kernel let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::BatchedHwDp4aQ4KGemv { k, n, m, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("batched_hw_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); + let cache_key = module_key!(self, "batched_hw_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -684,13 +671,13 @@ impl CudaExecutor { let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::BatchedHwDp4aQ4KGemv { k, n, m, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("batched_hw_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); + let cache_key = module_key!(self, "batched_hw_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -742,13 +729,13 @@ impl CudaExecutor { let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::InlineQ8Dp4aQ4KGemv { k, n, m, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("inline_q8_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); + let cache_key = module_key!(self, "inline_q8_dp4a_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -800,13 +787,13 @@ impl CudaExecutor { let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::FusedFp32Q4KGemv { k, n, m, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("fused_fp32_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); + let cache_key = module_key!(self, "fused_fp32_q4k_gemv_{}_{}_{}_{}", k, n, m, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -1019,13 +1006,13 @@ impl CudaExecutor { validate_device_ptr(up_weight_ptr, "fused_gate_up_q4k_gemv_into(up)")?; let kernel_type = KernelType::FusedGateUpQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("fused_gate_up_q4k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "fused_gate_up_q4k_gemv_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One block per output element, 256 threads per block diff --git a/crates/aprender-serve/src/cuda/executor/q6k_gemv_indexed.rs b/crates/aprender-serve/src/cuda/executor/q6k_gemv_indexed.rs index ae9e544199..7f74578eba 100644 --- a/crates/aprender-serve/src/cuda/executor/q6k_gemv_indexed.rs +++ b/crates/aprender-serve/src/cuda/executor/q6k_gemv_indexed.rs @@ -61,7 +61,7 @@ impl CudaExecutor { let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // Allocate output buffer @@ -166,7 +166,7 @@ impl CudaExecutor { let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -220,13 +220,13 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "coalesced_q4k_gemv_into")?; let kernel_type = KernelType::CoalescedQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("coalesced_q4k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "coalesced_q4k_gemv_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One warp (32 threads) per output element @@ -277,13 +277,13 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "wide_q4k_gemv_into")?; let kernel_type = KernelType::WideQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("wide_q4k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "wide_q4k_gemv_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // PAR-132: 8 warps (256 threads) per output element @@ -341,13 +341,13 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "vectorized_q4k_gemv_into")?; let kernel_type = KernelType::VectorizedQ4KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("vectorized_q4k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "vectorized_q4k_gemv_{}_{}", k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // One warp (32 threads) per output element diff --git a/crates/aprender-serve/src/cuda/executor/residual.rs b/crates/aprender-serve/src/cuda/executor/residual.rs index 7d40957247..d1e438688c 100644 --- a/crates/aprender-serve/src/cuda/executor/residual.rs +++ b/crates/aprender-serve/src/cuda/executor/residual.rs @@ -301,16 +301,17 @@ impl CudaExecutor { output: &GpuBuffer, n: u32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::ResidualAdd { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); // GH-129: PTX is n-independent (n is a runtime param), so use constant cache key. - let cache_key = "residual_add".to_string(); + let cache_key = "residual_add"; self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads_per_block = 256u32; @@ -339,7 +340,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/cuda/executor/silu.rs b/crates/aprender-serve/src/cuda/executor/silu.rs index d778fd4be5..74c6093acd 100644 --- a/crates/aprender-serve/src/cuda/executor/silu.rs +++ b/crates/aprender-serve/src/cuda/executor/silu.rs @@ -275,15 +275,16 @@ impl CudaExecutor { output: &GpuBuffer, n: u32, ) -> Result<(), GpuError> { + self.q8_activation_written(output.as_ptr()); // #4258 let kernel_type = KernelType::FusedSwiglu { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("fused_swiglu_{}", n); + let cache_key = module_key!(self, "fused_swiglu_{}", n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = 256; @@ -312,7 +313,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs b/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs new file mode 100644 index 0000000000..b6bed92563 --- /dev/null +++ b/crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs @@ -0,0 +1,188 @@ +//! #4258: the DP4A GEMVs cache their Q8_1 activation behind `q8_activation_valid`. Before +//! #4258 nothing on the Qwen3.5 path cleared that flag, so every DP4A GEMV after the first +//! reused the FIRST quantized activation — across projections, layers and tokens. +//! +//! Each test runs a DP4A Q4_K GEMV whose input is NOT the activation last quantized, and +//! judges it against the float (Mwv) GEMV of the same input. A stale activation reproduces +//! the previous GEMV's output exactly; a fresh one lands within Q8_1 rounding of the float. + +use super::CudaExecutor; +use crate::cuda::gpu_profile::Q4kVariant; +use serial_test::serial; +use trueno_gpu::driver::GpuBuffer; + +const N: u32 = 32; +const K: u32 = 256; +/// Q8_1 activation rounding on these inputs is well under 1% relative L2. +const FRESH_TOL: f32 = 0.05; +/// The two inputs are built so their float outputs sit far apart; below this the test +/// could not tell a stale activation from a fresh one. +const DISTINCT_FLOOR: f32 = 0.2; + +/// `N` rows of one Q4_K super-block each: d = 1.0, dmin = 0, every 6-bit scale 1, +/// pseudo-random nibbles. +fn q4k_weights() -> Vec { + let mut seed = 0x2545_f491_u32; + let mut out = Vec::with_capacity(N as usize * 144); + for _ in 0..N { + out.extend_from_slice(&[0x00, 0x3C, 0x00, 0x00]); + out.extend_from_slice(&[0x01; 12]); + for _ in 0..128 { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + out.push((seed >> 16) as u8); + } + } + out +} + +fn input_a() -> Vec { + (0..K).map(|i| (i as f32 * 0.37).sin()).collect() +} + +fn input_b() -> Vec { + (0..K) + .map(|i| (i as f32 * 0.11).cos() * 2.0 - 0.5) + .collect() +} + +fn rel_l2(got: &[f32], want: &[f32]) -> f32 { + let num: f32 = got.iter().zip(want).map(|(g, w)| (g - w).powi(2)).sum(); + let den: f32 = want.iter().map(|w| w * w).sum(); + (num / den).sqrt() +} + +struct Rig { + ex: CudaExecutor, + w: u64, + out: GpuBuffer, +} + +impl Rig { + fn new(mut ex: CudaExecutor) -> Self { + ex.init_workspace(K as usize, K as usize) + .expect("init_workspace"); + ex.load_quantized_weights("q8_staleness_4258", &q4k_weights()) + .expect("load Q4_K"); + let w = ex + .get_quantized_weight_ptr("q8_staleness_4258") + .expect("weight ptr"); + let out = GpuBuffer::new(ex.context(), N as usize).expect("out"); + Self { ex, w, out } + } + + fn buf(&self, host: &[f32]) -> GpuBuffer { + GpuBuffer::from_host(self.ex.context(), host).expect("buffer") + } + + fn gemv(&mut self, variant: Q4kVariant, input: &GpuBuffer) -> Vec { + self.ex.gpu_profile.q4k = variant; + self.ex + .q4k_gemv_into(self.w, input, &self.out, N, K) + .expect("gemv"); + self.ex.synchronize().expect("sync"); + let mut host = vec![0.0f32; N as usize]; + self.out.copy_to_host(&mut host).expect("copy out"); + host + } + + /// The float GEMV of whatever `input` holds right now. + fn float_ref(&mut self, input: &GpuBuffer) -> Vec { + self.ex.synchronize().expect("sync"); + let mut host = vec![0.0f32; K as usize]; + input.copy_to_host(&mut host).expect("copy in"); + let fresh = self.buf(&host); + self.gemv(Q4kVariant::Mwv, &fresh) + } +} + +const DP4A: [Q4kVariant; 2] = [Q4kVariant::HwDp4a, Q4kVariant::MwvDp4a]; + +/// Two DP4A GEMVs on two DIFFERENT buffers: the second must quantize its own input. +#[test] +#[serial] +fn a_dp4a_gemv_on_a_different_buffer_quantizes_that_buffer() { + let ex = crate::cuda_executor_or_skip!(0); + let mut rig = Rig::new(ex); + let (a, b) = (rig.buf(&input_a()), rig.buf(&input_b())); + let (ref_a, ref_b) = (rig.float_ref(&a), rig.float_ref(&b)); + assert!(rel_l2(&ref_a, &ref_b) > DISTINCT_FLOOR, "inputs too alike"); + + for variant in DP4A { + let got_b = rig.gemv(variant, &b); + let got_a = rig.gemv(variant, &a); + let (err_b, err_a) = (rel_l2(&got_b, &ref_b), rel_l2(&got_a, &ref_a)); + eprintln!("[4258] {variant:?}: first input {err_b:.3e}, second input {err_a:.3e}"); + assert!( + err_b < FRESH_TOL, + "{variant:?}: the first DP4A GEMV is off ({err_b})" + ); + assert!( + err_a < FRESH_TOL, + "{variant:?}: a DP4A GEMV on a second buffer is {err_a} from the float GEMV of \ + that buffer — it reused the first buffer's Q8_1 activation (#4258)" + ); + } +} + +/// One buffer, rewritten in place by each Qwen3.5 decode primitive that feeds a GEMV: the +/// DP4A GEMV after the write must quantize the NEW contents. +#[test] +#[serial] +fn a_dp4a_gemv_after_a_qwen35_writer_rewrites_its_input_requantizes() { + let ex = crate::cuda_executor_or_skip!(0); + let mut rig = Rig::new(ex); + let (a_host, b_host) = (input_a(), input_b()); + let (a, b) = (rig.buf(&a_host), rig.buf(&b_host)); + let ones = rig.buf(&vec![1.0f32; K as usize]); + + type Writer = + fn(&mut CudaExecutor, &GpuBuffer, &GpuBuffer, &GpuBuffer, &GpuBuffer); + let writers: [(&str, Writer); 5] = [ + ("rmsnorm_into", |ex, _a, b, one, x| { + ex.rmsnorm_into(b, one, x, K, 1e-6).expect("rmsnorm_into"); + }), + ("per_head_rmsnorm_into", |ex, _a, b, one, x| { + // `one` is K ones; per-head gamma reads only its first 64. + ex.per_head_rmsnorm_into(b, one, x, 64, K / 64, 1e-6) + .expect("per_head_rmsnorm_into"); + }), + ("residual_add_into", |ex, a, b, _one, x| { + ex.residual_add_into(a, b, x, K).expect("residual_add_into"); + }), + ("fused_swiglu_into", |ex, a, b, _one, x| { + ex.fused_swiglu_into(b, a, x, K).expect("fused_swiglu_into"); + }), + ("gdn_sigmoid_gate_into", |ex, _a, b, _one, x| { + ex.gdn_sigmoid_gate_into(x, b, K) + .expect("gdn_sigmoid_gate_into"); + }), + ]; + + // Every `x` stays alive for the whole test: a freed buffer's address is handed + // straight back to the next allocation, and a host upload into it is not a write + // the executor sees (see `ensure_q8_activation`). + let xs: Vec> = (0..DP4A.len() * writers.len()) + .map(|_| rig.buf(&a_host)) + .collect(); + let mut xs = xs.iter(); + for variant in DP4A { + for (name, write) in writers { + let x = xs.next().expect("one x per case"); + let before = rig.gemv(variant, x); + write(&mut rig.ex, &a, &b, &ones, x); + let want = rig.float_ref(x); + assert!( + rel_l2(&before, &want) > DISTINCT_FLOOR, + "{name}: write changed too little" + ); + let got = rig.gemv(variant, x); + let err = rel_l2(&got, &want); + eprintln!("[4258] {variant:?} after {name}: {err:.3e}"); + assert!( + err < FRESH_TOL, + "{variant:?}: the DP4A GEMV after `{name}` rewrote its input is {err} from the \ + float GEMV of the new contents — it reused the stale Q8_1 activation (#4258)" + ); + } + } +} diff --git a/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs b/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs index 7d9985fbd1..57f6a06130 100644 --- a/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs +++ b/crates/aprender-serve/src/cuda/executor/tiled_gemv.rs @@ -278,13 +278,13 @@ impl CudaExecutor { // Load kernel module let kernel_type = KernelType::Q8Quantize { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q8_quantize_{}", n); + let cache_key = module_key!(self, "q8_quantize_{}", n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // Q8_1 format: 36 bytes per 32 values @@ -317,6 +317,38 @@ impl CudaExecutor { Ok(buf_output) } + /// #4258: quantize the first `n` elements of `input` into `q8_buf` unless the + /// Q8 activation cache already holds exactly that. The cache is keyed on the + /// source buffer, so a GEMV over a different buffer quantizes its own input + /// instead of reusing the last one (PMAT-027 keyed only on a bool). + /// + /// The executor only sees writes made by its own kernels, which invalidate + /// through [`Self::q8_activation_written`]. A host upload into a buffer, or a + /// freed buffer whose address is reallocated, is invisible to it: the caller + /// must clear `q8_activation_valid` before a DP4A GEMV reads such a buffer. + pub(crate) fn ensure_q8_activation( + &mut self, + input: &GpuBuffer, + q8_buf: &GpuBuffer, + n: u32, + ) -> Result<(), GpuError> { + let src = (input.as_ptr(), n); + if !self.q8_activation_valid || self.q8_activation_src != src { + self.q8_quantize_into(input, q8_buf, n)?; + self.q8_activation_valid = true; + self.q8_activation_src = src; + } + Ok(()) + } + + /// #4258: a kernel is about to write device buffer `dst`; if that buffer is + /// the one the Q8 activation cache was quantized from, the cache is stale. + pub(crate) fn q8_activation_written(&mut self, dst: u64) { + if dst == self.q8_activation_src.0 { + self.q8_activation_valid = false; + } + } + /// PAR-PERF-DP4A: Q8 quantize into PRE-ALLOCATED buffer (zero allocation) /// /// Five-Whys root cause (2026-02-09): @@ -334,13 +366,13 @@ impl CudaExecutor { ) -> Result<(), GpuError> { let kernel_type = KernelType::Q8Quantize { n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q8_quantize_{}", n); + let cache_key = module_key!(self, "q8_quantize_{}", n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let num_blocks = (n + 31) / 32; @@ -366,7 +398,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/cuda/executor/weight.rs b/crates/aprender-serve/src/cuda/executor/weight.rs index cc2366795b..5ef8c8a931 100644 --- a/crates/aprender-serve/src/cuda/executor/weight.rs +++ b/crates/aprender-serve/src/cuda/executor/weight.rs @@ -19,13 +19,13 @@ impl CudaExecutor { let kernel_type = KernelType::BatchedQ6KGemv { k, n, m }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("batched_q6k_gemv_{}_{}_{}", m, k, n); + let cache_key = module_key!(self, "batched_q6k_gemv_{}_{}_{}", m, k, n); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); // Grid: N blocks (one per output row), 32 threads per block @@ -103,14 +103,14 @@ impl CudaExecutor { // Original Q6K kernel (CoalescedQ6K disabled due to CORRECTNESS-006) let kernel_type = KernelType::Q6KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q6k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q6k_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -137,7 +137,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -175,13 +175,13 @@ impl CudaExecutor { let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::MwvQ6KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("mwv_q6k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "mwv_q6k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -212,7 +212,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -259,22 +259,19 @@ impl CudaExecutor { let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; // Step 1: Quantize activations to Q8_1 (skip if already valid — PMAT-027) - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; // Step 2: Launch DP4A Q6K GEMV kernel let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::Dp4aQ6KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("dp4a_q6k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "dp4a_q6k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -307,7 +304,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -349,21 +346,18 @@ impl CudaExecutor { // SAFETY: constructs a non-owning `GpuBuffer` view over an already-allocated device region (`ptr`, element count `len`) that stays live for the kernel call; the view is `leak()`ed afterwards so its Drop never frees the borrowed device allocation (no double-free). let q8_buf = unsafe { GpuBuffer::::from_raw_parts(q8_ptr, q8_len) }; - if !self.q8_activation_valid { - self.q8_quantize_into(input, &q8_buf, k)?; - self.q8_activation_valid = true; - } + self.ensure_q8_activation(input, &q8_buf, k)?; let num_warps = self.gpu_profile.mwv_warps; let kernel_type = KernelType::HwDp4aQ6KGemv { k, n, num_warps }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("hw_dp4a_q6k_gemv_{}_{}_{}", k, n, num_warps); + let cache_key = module_key!(self, "hw_dp4a_q6k_gemv_{}_{}_{}", k, n, num_warps); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let threads = num_warps * 32; @@ -394,7 +388,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -433,14 +427,14 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "coalesced_q6k_gemv_into")?; let kernel_type = KernelType::CoalescedQ6KGemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("coalesced_q6k_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "coalesced_q6k_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -495,14 +489,14 @@ impl CudaExecutor { // PAR-058: Zero allocation Q8_0 GEMV for mixed-quantization models let kernel_type = KernelType::Q8_0Gemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q8_0_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q8_0_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -529,7 +523,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -568,14 +562,14 @@ impl CudaExecutor { // PAR-058: Zero allocation Q5_0 GEMV for Qwen 0.5B Q/K weights let kernel_type = KernelType::Q5_0Gemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q5_0_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q5_0_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -602,7 +596,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -641,14 +635,14 @@ impl CudaExecutor { // PAR-058: Zero allocation Q4_0 GEMV for GGUF qtype mismatch let kernel_type = KernelType::Q4_0Gemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("q4_0_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "q4_0_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -675,7 +669,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), @@ -706,14 +700,14 @@ impl CudaExecutor { validate_device_ptr(weight_ptr, "f32_gemv_into")?; let kernel_type = KernelType::Gemv { k, n }; let kernel_name = self.kernels.kernel_name(&kernel_type); - let cache_key = format!("f32_gemv_{}_{}", k, n); + let cache_key = module_key!(self, "f32_gemv_{}_{}", k, n); let config = LaunchConfig::grid_2d(n, 1, 32, 1); self.ensure_kernel_module(&cache_key, &kernel_type)?; let module = self .modules - .get_mut(&cache_key) + .get_mut(&*cache_key) .expect("module just inserted"); let mut ptr_output = output.as_ptr(); @@ -740,7 +734,7 @@ impl CudaExecutor { // trueno#243: Record kernel for manual graph construction if self.graph_recording { - let module = self.modules.get_mut(&cache_key).expect("module exists"); + let module = self.modules.get_mut(&*cache_key).expect("module exists"); let func = module.get_function(kernel_name)?; self.graph_recorded_kernels.push(RecordedKernel { func: SendCUfunction(func), diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs index d9c722ae61..18438e7622 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -257,6 +257,10 @@ pub struct Qwen35CudaModel<'a> { out_normed: GpuBuffer, /// `[vocab_size]`, the logits — the ONE buffer a token's forward downloads. logits_buf: GpuBuffer, + /// `[hidden_dim]`, the token's residual stream (#4215): the embedding row is + /// copied in, every layer updates it in place. `None` only while a forward + /// has it out. + hidden_buf: Option>, dims: Qwen35CudaDims, /// Positions the device KV caches hold. max_seq_len: usize, @@ -557,21 +561,16 @@ impl<'a> Qwen35CudaModel<'a> { // PRODUCTION DEFAULT, not a test affordance (PMAT-3477 / #3090): this // architecture runs the FLOAT Q4_K/Q6_K GEMV kernels, never the DP4A - // ones `GpuProfile::detect` picks for a dense decode. The DP4A kernels - // quantize the ACTIVATION to int8, and Qwen3.5 feeds its projections - // straight into a recurrence, which compounds that error instead of - // absorbing it. Measured on the real 0.8B file: with the float variants - // pinned, a whole DeltaNet layer's output is 0.000 relative from a - // second float run and inside the layer budget against the CPU; with - // `HwDp4a` the DeltaNet-only path lands **1.656 relative** away, and the - // end-to-end argmax is garbage — a wrong token at position 0, not a - // rounding difference. The falsifier lives in the tests file - // (`qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence`). + // ones `GpuProfile::detect` picks for a dense decode. // - // Recovering the DP4A throughput for this architecture (a higher- - // precision activation quantization, or DP4A only on the layers that do - // not feed the recurrence) is the DP4A-through-recurrence ticket, - // 0.69.0. Until it lands, correctness is not optional here. + // The pin's original reason, "DP4A is catastrophic through the + // recurrence" (1.656 relative, a wrong argmax at position 0), was + // #4258: no qwen35 writer kernel cleared `q8_activation_valid`, so + // every DP4A GEMV reused the first Q8_1 activation. With that fixed, + // DP4A holds the parity contract on 0.8B + // (`qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence`). The pin + // stays until DP4A is re-measured on 2B/4B (#4030). Lifting it is a + // perf decision to make on that evidence, not a side effect of this fix. Self::pin_float_gemv(&mut executor.gpu_profile); let mut layers = Vec::with_capacity(model.layers.len()); @@ -627,6 +626,7 @@ impl<'a> Qwen35CudaModel<'a> { let attn_scratch = Self::build_attn_scratch(&executor, dims)?; let out_normed = Self::zeros(&executor, dims.hidden_dim as usize)?; let logits_buf = Self::zeros(&executor, dims.vocab_size as usize)?; + let hidden_buf = Some(Self::zeros(&executor, dims.hidden_dim as usize)?); let prefill_attention = prefill::default_prefill_attention(&executor, dims); // #3596: the model's OWN state serves only the single-layer handles // (`forward_attention_layer`, `upload_attention_kv`, …), never a generation — @@ -650,6 +650,7 @@ impl<'a> Qwen35CudaModel<'a> { lm_head, out_normed, logits_buf, + hidden_buf, dims, max_seq_len, prefill_rows: prefill::PREFILL_MAX_CHUNK_ROWS, @@ -1518,6 +1519,51 @@ impl<'a> Qwen35CudaModel<'a> { state: &mut Qwen35CudaState, position: usize, ) -> Result> { + self.forward_to_logits(token, state, position)?; + // The ONE sync of the whole token, in front of the ONE download. + self.executor + .sync_stream() + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + let mut logits = vec![0.0f32; self.dims.vocab_size as usize]; + self.logits_buf + .copy_to_host(&mut logits) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + + state.kv_len = state.kv_len.max(position + 1); + Ok(logits) + } + + /// The token temperature 0 picks after `token` at `position` (#4215): the + /// argmax runs on the device over `logits_buf`, so the token downloads its + /// 4-byte id instead of the whole logits vector. Ties go to the LOWEST + /// index, as [`crate::gguf::ops::argmax`] does. + /// + /// # Errors + /// As [`Self::forward_single`]. + pub fn forward_single_greedy( + &mut self, + token: u32, + state: &mut Qwen35CudaState, + position: usize, + ) -> Result { + self.forward_to_logits(token, state, position)?; + let next = self + .executor + .gpu_argmax(self.logits_buf.as_ptr(), self.dims.vocab_size) + .map_err(|e| gpu_err("qwen35_cuda_greedy", &e))?; + state.kv_len = state.kv_len.max(position + 1); + Ok(next) + } + + /// Enqueue one token at `position` through every layer, the output norm and + /// the `lm_head` into `logits_buf`. Nothing is synced or downloaded, and the + /// caller advances `state.kv_len` once it has read the result. + fn forward_to_logits( + &mut self, + token: u32, + state: &mut Qwen35CudaState, + position: usize, + ) -> Result<()> { let hidden_dim = self.dims.hidden_dim as usize; let embedding = self.model.base.token_embedding(); let start = (token as usize) * hidden_dim; @@ -1538,15 +1584,37 @@ impl<'a> Qwen35CudaModel<'a> { }); } - let dev = GpuBuffer::from_host( - self.executor.context(), + // #4215: the residual stream lives in one persistent buffer; taking it + // out of `self` lets the layers borrow `self` mutably beside it. + let mut dev = self + .hidden_buf + .take() + .ok_or_else(|| RealizarError::InvalidShape { + reason: "qwen35_cuda: the hidden buffer is already in use".to_string(), + })?; + let run = self.run_layers_and_head( + &mut dev, &embedding[start..start + hidden_dim], - ) - .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + state, + position, + ); + self.hidden_buf = Some(dev); + run + } + + fn run_layers_and_head( + &mut self, + dev: &mut GpuBuffer, + embedding_row: &[f32], + state: &mut Qwen35CudaState, + position: usize, + ) -> Result<()> { + dev.copy_from_host(embedding_row) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; for il in 0..self.layers.len() { match self.layers[il] { - CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, &dev)?, - CudaLayer::Attention(_) => self.attention_layer(state, il, &dev, position)?, + CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, dev)?, + CudaLayer::Attention(_) => self.attention_layer(state, il, dev, position)?, } } // The tail stays on the device (#3090 review). `hidden_to_logits` would @@ -1559,7 +1627,7 @@ impl<'a> Qwen35CudaModel<'a> { let d = self.dims; self.executor .rmsnorm_into( - &dev, + dev, &self.output_norm, &self.out_normed, d.hidden_dim, @@ -1576,18 +1644,7 @@ impl<'a> Qwen35CudaModel<'a> { self.lm_head.k, ) .map_err(|e| gpu_err("qwen35_cuda_lm_head", &e))?; - - // The ONE sync of the whole token, in front of the ONE download. - self.executor - .sync_stream() - .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; - let mut logits = vec![0.0f32; d.vocab_size as usize]; - self.logits_buf - .copy_to_host(&mut logits) - .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; - - state.kv_len = state.kv_len.max(position + 1); - Ok(logits) + Ok(()) } /// Run every Gated `DeltaNet` layer over a host hidden state and read the @@ -1637,3 +1694,8 @@ pub use prefill::{ #[cfg(test)] #[path = "forward_qwen35_cuda_tests.rs"] mod qwen35_cuda_tests; + +/// #4215: host allocations, device allocations and D2H bytes per decode token. +#[cfg(test)] +#[path = "forward_qwen35_decode_overhead_tests.rs"] +mod qwen35_decode_overhead_tests; diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs index 05d39e8c8a..f15eb70f89 100644 --- a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs @@ -1230,21 +1230,22 @@ fn qwen35_cuda_a_fresh_model_pins_the_float_gemv_variants() { ); } -/// WHY THE PIN EXISTS (PMAT-3477 / #3090): with the DP4A GEMV kernels armed, -/// the same forward that matches the CPU token for token produces a DIFFERENT -/// token, or a direction nowhere near the CPU's. +/// DP4A through the recurrence holds the parity contract (#4258). /// -/// This is a falsifier, not a bug reproduction: it asserts the failure, so if a -/// future DP4A activation-quantization change makes the path correct, this test -/// goes RED and the pin (and the DP4A-through-recurrence ticket, 0.69.0) can be -/// reconsidered on evidence. +/// This was the falsifier `qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence` +/// (PMAT-3477 / #3090), which asserted the opposite: 1.656 relative away and a wrong +/// argmax under `HwDp4a`. That reading was not Q8_1 quantization error. No qwen35 +/// writer kernel ever cleared `q8_activation_valid`, so every DP4A GEMV after the +/// first reused the FIRST token's Q8_1 activation (#4258). With the activation cache +/// keyed to its input and invalidated by the writers, the same run passes at every +/// position. So this now guards the fix: if the stale-activation reuse comes back, +/// the argmax breaks here. /// -/// Measured on the real 0.8B file at position 0: see the printed reading. The -/// DeltaNet-only path is 1.656 relative away from the CPU under `HwDp4a` versus -/// 0.000 float-vs-float; end to end the argmax is simply wrong. +/// The float pin in `Qwen35CudaModel::with_max_seq_len` stays until DP4A is +/// re-measured on 2B/4B (#4030). This test only says the pin's stated reason is gone. #[test] #[serial_test::serial] -fn qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence() { +fn qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence() { use crate::cuda::gpu_profile::{Q4kVariant, Q6kVariant}; let executor = qwen35_cuda_fixture_or_skip!(); let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); @@ -1289,11 +1290,12 @@ fn qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence() { } } - assert!( - broken > 0, - "the DP4A GEMV path passed the parity contract at every position — it is no longer \ - catastrophic through the recurrence, so re-measure and revisit the pin in \ - Qwen35CudaModel::with_max_seq_len (DP4A-through-recurrence ticket, 0.69.0)" + assert_eq!( + broken, + 0, + "the DP4A GEMV path broke the parity contract at {broken} of {} positions: a stale \ + Q8_1 activation is being reused again (#4258)", + LONG_PROMPT.len() ); } diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_decode_overhead_tests.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_decode_overhead_tests.rs new file mode 100644 index 0000000000..bfbb3d3f12 --- /dev/null +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_decode_overhead_tests.rs @@ -0,0 +1,232 @@ +//! #4215: the host cost of one steady-state Qwen3.5 greedy decode token. +//! +//! Decode is close to launch-bound (~634 launches/token), so the host work +//! around each launch is on the critical path. Before #4215 every token +//! allocated a device buffer for its embedding row, downloaded the whole +//! 248,320-entry logits vector (~1 MB) to run the argmax on the CPU, and +//! allocated on the host around every GDN launch (a `format!` per pointer, the +//! argument `Vec`s, a `format!` cache key per wrapper). +//! +//! These tests read three per-thread counters across steady-state tokens (after +//! warm-up has compiled every module and sized every lazy buffer): host heap +//! allocations (a counting global allocator), device allocations and +//! device→host bytes (`trueno_gpu::driver`'s `GpuBuffer` counters). + +use super::Qwen35CudaModel; +use crate::gguf::forward_qwen35::Qwen35Model; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use trueno_gpu::driver::{device_allocs_total, device_to_host_bytes_total}; + +const MODEL_PATH: &str = "/home/noah/models/Qwen3.5-0.8B-Q4_K_M.gguf"; + +/// Prompt ids (all well inside the 248,320-row vocabulary). +const PROMPT: [u32; 6] = [9707, 11, 1879, 0, 3555, 374]; + +/// Steady-state tokens measured, after the prompt and `WARMUP` decode tokens. +const MEASURED: u64 = 4; +const WARMUP: usize = 2; + +/// Counts every heap allocation the current thread makes; forwards to `System`. +struct CountingAlloc; + +thread_local! { + static HOST_ALLOCS: Cell = const { Cell::new(0) }; +} + +fn count_host_alloc() { + // `try_with`: an allocation during thread-local teardown is simply not counted. + let _ = HOST_ALLOCS.try_with(|c| c.set(c.get() + 1)); +} + +// SAFETY: every method forwards to `System` with the caller's arguments unchanged; +// the only addition is a thread-local counter increment, which never allocates. +unsafe impl GlobalAlloc for CountingAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + count_host_alloc(); + // SAFETY: forwarded verbatim; the caller upholds `GlobalAlloc::alloc`'s contract. + unsafe { System.alloc(layout) } + } + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + count_host_alloc(); + // SAFETY: forwarded verbatim. + unsafe { System.alloc_zeroed(layout) } + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + count_host_alloc(); + // SAFETY: forwarded verbatim. + unsafe { System.realloc(ptr, layout, new_size) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: forwarded verbatim. + unsafe { System.dealloc(ptr, layout) } + } +} + +#[global_allocator] +static COUNTING_ALLOC: CountingAlloc = CountingAlloc; + +fn host_allocs_total() -> u64 { + HOST_ALLOCS.with(Cell::get) +} + +/// One greedy decode step: the token temperature 0 picks after `token` at `position`. +fn greedy_step( + gpu: &mut Qwen35CudaModel<'_>, + token: u32, + state: &mut super::Qwen35CudaState, + position: usize, +) -> u32 { + gpu.forward_single_greedy(token, state, position) + .expect("greedy forward") +} + +macro_rules! model_or_skip { + () => {{ + if !std::path::Path::new(MODEL_PATH).exists() { + eprintln!("SKIP: {MODEL_PATH} is absent"); + return; + } + crate::cuda_executor_or_skip!(0) + }}; +} + +/// A steady-state greedy token makes no host or device allocation and downloads +/// exactly the 4-byte token id. +#[test] +#[serial_test::serial] +fn a_steady_state_greedy_decode_token_allocates_nothing_and_downloads_only_the_token_id() { + let executor = model_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = Qwen35Model::create_base_model(&mapped.model, mapped.data()).expect("base"); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let mut state = gpu.new_state().expect("device state"); + + let mut pos = 0; + let mut next = 0; + for &t in &PROMPT { + next = greedy_step(&mut gpu, t, &mut state, pos); + pos += 1; + } + for _ in 0..WARMUP { + next = greedy_step(&mut gpu, next, &mut state, pos); + pos += 1; + } + + // The #3759 key proof `format!`s every cache hit in test builds only; the warmup above + // ran it over every key this token uses. Measure the lookup a release build does. + gpu.executor.suspend_module_key_proof(true); + let (h0, g0, d0) = ( + host_allocs_total(), + device_allocs_total(), + device_to_host_bytes_total(), + ); + for _ in 0..MEASURED { + next = greedy_step(&mut gpu, next, &mut state, pos); + pos += 1; + } + let (h1, g1, d1) = ( + host_allocs_total(), + device_allocs_total(), + device_to_host_bytes_total(), + ); + gpu.executor.suspend_module_key_proof(false); + let (host, dev, d2h) = (h1 - h0, g1 - g0, d1 - d0); + eprintln!( + "[4215] per steady-state token: host allocs {:.1}, device allocs {:.1}, D2H bytes {:.1}", + host as f64 / MEASURED as f64, + dev as f64 / MEASURED as f64, + d2h as f64 / MEASURED as f64, + ); + assert_eq!( + dev, 0, + "device allocations across {MEASURED} steady-state tokens" + ); + assert_eq!( + d2h, + 4 * MEASURED, + "device→host bytes across {MEASURED} tokens: one u32 token id each" + ); + assert_eq!( + host, 0, + "host heap allocations across {MEASURED} steady-state tokens" + ); +} + +/// Temperature 0 on the device picks exactly the token the CPU argmax of the full +/// logits picks, token after token. +#[test] +#[serial_test::serial] +fn greedy_on_the_device_is_token_identical_to_the_cpu_argmax_of_the_logits() { + let executor = model_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = Qwen35Model::create_base_model(&mapped.model, mapped.data()).expect("base"); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let mut logits_state = gpu.new_state().expect("device state"); + let mut greedy_state = gpu.new_state().expect("device state"); + + let steps = 32; + let (mut via_logits, mut via_greedy) = (Vec::new(), Vec::new()); + let (mut a, mut b) = (PROMPT[0], PROMPT[0]); + for pos in 0..PROMPT.len() + steps { + let (ta, tb) = match PROMPT.get(pos) { + Some(&t) => (t, t), + None => (a, b), + }; + let logits = gpu + .forward_single(ta, &mut logits_state, pos) + .expect("logits forward"); + a = crate::gguf::ops::argmax(&logits); + b = greedy_step(&mut gpu, tb, &mut greedy_state, pos); + via_logits.push(a); + via_greedy.push(b); + } + eprintln!("[4215] greedy tokens: {via_greedy:?}"); + assert_eq!( + via_greedy, via_logits, + "device greedy diverged from the CPU argmax of the downloaded logits" + ); +} + +/// Decode tokens/s after an 850-token prompt (#4215's reported number). Timing, +/// not a gate: run in release with `--ignored`; `APR_4215_MODEL` picks the GGUF. +#[test] +#[ignore = "timing: run in release with --ignored"] +#[serial_test::serial] +fn bench_greedy_decode_tok_s_at_850() { + let path = std::env::var("APR_4215_MODEL").unwrap_or_else(|_| MODEL_PATH.to_string()); + if !std::path::Path::new(&path).exists() { + eprintln!("SKIP: {path} is absent"); + return; + } + let executor = crate::cuda_executor_or_skip!(0); + let mapped = crate::gguf::MappedGGUFModel::from_path(&path).expect("map the GGUF"); + let base = Qwen35Model::create_base_model(&mapped.model, mapped.data()).expect("base"); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let mut gpu = + Qwen35CudaModel::with_max_seq_len(&qwen, executor, 1024).expect("build the CUDA model"); + let mut state = gpu.new_state().expect("device state"); + + const PROMPT_LEN: usize = 850; + const DECODE: usize = 128; + let mut next = 0; + for pos in 0..PROMPT_LEN { + let t = PROMPT[pos % PROMPT.len()] + (pos as u32 % 97) * 13; + next = greedy_step(&mut gpu, t, &mut state, pos); + } + let t0 = std::time::Instant::now(); + for i in 0..DECODE { + next = greedy_step(&mut gpu, next, &mut state, PROMPT_LEN + i); + } + let secs = t0.elapsed().as_secs_f64(); + eprintln!( + "[4215] {path}: decode {:.1} tok/s at {PROMPT_LEN} ({DECODE} tokens, {:.3} ms/token)", + DECODE as f64 / secs, + secs * 1e3 / DECODE as f64 + ); +} diff --git a/docs/audits/impl-PMAT-4258-receipt.md b/docs/audits/impl-PMAT-4258-receipt.md new file mode 100644 index 0000000000..b83c030982 --- /dev/null +++ b/docs/audits/impl-PMAT-4258-receipt.md @@ -0,0 +1,33 @@ +# Implementation receipt — PMAT-4258 (#4258), carried onto batch PR #4322 + +## Provenance +The fix was written on `fix/q8-flag-qwen35-4258` (PR #4302). That round was AGREED 3/3 PASS on head 4482ab98b. The +PR was then parked by the emergency fold #4311 because of a conflict against release/0.69.3. Here it is cherry-picked +(`-x`) onto main-based `perf/0694-qwen35-persistent-embed`: + +- 263958fc3 ← 1f14aea95 (fix) +- 56c9f9112 ← e9d074b83 (test inputs) +- 4d4a7841c ← 4482ab98b (roadmap) + +The code commits applied cleanly. The roadmap conflict was resolved as the union of both sides. + +## Change carried +`ensure_q8_activation` keys the cached Q8_1 activation by `(ptr, len)`. The qwen35 writer kernels invalidate it: +`rmsnorm_into`, `per_head_rmsnorm_into`, `residual_add_into`, `fused_swiglu_into` and `gdn_sigmoid_gate_into`. + +## New on this branch (555a6774d) +`qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence` asserted DP4A breaks parity. It now fails, as its own +message intends. It becomes `qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence`, which asserts 0 broken +positions, so a return of the stale reuse is caught. The pin comment in `with_max_seq_len` now states the real cause. +The float pin itself is NOT changed: lifting it waits on a 2B/4B re-measure (#4030). + +## Gates (head 555a6774d) +- Build (`--release --features cuda --no-run`): rc=0. +- `clippy --lib --features cuda -D warnings`: rc=0. +- `fmt --check`: rc=0. +- Roadmap sorted/unique: PASS. +- GPU tests on RTX 4090 via gpu-q, with compute-apps empty at start (logs in `/mnt/nvme-raid0/tmp/q8-gpu/`): + - `qwen35_cuda*`: 17 passed, 0 failed. + - `tests_q8_activation_staleness`: 2 passed. Max |DP4A − float| is 3.119e-3 for the second buffer and 6.488e-4 + after `rmsnorm_into`. + - DP4A on 0.8B over 6 positions: argmax equal at every position, cosine 0.997423 to 0.999476. diff --git a/docs/audits/quorum-PMAT-4258-receipt.md b/docs/audits/quorum-PMAT-4258-receipt.md new file mode 100644 index 0000000000..7c5a55b5c9 --- /dev/null +++ b/docs/audits/quorum-PMAT-4258-receipt.md @@ -0,0 +1,25 @@ +# Quorum receipt — batch #4322, delta PMAT-4258 (#4258 stale-Q8 carry + DP4A parity guard) + +Author: claude-opus-5-5. Base: 86498a9db, the last head of this branch that a quorum or the GPU had judged; the +PMAT-4316 part is covered by `quorum-PMAT-4316-receipt.md`. Shape: sonnet-5 + 1 agy gemini-3.1-pro-high + haiku-4-5 +(operator 2026-09-24). + +Why a delta round: the full-batch brief was 52 KB. Everything below 86498a9db was already judged 3/3, and the carried +#4302 code was agreed 3/3 on 4482ab98b. This round judges the cherry-picks as landed here, plus the new 555a6774d. + +## Round 1 — AGREED 3/3 PASS on head 36f1ea707 +Every lane got the same brief: 33300 B, sha256 aa728804…, diff_sha256 7f8fff34f272…. + +| lane | model | verdict | +|------|-------|---------| +| agy | gemini-3.1-pro-high (measured) | PASS. Its 3 findings cite the acceptance criteria as met. Artifact: `quorum-PMAT-4258.json` | +| claude | claude-sonnet-5 | PASS. It traced every qwen35 GEMV input. The residual that the host upload writes feeds only rmsnorm/residual_add, never a GEMV, so the (ptr,len) key's host-upload limit does not apply. The dense float path never calls `ensure_q8_activation`. 1 finding: stale test name in `evidence/parity/thresholds.yaml:95` | +| claude | claude-haiku-4-5 | PASS, 0 findings. It checked that the receipt matches the GPU logs | + +## After the round +The one post-round commit touches only a comment: it annotates the stale test name in `evidence/parity/thresholds.yaml`, +as sonnet's finding asked. It changes no code or thresholds, and the YAML still parses. + +## GPU evidence +GREEN on 555a6774d: `qwen35_cuda*` passed 17/17 and the q8 staleness tests passed 2/2. See +`impl-PMAT-4258-receipt.md` §Gates. diff --git a/docs/audits/quorum-PMAT-4258.json b/docs/audits/quorum-PMAT-4258.json new file mode 100644 index 0000000000..d9e67315ff --- /dev/null +++ b/docs/audits/quorum-PMAT-4258.json @@ -0,0 +1,157 @@ +{ + "ticket": "PMAT-4258", + "base": "86498a9db", + "base_resolved": "86498a9db", + "base_note": "no origin/86498a9db exists; judged against the local ref", + "head": "36f1ea707d785d49f9bac0f989306b8f3ab0f457", + "diff_sha256": "7f8fff34f272251144f739a18252213231e23a294d9fc2c708ea3d8c2990faf6", + "width": 1, + "executor": "agy", + "prompt_mode": "inline", + "prompt_bytes": 33300, + "author": { + "model": "claude-opus-5-5", + "family": "claude", + "source": "flag" + }, + "agreed": true, + "lanes": [ + { + "lane": 1, + "status": "SUCCESS", + "verdict": "PASS", + "summary": "The diff correctly implements the required cache invalidation logic for Q8_1 activations. It keys the cached activation by `(ptr, len)` and invalidates it across all requested qwen35 writer kernels (`rmsnorm_into`, `per_head_rmsnorm_into`, `residual_add_into`, `fused_swiglu_into`, `gdn_sigmoid_gate_into`). The AC is fully met and accompanied by the requested tests, and the existing falsifier is properly updated to reflect that DP4A now holds parity through the recurrence.", + "findings": [ + { + "claim": "A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation", + "file": "crates/aprender-serve/src/cuda/executor/tiled_gemv.rs", + "grounding": "cited" + }, + { + "claim": "or a qwen35 writer kernel rewrote it", + "file": "crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs", + "grounding": "cited" + }, + { + "claim": "a red-first test covers both", + "file": "crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs", + "grounding": "cited" + } + ], + "raw_bytes": 3179, + "err_bytes": 0, + "envelope_status": "SUCCESS", + "verdict_source": "structured_output", + "grounding_check": "parity", + "model": "gemini-3.1-pro-high", + "model_measured": "gemini-3.1-pro-high", + "model_source": "measured", + "family": "gemini", + "role": "independent", + "fallback": { + "judged_by": "gemini-3.1-pro-high", + "exhausted": false, + "attempts": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "outcome": "answered" + } + ], + "skipped": [] + } + } + ], + "dissent": [], + "dedup": [ + { + "file": "crates/aprender-serve/src/cuda/executor/layer_norm_gpu.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "or a qwen35 writer kernel rewrote it" + ] + }, + { + "file": "crates/aprender-serve/src/cuda/executor/tests_q8_activation_staleness.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "a red-first test covers both" + ] + }, + { + "file": "crates/aprender-serve/src/cuda/executor/tiled_gemv.rs", + "line": null, + "lanes_agreeing": [ + 1 + ], + "claims": [ + "A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation" + ] + } + ], + "uncovered": [], + "coverage_source": "lanes", + "partial": false, + "partial_reasons": [], + "fallback": { + "same_family_width": 1, + "chain": [ + { + "model": "gemini-3.1-pro-high", + "family": "gemini", + "disposition": "configured" + }, + { + "model": "gpt-oss-120b-medium", + "family": "openai", + "disposition": "fallback" + }, + { + "model": "qwen3.5", + "family": "qwen", + "disposition": "not-run", + "why": "no quorum.local_lane in the config — the aprender lane has no model to load" + }, + { + "model": "claude-opus-4-6-thinking", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + }, + { + "model": "claude-sonnet-4-6", + "family": "claude", + "disposition": "width", + "why": "same family as the author: at most 1 lane, recorded role width, counted toward no floor (R-15a)" + } + ], + "precheck": [ + { + "family": "gemini", + "model": "gemini-3.1-pro-high", + "probe": 1, + "outcome": "live" + } + ], + "prah": { + "source": "install-receipt", + "path": "/home/noah/.claude/skills/paiml-implement/bin/prah" + } + }, + "auto_merge": { + "checked": false, + "was_armed": false, + "disarmed": false, + "note": "no --pr given: nothing to disarm" + }, + "lint": { + "ok": true, + "output": "receipt complete: kind=artifact lanes=1 author=claude-opus-5-5/claude" + } +} diff --git a/docs/roadmaps/entries/PMAT-4215.yaml b/docs/roadmaps/entries/PMAT-4215.yaml new file mode 100644 index 0000000000..3af47a7c7e --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4215.yaml @@ -0,0 +1,19 @@ +- id: PMAT-4215 + github_issue: 4215 + item_type: task + title: 'Qwen3.5 CUDA greedy decode: persistent embedding buffer, GPU argmax, no format!/Vec on the launch path' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T08:28:19Z + updated: 2026-09-24T20:45:00Z + spec: null + acceptance_criteria: + - 'A steady-state greedy decode token makes 0 host heap allocations, 0 device allocations and downloads exactly the 4-byte token id (counted, not asserted by reading).' + - 'Greedy on the device is token-identical to the CPU argmax of the downloaded logits at temperature 0.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Scope is #4215 (a)(b)(c) from /mnt/nvme-raid0/tmp/alt0693/REPORT.md §A. Measured +2% at 850 on 4B (78.3 -> 79.9 tok/s); the 0.58x gap to ollama is NOT closed by this slice.' diff --git a/docs/roadmaps/entries/PMAT-4258.yaml b/docs/roadmaps/entries/PMAT-4258.yaml new file mode 100644 index 0000000000..02fabb1eeb --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4258.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4258 + github_issue: 4258 + item_type: task + title: 'Qwen3.5 CUDA: q8_activation_valid is never cleared on the qwen35 path, so DP4A reuses a stale Q8_1 activation' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T10:25:13Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation or a qwen35 writer kernel (rmsnorm_into, per_head_rmsnorm_into, residual_add_into, fused_swiglu_into, gdn_sigmoid_gate_into) rewrote it; a red-first test covers both.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found in /mnt/nvme-raid0/tmp/alt0693/REPORT.md "Extra finding". The DP4A-catastrophic falsifier now passes parity at every position on 0.8B; re-measure the #4030 pin on 2B/4B.' diff --git a/docs/roadmaps/entries/PMAT-4259.yaml b/docs/roadmaps/entries/PMAT-4259.yaml new file mode 100644 index 0000000000..33f18399bc --- /dev/null +++ b/docs/roadmaps/entries/PMAT-4259.yaml @@ -0,0 +1,18 @@ +- id: PMAT-4259 + github_issue: 4259 + item_type: task + title: 'GPU argmax kernel keeps the later index on an exact tie; CPU argmax keeps the first — breaks greedy token identity once #4215(b) moves argmax to GPU' + status: inprogress + priority: medium + assigned_to: null + created: 2026-09-24T12:01:41Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'The GPU argmax returns the FIRST index of an exact maximum, as the CPU argmax does; a red-first tie-break falsifier covers ties within a thread, across threads, and across blocks.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Ported from c8ce988c3 (perf/4215, based on 0.69.3) onto main; prerequisite for #4215(b), greedy GPU argmax.' diff --git a/docs/roadmaps/roadmap.yaml b/docs/roadmaps/roadmap.yaml index d43abbef4c..cd5439ee7d 100644 --- a/docs/roadmaps/roadmap.yaml +++ b/docs/roadmaps/roadmap.yaml @@ -21626,6 +21626,25 @@ roadmap: labels: - kind:code notes: null +- id: PMAT-4215 + github_issue: 4215 + item_type: task + title: 'Qwen3.5 CUDA greedy decode: persistent embedding buffer, GPU argmax, no format!/Vec on the launch path' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T08:28:19Z + updated: 2026-09-24T20:45:00Z + spec: null + acceptance_criteria: + - 'A steady-state greedy decode token makes 0 host heap allocations, 0 device allocations and downloads exactly the 4-byte token id (counted, not asserted by reading).' + - 'Greedy on the device is token-identical to the CPU argmax of the downloaded logits at temperature 0.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Scope is #4215 (a)(b)(c) from /mnt/nvme-raid0/tmp/alt0693/REPORT.md §A. Measured +2% at 850 on 4B (78.3 -> 79.9 tok/s); the 0.58x gap to ollama is NOT closed by this slice.' - id: PMAT-4242 github_issue: 4242 item_type: task @@ -21660,6 +21679,42 @@ roadmap: labels: - kind:code notes: null +- id: PMAT-4258 + github_issue: 4258 + item_type: task + title: 'Qwen3.5 CUDA: q8_activation_valid is never cleared on the qwen35 path, so DP4A reuses a stale Q8_1 activation' + status: inprogress + priority: high + assigned_to: null + created: 2026-09-24T10:25:13Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'A DP4A GEMV requantizes its input whenever the input buffer differs from the cached Q8_1 activation or a qwen35 writer kernel (rmsnorm_into, per_head_rmsnorm_into, residual_add_into, fused_swiglu_into, gdn_sigmoid_gate_into) rewrote it; a red-first test covers both.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Found in /mnt/nvme-raid0/tmp/alt0693/REPORT.md "Extra finding". The DP4A-catastrophic falsifier now passes parity at every position on 0.8B; re-measure the #4030 pin on 2B/4B.' +- id: PMAT-4259 + github_issue: 4259 + item_type: task + title: 'GPU argmax kernel keeps the later index on an exact tie; CPU argmax keeps the first — breaks greedy token identity once #4215(b) moves argmax to GPU' + status: inprogress + priority: medium + assigned_to: null + created: 2026-09-24T12:01:41Z + updated: 2026-09-24T12:01:41Z + spec: null + acceptance_criteria: + - 'The GPU argmax returns the FIRST index of an exact maximum, as the CPU argmax does; a red-first tie-break falsifier covers ties within a thread, across threads, and across blocks.' + phases: [] + subtasks: [] + estimated_effort: null + labels: + - kind:code + notes: 'Ported from c8ce988c3 (perf/4215, based on 0.69.3) onto main; prerequisite for #4215(b), greedy GPU argmax.' - id: PMAT-4263 github_issue: 4263 item_type: task diff --git a/evidence/parity/thresholds.yaml b/evidence/parity/thresholds.yaml index b182d28a4a..57114f5912 100644 --- a/evidence/parity/thresholds.yaml +++ b/evidence/parity/thresholds.yaml @@ -96,7 +96,11 @@ models: # on lambda: cosine 0.466 down to -0.246; on gx10 (peer session, same # head, 16:03Z): +0.179 down to -0.765 with the GPU argmax stuck at one # token at every position (the model pins the float GEMV for exactly - # this reason; the test is the standing falsifier on both hosts) + # this reason; the test is the standing falsifier on both hosts). + # #4258: that reading was a stale Q8_1 activation, not DP4A error. The + # test is now `qwen35_cuda_dp4a_gemv_holds_parity_through_the_recurrence` + # (0.8B, lambda: cosine 0.997423..0.999476, argmax 6/6). The numbers above + # stay as a known-bad record of a real stale-activation defect. # margin: 0.998971 - 0.99 = 0.008971 under the lower good floor; 0.99 - 0.466 = 0.524 over the higher bad ceiling qwen3.5-0.8b: compute_class: gpu diff --git a/scripts/tests/guard_tree_test.sh b/scripts/tests/guard_tree_test.sh index 884942ec75..1574ff613a 100755 --- a/scripts/tests/guard_tree_test.sh +++ b/scripts/tests/guard_tree_test.sh @@ -708,8 +708,16 @@ done printf '#!/usr/bin/env bash\n# guard-tree: serial\n[ "${1:-}" = --help ] && { echo usage; exit 0; }\necho "start l_serial" >> "%s/events"\nsleep 2\necho "end l_serial" >> "%s/events"\nexit 0\n' \ "$lfix" "$lfix" >"$lfix/scripts/check_l_serial.sh" git -C "$lfix" add -A && git -C "$lfix" commit -q -m fixture -serial_alone() { # -> 0 iff no other event falls between "start l_serial" and "end l_serial" - awk '/^start l_serial$/{f=1; next} /^end l_serial$/{f=0; next} f{bad=1} END{exit bad}' "$lfix/events" +serial_alone() { # -> 0 iff no other guard is running at any point while l_serial runs + # An event inside the serial window is not the only overlap: a pool guard that + # started BEFORE l_serial and ended AFTER it leaves the window empty. Count the + # guards still open when l_serial starts (#4439: a|b|serial-start|serial-end|a|b + # read as "alone"). + awk '/^start l_serial$/{if (open > 0) bad=1; f=1; next} + /^end l_serial$/{f=0; next} + f{bad=1} + /^start /{open++} /^end /{open--} + END{exit bad}' "$lfix/events" } : > "$lfix/events" l_out="$(cd "$lfix" && GUARD_TREE_JOBS=8 bash scripts/guard_tree.sh 2>&1)"