diff --git a/crates/aprender-serve/src/cuda/gpu_profile.rs b/crates/aprender-serve/src/cuda/gpu_profile.rs index a79bb48c4a..304b0e6f8c 100644 --- a/crates/aprender-serve/src/cuda/gpu_profile.rs +++ b/crates/aprender-serve/src/cuda/gpu_profile.rs @@ -1124,38 +1124,59 @@ impl GpuProfile { /// argmax). FP16 → 4 of 4 pass. /// qwen2.5-1.5b (QKV bias, no QK-norm) → 4 of 4 pass under FP8. /// The defect is the FP8 GEMM feeding the per-head norm, not the norm; its - /// root cause is #3483. Until then a QK-norm model takes the SERIAL prefill - /// (per-token `forward_gpu_resident`, byte-correct in the same matrix, no - /// prefill weight cache — the FP16 HGEMM cache is 2× the FP8 one and put - /// Qwen3-8B + `apr qa`'s CPU twin over 24 GB: `CUDA_ERROR_OUT_OF_MEMORY` in - /// `prefill_all_layers_gpu`) with FP8 off, which is what GB10 (cc ≥ 120) - /// already does by default. `FP8_PREFILL=1` forces FP8 back for A/B testing, - /// and an explicit `BATCHED_PREFILL` keeps its own answer. Returns `true` - /// when it changed anything, so the caller can print why. - pub fn disable_fp8_for_qk_norm( + /// root cause is #3483. FP8 goes off for a QK-norm model; the prefill stays + /// on the path `select_prefill_path` chose, since FP16 passes (above). + /// + /// #3715: this used to move a QK-norm model to the SERIAL prefill as well, + /// because the unchunked FP16 batched prefill ran Qwen3-8B beside `apr qa`'s + /// CPU twin out of 24 GB (`CUDA_ERROR_OUT_OF_MEMORY` in + /// `prefill_all_layers_gpu`). Serial is one `forward_gpu_resident` per prompt + /// token: Qwen3-1.7B took 109 s for 7955 tokens and did not finish 26131 in + /// 600 s on the 4090 (GPU 99-100 %), so every long-context Qwen3 cell timed + /// out. The batched prefill is now chunked to a fixed score budget, which is + /// what bounded the OOM. `FP8_PREFILL=1` forces FP8 back for A/B testing. + /// Returns `true` when it changed anything, so the caller can print why. + pub fn disable_fp8_for_qk_norm(&mut self, has_qk_norm: bool, forced_fp8: Option<&str>) -> bool { + if !has_qk_norm || forced_fp8 == Some("1") || !(self.fp8_prefill || self.fp8_decode) { + return false; + } + self.fp8_prefill = false; + self.fp8_decode = false; + true + } + + /// #3715: a QK-norm model whose FP16 batched prefill does not fit on the device + /// keeps the serial prefill, as it did before. + /// + /// With FP8 off, the FP16 weight cache is warmed on either path, but only the + /// batched prefill also needs its workspace and score buffers. Qwen3-8B on the + /// 24 GB 4090 takes 4.7 GB of weights plus a 15.1 GB FP16 cache, which leaves + /// 0.1 GB. The batched workspace then fails to allocate and the dense session + /// falls back to the CPU (a 2k chat turn: 382 s). Serial prefills on the device + /// in the room that is left. `need_bytes` is weights + cache + KV + the prefill + /// reserve. An explicit `BATCHED_PREFILL` keeps its own answer. Returns `true` + /// when it moved the path. + pub fn serial_when_fp16_prefill_does_not_fit( &mut self, has_qk_norm: bool, - forced_fp8: Option<&str>, forced_batched: Option<&str>, + need_bytes: usize, + free_bytes: usize, ) -> bool { - if !has_qk_norm || forced_fp8 == Some("1") { + if !has_qk_norm + || self.fp8_prefill + || forced_batched.is_some() + || self.prefill_path.path != PrefillPath::Batched + || need_bytes <= free_bytes + { return false; } - let mut changed = false; - if self.fp8_prefill || self.fp8_decode { - self.fp8_prefill = false; - self.fp8_decode = false; - changed = true; - } - if forced_batched.is_none() && self.prefill_path.path == PrefillPath::Batched { - self.prefill_path = PrefillPathChoice { - path: PrefillPath::Serial, - reason: "qk-norm model (#3413/#3483)", - cc: self.cc, - }; - changed = true; - } - changed + self.prefill_path = PrefillPathChoice { + path: PrefillPath::Serial, + reason: "qk-norm model, FP16 prefill does not fit (#3715)", + cc: self.cc, + }; + true } } @@ -1180,15 +1201,15 @@ mod pmat3477_fp8_qk_norm_tests { } // The measured case: a QK-norm model on sm_89 loses FP8 (Qwen3-8B under FP8 - // batched prefill scored cosine −0.0972 vs CPU) AND takes the serial prefill - // (the FP16 batched cache OOMed 8B beside apr qa's CPU twin on 24 GB). + // batched prefill scored cosine −0.0972 vs CPU) and KEEPS the batched prefill: + // the serial one could not prefill 26131 tokens in 600 s (#3715). #[test] - fn qk_norm_model_gets_serial_prefill_without_fp8() { + fn qk_norm_model_loses_fp8_keeps_batched_prefill() { let mut p = profile(true); - assert!(p.disable_fp8_for_qk_norm(true, None, None)); + assert!(p.disable_fp8_for_qk_norm(true, None)); assert!(!p.fp8_prefill && !p.fp8_decode); - assert_eq!(p.prefill_path.path, PrefillPath::Serial); - assert_eq!(p.prefill_path.reason, "qk-norm model (#3413/#3483)"); + assert_eq!(p.prefill_path.path, PrefillPath::Batched); + assert_eq!(p.prefill_path.reason, "default"); } // A model without QK-norm (the qwen2.5 control: 4/4 prompts pass under @@ -1196,7 +1217,7 @@ mod pmat3477_fp8_qk_norm_tests { #[test] fn no_qk_norm_keeps_fp8_and_batched() { let mut p = profile(true); - assert!(!p.disable_fp8_for_qk_norm(false, None, None)); + assert!(!p.disable_fp8_for_qk_norm(false, None)); assert!(p.fp8_prefill && p.fp8_decode); assert_eq!(p.prefill_path.path, PrefillPath::Batched); } @@ -1205,19 +1226,19 @@ mod pmat3477_fp8_qk_norm_tests { #[test] fn forced_fp8_is_honoured() { let mut p = profile(true); - assert!(!p.disable_fp8_for_qk_norm(true, Some("1"), None)); + assert!(!p.disable_fp8_for_qk_norm(true, Some("1"))); assert!(p.fp8_prefill); assert_eq!(p.prefill_path.path, PrefillPath::Batched); } - // An explicit BATCHED_PREFILL keeps its own answer; FP8 still goes off. + // An explicit BATCHED_PREFILL=0 keeps its own answer; FP8 still goes off. #[test] - fn explicit_batched_prefill_is_kept() { + fn explicit_serial_prefill_is_kept() { let mut p = profile(true); - p.prefill_path = select_prefill_path(89, Some("1")); - assert!(p.disable_fp8_for_qk_norm(true, None, Some("1"))); + p.prefill_path = select_prefill_path(89, Some("0")); + assert!(p.disable_fp8_for_qk_norm(true, None)); assert!(!p.fp8_prefill); - assert_eq!(p.prefill_path.path, PrefillPath::Batched); + assert_eq!(p.prefill_path.path, PrefillPath::Serial); } // Already serial and FP8 already off (GB10 defaults, or FP8_PREFILL=0): no change. @@ -1225,6 +1246,45 @@ mod pmat3477_fp8_qk_norm_tests { fn already_serial_and_off_is_not_a_change() { let mut p = profile(false); p.prefill_path = select_prefill_path(121, None); - assert!(!p.disable_fp8_for_qk_norm(true, None, None)); + assert!(!p.disable_fp8_for_qk_norm(true, None)); + assert_eq!(p.prefill_path.path, PrefillPath::Serial); + } + + // #3715: Qwen3-8B on the 4090 (4.7 GB weights + 15.1 GB FP16 cache + 4.6 GB + // reserve > 23.4 GB free) keeps serial; Qwen3-1.7B (1.1 + 3.4 + 4.6) keeps batched. + #[test] + fn fp16_prefill_that_does_not_fit_keeps_serial() { + const GB: usize = 1_000_000_000; + let mut p = profile(false); + assert!(p.serial_when_fp16_prefill_does_not_fit( + true, + None, + 24_400_000_000, + 23_400_000_000 + )); + assert_eq!(p.prefill_path().path, PrefillPath::Serial); + + let mut p = profile(false); + assert!(!p.serial_when_fp16_prefill_does_not_fit(true, None, 9_100_000_000, 23 * GB)); + assert_eq!(p.prefill_path().path, PrefillPath::Batched); + } + + #[test] + fn fit_check_leaves_non_qk_norm_fp8_and_forced_paths_alone() { + let (need, free) = (30_000_000_000, 1_000_000_000); + // (has_qk_norm, fp8, forced): no QK norm, FP8 on, BATCHED_PREFILL=1. + for (qk, fp8, forced) in [ + (false, false, None), + (true, true, None), + (true, false, Some("1")), + ] { + let mut p = profile(fp8); + assert!(!p.serial_when_fp16_prefill_does_not_fit(qk, forced, need, free)); + assert_eq!( + p.prefill_path().path, + PrefillPath::Batched, + "{qk} {fp8} {forced:?}" + ); + } } } diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index 14f99f7b43..f14d73ad70 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -40,6 +40,7 @@ mod forward_qwen35_cuda; /// #3714: the Qwen3-MoE decoder resident on the GPU. mod forward_qwen3_moe_resident; mod generation; +mod resident; mod speculative; mod weights; @@ -67,6 +68,18 @@ use super::utils::verbose; // IMP-800: CUDA-Accelerated Model Wrapper // ============================================================================= +/// #3715: device bytes load makes resident, from [`OwnedQuantizedModelCuda::resident_estimate`]. +struct ResidentEstimate { + /// f32 K+V bytes of one position over every layer. + kv_per_pos: usize, + /// Quantized weights. + weights: usize, + /// FP16/FP8 prefill weight cache (0 when this profile warms none). + cache: usize, + /// GH-178 workspace reserve plus the chunked-prefill score budget. + reserve: usize, +} + /// Error from CUDA model initialization that preserves the unconsumed model. /// /// When `OwnedQuantizedModelCuda::new()` fails, the model is returned inside this error @@ -732,10 +745,9 @@ impl OwnedQuantizedModelCuda { if executor.gpu_profile.disable_fp8_for_qk_norm( model.config.constraints.has_qk_norm, std::env::var("FP8_PREFILL").ok().as_deref(), - std::env::var("BATCHED_PREFILL").ok().as_deref(), ) { eprintln!( - "[#3413] architecture '{}' uses per-head QK-norm: FP8 prefill off and serial prefill in use — its FP8 batched prefill fails CPU parity (#3483; FP8_PREFILL=1 / BATCHED_PREFILL=1 override)", + "[#3413] architecture '{}' uses per-head QK-norm: FP8 prefill off, FP16 batched prefill — its FP8 batched prefill fails CPU parity (#3483; FP8_PREFILL=1 overrides)", model.config.architecture ); } @@ -765,6 +777,24 @@ impl OwnedQuantizedModelCuda { let num_layers = model.layers.len(); let num_kv_heads = model.config.num_kv_heads; // PAR-021 GQA support let head_dim = model.config.hidden_dim / model.config.num_heads; + // #3715: the FP16 batched prefill a QK-norm model now takes must fit beside + // its cache; when it does not (Qwen3-8B on 24 GB) the model keeps serial. + { + let r = Self::resident_estimate(&executor, &model); + let need = r.weights + r.cache + r.reserve + r.kv_per_pos * max_seq_len; + if executor.gpu_profile.serial_when_fp16_prefill_does_not_fit( + model.config.constraints.has_qk_norm, + std::env::var("BATCHED_PREFILL").ok().as_deref(), + need, + memory_info.0, + ) { + eprintln!( + "[#3715] serial prefill: the FP16 batched prefill needs {:.1} GB and {:.1} GB is free (BATCHED_PREFILL=1 overrides)", + need as f64 / 1e9, + memory_info.0 as f64 / 1e9, + ); + } + } if let Err(error) = Self::configure_executor(&mut executor, &model, max_seq_len) { return Err(CudaInitError { @@ -801,6 +831,46 @@ impl OwnedQuantizedModelCuda { Ok(cuda_model) } + /// #3715: what load makes resident on the device: the quantized weights, the + /// FP16 (or FP8) prefill weight cache when this profile warms one, and the + /// GH-178 reserve plus the chunked-prefill score budget. Also the f32 K+V bytes + /// of one position over every layer. + fn resident_estimate( + executor: &crate::cuda::CudaExecutor, + model: &OwnedQuantizedModel, + ) -> ResidentEstimate { + let c = &model.config; + let head_dim = c.hidden_dim / c.num_heads.max(1); + let kv_per_pos = + 2 * c.num_kv_heads * head_dim * std::mem::size_of::() * model.layers.len(); + let (weights, elems) = resident::projection_sizes( + model + .layers + .iter() + .flat_map(resident::layer_projections) + .chain(std::iter::once(&model.lm_head_weight)), + ); + let profile = &executor.gpu_profile; + let cache_bytes_per_elem = if profile.fp8_prefill { + 1 + } else if std::env::var("HGEMM_PREFILL").as_deref() == Ok("0") + || (profile.cc >= 120 && std::env::var("FORCE_FP16_CACHE").as_deref() != Ok("1")) + { + 0 + } else { + 2 + }; + let cache = cache_bytes_per_elem * elems; + let reserve = + 3_500_000_000 + crate::gguf::cuda::forward_qwen35_cuda::PREFILL_SCORES_BUDGET_BYTES; + ResidentEstimate { + kv_per_pos, + weights, + cache, + reserve, + } + } + /// GH-129: Free CPU projection weight copies after GPU preload. /// /// On Jetson (unified memory), the CPU `Vec` copies are redundant after diff --git a/crates/aprender-serve/src/gguf/cuda/resident.rs b/crates/aprender-serve/src/gguf/cuda/resident.rs new file mode 100644 index 0000000000..57dbf3ee1b --- /dev/null +++ b/crates/aprender-serve/src/gguf/cuda/resident.rs @@ -0,0 +1,108 @@ +//! #3715: what a CUDA model's load makes resident on the device, so a decision taken +//! before the upload (the QK-norm batched-prefill fit check) can see whether the +//! FP16 prefill weight cache and its workspace will fit beside the weights. + +use crate::gguf::quantized::{OwnedQKVWeights, OwnedQuantizedLayer, OwnedQuantizedTensor}; + +/// The prefill weight cache holds the seven projection matrices of every layer +/// plus the LM head (the "197 matrices" a 28-layer model logs): the quantized +/// tensors load uploads. Returns `(quantized bytes, elements)` of those tensors; +/// the cache is `elements` × its bytes per element. Read from the tensors +/// themselves, so it cannot drift from the model the way a dims formula can. +pub(crate) fn projection_sizes<'a>( + tensors: impl IntoIterator, +) -> (usize, usize) { + tensors.into_iter().fold((0, 0), |(bytes, elems), t| { + (bytes + t.data.len(), elems + t.in_dim * t.out_dim) + }) +} + +/// The projection tensors of one layer, fused QKV or separate. +pub(crate) fn layer_projections(l: &OwnedQuantizedLayer) -> Vec<&OwnedQuantizedTensor> { + let mut v = match &l.qkv_weight { + OwnedQKVWeights::Fused(t) => vec![t], + OwnedQKVWeights::Separate { q, k, v } => vec![q, k, v], + }; + v.extend([&l.attn_output_weight, &l.ffn_up_weight, &l.ffn_down_weight]); + v.extend(l.ffn_gate_weight.as_ref()); + v +} + +#[cfg(test)] +mod tests { + use super::*; + + fn t(in_dim: usize, out_dim: usize, bytes: usize) -> OwnedQuantizedTensor { + OwnedQuantizedTensor { + data: vec![0; bytes], + in_dim, + out_dim, + qtype: 12, + } + } + + /// A Qwen3 layer: separate Q/K/V, gated FFN. `bytes` goes on every tensor. + fn layer(hidden: usize, kv: usize, inter: usize, bytes: usize) -> OwnedQuantizedLayer { + OwnedQuantizedLayer { + attn_norm_weight: Vec::new(), + attn_norm_bias: None, + qkv_weight: OwnedQKVWeights::Separate { + q: t(hidden, hidden, bytes), + k: t(hidden, kv, bytes), + v: t(hidden, kv, bytes), + }, + qkv_bias: None, + attn_output_weight: t(hidden, hidden, bytes), + attn_output_bias: None, + ffn_up_weight: t(hidden, inter, bytes), + ffn_up_bias: None, + ffn_down_weight: t(inter, hidden, bytes), + ffn_down_bias: None, + ffn_gate_weight: Some(t(hidden, inter, bytes)), + ffn_gate_bias: None, + ffn_norm_weight: None, + ffn_norm_bias: None, + attn_q_norm_weight: None, + attn_k_norm_weight: None, + post_attn_norm_weight: None, + post_ffw_norm_weight: None, + } + } + + /// FP16 cache bytes of a Qwen3 model with these dims, via the tensor walk. + fn fp16_cache(layers: usize, hidden: usize, kv: usize, inter: usize) -> usize { + let l = layer(hidden, kv, inter, 0); + let head = t(hidden, 151_936, 0); + let per_layer = projection_sizes(layer_projections(&l)).1; + 2 * (layers * per_layer + projection_sizes([&head]).1) + } + + #[test] + fn fp16_cache_estimate_matches_the_logged_qwen3_1_7b_cache() { + // [PMAT-037] FP16 weight cache: 197 matrices cached (3281.5 MB), measured on lambda. + let bytes = fp16_cache(28, 2048, 1024, 6144); + let logged = 3281.5 * 1024.0 * 1024.0; + assert!( + (bytes as f64 - logged).abs() / logged < 0.02, + "{bytes} vs {logged}" + ); + } + + #[test] + fn projection_walk_counts_every_tensor_once() { + // Seven projections, distinct dims so a dropped or doubled tensor shows. + let mut l = layer(2, 3, 5, 7); + assert_eq!(layer_projections(&l).len(), 7); + let (bytes, elems) = projection_sizes(layer_projections(&l)); + // q 2x2, k 2x3, v 2x3, o 2x2, up 2x5, down 5x2, gate 2x5 + assert_eq!((bytes, elems), (7 * 7, 4 + 6 + 6 + 4 + 10 + 10 + 10)); + // A fused QKV is one tensor; an ungated FFN has no gate. + l.qkv_weight = OwnedQKVWeights::Fused(t(2, 8, 7)); + l.ffn_gate_weight = None; + assert_eq!(layer_projections(&l).len(), 4); + assert_eq!( + projection_sizes(layer_projections(&l)), + (4 * 7, 16 + 4 + 10 + 10) + ); + } +} diff --git a/docs/audits/quorum-PMAT-4621.json b/docs/audits/quorum-PMAT-4621.json index 7804fd1cbe..0b1a17f8b8 100644 --- a/docs/audits/quorum-PMAT-4621.json +++ b/docs/audits/quorum-PMAT-4621.json @@ -3,13 +3,13 @@ "base": "origin/main", "base_resolved": "origin/main", "base_note": "no origin/origin/main exists; judged against the local ref", - "head": "6562916bee0984a4f70ed978a68d9a9f306de7f4", - "diff_sha256": "fcafa65c3b9eda89ed1049c61651156561f1c38fadb13e30a946930446d0ff46", + "head": "1c97bea00e1f42564a6c1eeadb3a254c6ecd8e2f", + "diff_sha256": "9c5b6d948e06b6a1d9f52a2d7f8c32ccc58f1c99155bc421ff378e7056b58441", "width": 3, "executor": "agy", "prompt_mode": "inline", - "prompt_bytes": 54780, - "prompt_sha256": "d4c7d13afef755bcef254ebbef70d2c5c0c3a0c9d9b2171d5a2a91d33b3607dd", + "prompt_bytes": 83612, + "prompt_sha256": "cbc00dae6bcc1ff7306a92448ba0959a21aec6712096b5dec824a756f9d34871", "author": { "model": "claude-opus-5-5", "family": "claude", @@ -21,24 +21,24 @@ "lane": 1, "status": "SUCCESS", "verdict": "PASS", - "summary": "The diff correctly implements 2 of the 4 sub-problems named in PMAT-4621's title: it adds F16/IQ4_XS CUDA dequant kernels (verified variants exist in WeightQuantType and are wired into qwen35_dequant_f32) and chunks the dense prefill (generate_2.rs) to keep the attention-score buffer within a fixed byte budget, with a matching fix (graph_may_run) so the opt-in CUDA-graph replay path can't be taken by a non-initial chunk. Traced the last_row/hidden_buf2 indexing and the chunk-loop's KV-append semantics (contract_pre_prefill_incremental!) and found them internally consistent. qk-norm forced-serial and chat-KV-2048 (the other two named sub-problems) are untouched here, but git log/commit messages (\"#4620 carries roots 1+5 only\") and the ticket's own 50%/InProgress status show this is a disclosed, intentional split rather than an omission.", + "summary": "Diff adds F16/IQ4_XS CUDA dequant kernels for Qwen3.5 GDN prefill, chunks the general dense prefill (generate_2.rs) to bound the attention-score buffer and avoid u32 overflow/OOM, gates CUDA-graph replay to first-chunk-only to prevent KV corruption on later chunks, and reworks qk-norm handling so FP8 is disabled but the batched prefill is kept unless a resident-memory estimate says the FP16 cache+workspace won't fit. Verified the last_row/chunk-boundary arithmetic algebraically (matches for both exact-multiple and remainder cases), confirmed WeightQuantType::F16/IQ4XS pre-exist so the new match arms compile, and confirmed dense_prefill_chunk_rows (generate_2.rs) and attention_rows_for (forward_qwen35_cuda_prefill.rs, unchanged) are genuinely separate call sites (general dense path vs Qwen3.5 GDN path) so the near-duplication is cosmetic, not a double-chunk hazard. The ticket's other two sub-issues (qk-norm forced serial removal is addressed; \"chat KV 2048\" is not) are untouched here, but recent commit history in this worktree shows this is a disclosed, intentional stacked-PR split (#4620 split into root pieces), consistent with the ticket's own 50%/InProgress status — not scope creep or a silent omission.", "findings": [ { - "file": "docs/roadmaps/PMAT-4621 ticket text", - "line": 0, - "claim": "Ticket PMAT-4621 names four sub-problems (unchunked dense prefill OOM, qk-norm forced serial, chat KV 2048, Qwen3.5 no F16/IQ4_XS dequant); this diff addresses only the first and last.", + "file": "crates/aprender-serve/src/gguf/cuda/generate_2.rs", + "line": 1143, + "claim": "dense_prefill_chunk_rows sizes chunks using raw num_heads rather than the GQA-aware heads_per_kv the pre-existing, unchanged attention_rows_for (forward_qwen35_cuda_prefill.rs:209-212) uses for the same kind of budget calc. Confirmed these are different call sites (general dense prefill vs Qwen3.5 GDN prefill), so it is not a double-chunking correctness bug — only a more-conservative (smaller than necessary under GQA), non-blocking near-duplication.", "grounding": "cited", - "fix": "Not a defect: commit 23f058b6ba ('split(#4621): #4620 carries roots 1+5 only (chunked dense prefill, F16/IQ4_XS device dequant)') and the ticket's own 50%/InProgress status document this as a deliberate stacked-PR split, not scope creep or neglect. No action needed unless the remaining two items lack a tracked follow-up." + "fix": "Non-blocking; optionally note in the PR why the existing helper wasn't reused, or factor a shared GQA-aware sizing helper in a follow-up." }, { - "file": "crates/aprender-serve/src/gguf/cuda/generate_2.rs", - "line": 1146, - "claim": "dense_prefill_chunk_rows multiplies by raw num_heads rather than the GQA-aware heads_per_kv used by the pre-existing attention_rows_for/chunk_rows_for helper in forward_qwen35_cuda_prefill.rs, making the new helper a semantic near-duplicate that is more conservative (smaller chunks) under GQA but not incorrect.", - "grounding": "cited", - "fix": "Already flagged non-blocking in the diff's own embedded SARIF (DENSE-PREFILL-ROWS-DUPLICATION); worth a one-line author response on why the existing helper wasn't reused, but does not affect correctness." + "file": "docs/roadmaps (ticket PMAT-4621)", + "line": 0, + "claim": "Ticket names four sub-problems; this diff addresses unchunked dense prefill OOM, qk-norm forced serial, and Qwen3.5 F16/IQ4_XS dequant, but not 'chat KV 2048'.", + "grounding": "measured", + "fix": "Not a defect given ticket is 50%/InProgress and worktree git log shows an active PR-split cascade (#4620 roots); confirm a follow-up ticket/PR tracks the chat-KV-2048 piece before closing PMAT-4621." } ], - "raw_bytes": 5113, + "raw_bytes": 5728, "err_bytes": 0, "envelope_status": "SUCCESS", "verdict_source": "structured_output", @@ -49,10 +49,10 @@ "model_measured": "claude-sonnet-5", "model_source": "flag", "family": "claude", - "brief_sha256": "d4c7d13afef755bcef254ebbef70d2c5c0c3a0c9d9b2171d5a2a91d33b3607dd", + "brief_sha256": "cbc00dae6bcc1ff7306a92448ba0959a21aec6712096b5dec824a756f9d34871", "trace": { - "input_sha256": "7be5bda6ca3fc406ff03a48cc2ef3b6c6313bde8be082ab5e9d45f047b098a0d", - "output_sha256": "f691401be00b1d37ea82ed59e074c892980cfc750ea7eddef64d3dfe5518462c", + "input_sha256": "d945f7e726ca593af4d990bc7e66da0965388d3b22229adf9d413a109a0822ab", + "output_sha256": "be960ff18b1d91064125062cb42eddd0c9baa216dd6b9bf86a458ae0a279d699", "store": null, "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" }, @@ -73,10 +73,10 @@ "lane": 2, "status": "SUCCESS", "verdict": "PASS", - "summary": "Reviewed the diff against PMAT-4621. Verified that the chunked dense prefill accurately bounds the score buffer size to avoid OOM, and that CUDA graph replay is safely disabled for subsequent prefill chunks to prevent KV resets. The added F16/IQ4_XS PTX dequantization kernels correctly load and transform weights, matching the CPU reader logic according to the bitwise parity test. As the ticket is 50% in progress due to a PR split, the omission of QK-norm serial and chat KV 2048 fixes is expected and doesn't refute the changes present. The receipt claims align exactly with the diff.", + "summary": "Reviewed the diff against PMAT-4621 and found it sound. The chunked dense prefill logic correctly restricts CUDA graph replay to the initial chunk (via `graph_may_run`), preventing KV cache corruption, and safely bounds the attention score buffer allocation using `dense_prefill_chunk_rows`. The newly added F16 and IQ4_XS PTX dequantization kernels correctly process weights, which is backed by bitwise parity tests against the CPU reader. The diff also successfully resolves the 'qk-norm forced serial' issue by replacing the unconditional fallback with a dynamic VRAM capacity check (`serial_when_fp16_prefill_does_not_fit`). The `DEGRADED` status in the receipt correctly reflects a known environmental constraint (mutation test blocked by lack of GPU on the test host) rather than a code defect. I found no evidence that refutes the diff's correctness or violates the ticket's constraints.", "findings": [], - "raw_bytes": 4044, - "err_bytes": 103, + "raw_bytes": 3050, + "err_bytes": 0, "envelope_status": "SUCCESS", "verdict_source": "structured_output", "grounding_check": "parity", @@ -86,10 +86,10 @@ "model_source": "measured", "family": "gemini", "executor": "agy", - "brief_sha256": "d4c7d13afef755bcef254ebbef70d2c5c0c3a0c9d9b2171d5a2a91d33b3607dd", + "brief_sha256": "cbc00dae6bcc1ff7306a92448ba0959a21aec6712096b5dec824a756f9d34871", "trace": { - "input_sha256": "85113119bdd9e825dc09bee236b8d30865e83bf401de939754d43fbe63a4fd05", - "output_sha256": "7a7065ac112ca13c0447fd131f8de06d0fdb34e28cc39725741e70b03d4a8800", + "input_sha256": "500cfcb3eb7469574885844995854ee3e6195cc8c02df020ed97e2ab5d578a9b", + "output_sha256": "172934d7f5bc928256f7aefa54d258ad46d61377f822dadcb4da806a81492516", "store": null, "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" }, @@ -110,9 +110,17 @@ "lane": 3, "status": "SUCCESS", "verdict": "PASS", - "summary": "Diff implements all three core fixes for PMAT-4621: F16/IQ4_XS dequant kernels with device parity tests, chunked dense prefill with correct memory budgeting and position tracking, and graph gating to prevent KV corruption on multi-chunk prefills. All kernel integrations are correct, formula validation is sound, and unit tests cover the logic paths. No errors found.", - "findings": [], - "raw_bytes": 1641, + "summary": "Diff correctly implements F16/IQ4_XS dequant kernels with device parity tests, chunked dense prefill with per-row memory budgeting, and CUDA graph replay gating to prevent KV corruption on multi-chunk prefills. All kernel integrations are sound, loop logic is correct, and unit tests validate behavior. Addresses 2–3 of 4 ticket sub-problems per the 50% InProgress status; scope matches documented PR split.", + "findings": [ + { + "file": "crates/aprender-serve/src/gguf/cuda/generate_2.rs", + "line": 1146, + "claim": "dense_prefill_chunk_rows() duplicates the existing chunk_rows_for() helper's logic (both budget attention scores to PREFILL_SCORES_BUDGET_BYTES), but uses raw num_heads instead of GQA-grouped heads_per_kv, making it more conservative (smaller chunks).", + "grounding": "cited", + "fix": "Advisory, not a defect: the new helper is not incorrect and the trade-off (conservatism under GQA) is safe. Worth a one-line author note on whether reuse was considered." + } + ], + "raw_bytes": 2845, "err_bytes": 0, "envelope_status": "SUCCESS", "verdict_source": "structured_output", @@ -123,10 +131,10 @@ "model_measured": "claude-haiku-4-5", "model_source": "flag", "family": "claude", - "brief_sha256": "d4c7d13afef755bcef254ebbef70d2c5c0c3a0c9d9b2171d5a2a91d33b3607dd", + "brief_sha256": "cbc00dae6bcc1ff7306a92448ba0959a21aec6712096b5dec824a756f9d34871", "trace": { - "input_sha256": "50b3f091290b5df14488f009e572f00a4e9c8361cabcc5344d771be565a6bff8", - "output_sha256": "bbb59399e4419e7c3c0ee14080b203e5e4246e804f7e0a0f0252085b9f47a581", + "input_sha256": "ec7905583edb77424dba24457e3c230a8f347cb94e10700fcb66d4ea9b66a819", + "output_sha256": "39079e7c883a1bfcc7979a4772d9d38d7b4df19b18c8a5fc02039a5f9c2495d0", "store": null, "store_why": "almacen not provisioned yet (infra-27): the blobs stay in the round's gitignored .lanes dir" }, @@ -148,22 +156,32 @@ "dedup": [ { "file": "crates/aprender-serve/src/gguf/cuda/generate_2.rs", - "line": 1146, + "line": 1143, "lanes_agreeing": [ 1 ], "claims": [ - "dense_prefill_chunk_rows multiplies by raw num_heads rather than the GQA-aware heads_per_kv used by the pre-existing attention_rows_for/chunk_rows_for helper in forward_qwen35_cuda_prefill.rs, making the new helper a semantic near-duplicate that is more conservative (smaller chunks) under GQA but not incorrect." + "dense_prefill_chunk_rows sizes chunks using raw num_heads rather than the GQA-aware heads_per_kv the pre-existing, unchanged attention_rows_for (forward_qwen35_cuda_prefill.rs:209-212) uses for the same kind of budget calc. Confirmed these are different call sites (general dense prefill vs Qwen3.5 GDN prefill), so it is not a double-chunking correctness bug — only a more-conservative (smaller than necessary under GQA), non-blocking near-duplication." + ] + }, + { + "file": "crates/aprender-serve/src/gguf/cuda/generate_2.rs", + "line": 1146, + "lanes_agreeing": [ + 3 + ], + "claims": [ + "dense_prefill_chunk_rows() duplicates the existing chunk_rows_for() helper's logic (both budget attention scores to PREFILL_SCORES_BUDGET_BYTES), but uses raw num_heads instead of GQA-grouped heads_per_kv, making it more conservative (smaller chunks)." ] }, { - "file": "docs/roadmaps/PMAT-4621 ticket text", + "file": "docs/roadmaps (ticket PMAT-4621)", "line": 0, "lanes_agreeing": [ 1 ], "claims": [ - "Ticket PMAT-4621 names four sub-problems (unchunked dense prefill OOM, qk-norm forced serial, chat KV 2048, Qwen3.5 no F16/IQ4_XS dequant); this diff addresses only the first and last." + "Ticket names four sub-problems; this diff addresses unchunked dense prefill OOM, qk-norm forced serial, and Qwen3.5 F16/IQ4_XS dequant, but not 'chat KV 2048'." ] } ], @@ -226,9 +244,12 @@ "crates/aprender-gpu/src/kernels/quantize/mod.rs", "crates/aprender-serve/src/cuda/executor/gdn_prefill_ops.rs", "crates/aprender-serve/src/cuda/executor/layers/prefill.rs", - "crates/aprender-serve/src/gguf/cuda/generate_2.rs" + "crates/aprender-serve/src/cuda/gpu_profile.rs", + "crates/aprender-serve/src/gguf/cuda/generate_2.rs", + "crates/aprender-serve/src/gguf/cuda/mod.rs", + "crates/aprender-serve/src/gguf/cuda/resident.rs" ], - "paths": 9, + "paths": 12, "patterns": [ "^\\.github/workflows/", "(^|/)release[^/]*\\.(ya?ml|sh|rs|toml)$", @@ -248,10 +269,10 @@ "bucket": "gemini", "budget": 86, "basis": "[U] default 30, raised to the 86 calls a window has made without a 429; no window reached one", - "window_start": 1790642648, - "window_calls": 18, + "window_start": 1790661814, + "window_calls": 2, "window_429": 0, - "hour_calls": 2, + "hour_calls": 4, "share": 17, "want": 2, "state": "open", @@ -269,7 +290,7 @@ }, "cheap_seat": "claude-haiku-4-5", "canary": { - "skipped": "brief 54780 bytes > 24576" + "skipped": "brief 83612 bytes > 24576" }, "advisory_lane": { "state": "unavailable", @@ -290,16 +311,16 @@ "model_sha256": null, "rc": 2, "wall_s": 0, - "collected_s": 1532, + "collected_s": 294, "budget_s": 170, "budget_basis": "ledger: 2 x p95 85 s over 86 gx10-cuda Verdict rows", "brief": { - "bytes": 54780, - "sent_bytes": 54949, + "bytes": 83612, + "sent_bytes": 83781, "max_bytes": 196608 }, "trace": { - "input_sha256": "f14f02d25987fe655476ca3b0150d5d296af641a9bb2fdc92abd8fd516079a0c", + "input_sha256": "6b52b2f8e5534d44b19754f287ae394560b6da89b2dc9588e2ed5b5f642f00b8", "raw": "advisory.json" }, "attempts": [ @@ -308,10 +329,10 @@ "ok": false, "wall_s": 0, "error": "busy: executor yielded (unit inactive; last yield: 2026-09-28T12:47:02Z host=gx10-a5b5 event=stopped gpu-priority lock=/tmp/apr-gpu.priority)", - "ts": "2026-09-29T04:18:44.215Z", - "wall_ms": 357, - "request_id": "01a0eb62-ed75-794b-b226-6d0df11d2ca5", - "load1": 1.34, + "ts": "2026-09-29T06:07:56.252Z", + "wall_ms": 384, + "request_id": "01a0ebc6-e759-724f-ae55-a85775b77baa", + "load1": 7.75, "cold": null } ],