diff --git a/crates/apr-cli/src/commands/chat_generate_session_02.rs b/crates/apr-cli/src/commands/chat_generate_session_02.rs index 34045a5a0f..70d4fba7b4 100644 --- a/crates/apr-cli/src/commands/chat_generate_session_02.rs +++ b/crates/apr-cli/src/commands/chat_generate_session_02.rs @@ -283,9 +283,14 @@ impl ChatSession { Ok(decoded) } - /// CPU generation for one GGUF chat turn: the Qwen3.5 hybrid forward (#3091), which the - /// dense `OwnedQuantizedModel` loader refuses, or the dense model. Returns the prompt - /// followed by the generated tokens. + /// Generation for one GGUF chat turn that the cached dense CUDA model did not serve: + /// the Qwen3.5 hybrid forward, which the dense `OwnedQuantizedModel` loader refuses, + /// or the dense CPU model. Returns the prompt followed by the generated tokens. + /// + /// #3477: the hybrid branch goes through the same serve entry point `apr run` uses + /// (`run_qwen35_generate_dispatch`), so `apr chat` without `--cpu` gets the GPU + /// forward (#3090) and its fallbacks are printed by the runtime, not re-implemented + /// here. `force_cpu` is the chat spelling of `--no-gpu`. fn generate_gguf_cpu_tokens( mapped: &realizar::gguf::MappedGGUFModel, prompt_tokens: &[u32], @@ -294,18 +299,22 @@ impl ChatSession { ) -> Result, String> { use realizar::gguf::OwnedQuantizedModel; - if mapped.model.architecture() == Some("qwen35") { + if realizar::gguf::hybrid_forward_handles( + mapped.model.architecture().unwrap_or_default(), + ) { let base = realizar::gguf::forward_qwen35::Qwen35Model::create_base_model( &mapped.model, mapped.data(), ) .map_err(|e| format!("Failed to load the Qwen3.5 base model: {e}"))?; - return realizar::gguf::forward_qwen35::run_qwen35_generate( + return realizar::gguf::forward_qwen35::run_qwen35_generate_dispatch( mapped, &base, prompt_tokens, gen_config, + config.force_cpu, ) + .map(|(tokens, _used_gpu)| tokens) .map_err(|e| format!("Qwen3.5 generate failed: {e}")); } diff --git a/crates/apr-cli/src/commands/output_verification.rs b/crates/apr-cli/src/commands/output_verification.rs index 641bfad9da..fe5fe17a0c 100644 --- a/crates/apr-cli/src/commands/output_verification.rs +++ b/crates/apr-cli/src/commands/output_verification.rs @@ -1,4 +1,3 @@ - /// Gate 1: Metadata Plausibility Validation (Bug 210, GH-222) /// /// Validates that model hyperparameters (rope_theta, max_position_embeddings, rms_norm_eps) @@ -85,7 +84,11 @@ fn run_metadata_plausibility_gate(path: &Path, config: &QaConfig) -> Result) -> (f64, f64, &'static str) { match arch { - Some("qwen2" | "qwen2.5" | "qwen") => (100_000.0, f64::MAX, "expected ~1000000.0 (100x too low, will produce garbage)"), + Some("qwen2" | "qwen2.5" | "qwen") => ( + 100_000.0, + f64::MAX, + "expected ~1000000.0 (100x too low, will produce garbage)", + ), Some("llama" | "llama2" | "llama3") => (1000.0, 10_000_000.0, "expected 10000-500000"), _ => (100.0, 100_000_000.0, "outside plausible range [100, 100M]"), } @@ -112,7 +115,10 @@ fn check_rope_theta( if theta_f64 >= min && theta_f64 <= max { *checks_passed += 1; } else { - violations.push(format!("rope_theta={theta} for {} — {msg}", arch.unwrap_or("unknown"))); + violations.push(format!( + "rope_theta={theta} for {} — {msg}", + arch.unwrap_or("unknown") + )); } } @@ -477,16 +483,18 @@ fn validate_gpu_golden_output( config: &QaConfig, ) -> Result> { use realizar::gguf::{OwnedQuantizedModel, OwnedQuantizedModelCuda}; - // #3432 / #3477: Qwen3.5 (Gated DeltaNet) runs on the CPU only — the runtime - // (`realizar::infer::run_inference`) routes `qwen35` around the dense loader, - // and the Capability Match gate reports the GPU capability as declined - // (#3090). Loading it through `OwnedQuantizedModel::from_mapped` here turned a - // declined GPU rung into a hard `Validation failed` that aborted every later - // gate. A declined backend is a skip with a reason, not an abort. - if mapped.model.architecture() == Some("qwen35") { + // #3432 / #3477: the Qwen3.5 hybrid never reaches this dense-loader gate — + // `golden_gate_for` routes it to `run_golden_output_gate_runtime`, which goes + // through `realizar::run_inference` and therefore through the hybrid's own + // GPU forward (#3090). Loading it through `OwnedQuantizedModel::from_mapped` + // here would turn that into a hard `Validation failed` aborting every later + // gate, so the guard stays as a fail-safe and says where the GPU output IS + // judged rather than claiming it is not. + if realizar::gguf::hybrid_forward_handles(mapped.model.architecture().unwrap_or_default()) { note_gpu_golden_skip( config, - "GPU golden output skipped: Gated DeltaNet runs on the CPU only (#3090)", + "GPU golden output for the Gated DeltaNet hybrid is judged by the runtime rung \ + (run_inference → Qwen35CudaModel, #3090), not by the dense loader", ); return Ok(None); } @@ -525,7 +533,7 @@ fn note_gpu_golden_skip(config: &QaConfig, message: &str) { } /// #3477: golden generation for a GGUF the dense loader refuses but the runtime -/// serves on the CPU (Qwen3.5 Gated DeltaNet, #3091). +/// serves (Qwen3.5 Gated DeltaNet — CPU #3091, GPU #3090). /// /// `golden_output_gguf_cpu` builds the model with /// `OwnedQuantizedModel::from_mapped`; for `qwen35` that call returns the @@ -535,12 +543,18 @@ fn note_gpu_golden_skip(config: &QaConfig, message: &str) { /// dispatches `qwen35` to `forward_qwen35` — so the gate certifies the backend /// that actually serves the model rather than re-implementing the dispatch. /// +/// The GPU is NOT disabled here: since #3090 the runtime routes the hybrid to +/// `Qwen35CudaModel` on a cuda build with a device, so this gate judges the GPU's +/// output — which is the point of #3477. On a CPU-only build or host the same +/// call serves CPU tokens and the gate judges those. Either way the backend +/// under judgement is the backend a user gets. +/// /// The golden prompts are already ChatML, so they are tokenized here and passed /// via `with_input_tokens` to bypass `prepare_tokens`' chat-template auto-wrap /// (the same reason `golden_output_apr` does it). Stop tokens come from the /// model's own EOS, which `run_gguf_inference` merges in. #[cfg(feature = "inference")] -fn golden_output_cpu_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result { +fn golden_output_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Result { use realizar::gguf::MappedGGUFModel; use realizar::{run_inference, InferenceConfig}; @@ -558,27 +572,26 @@ fn golden_output_cpu_runtime(path: &Path, prompt: &str, max_tokens: usize) -> Re .with_input_tokens(prompt_tokens) .with_max_tokens(max_tokens) .with_temperature(0.0) - .with_top_k(1) - .without_gpu(); + .with_top_k(1); let result = run_inference(&infer_config) - .map_err(|e| CliError::ValidationFailed(format!("CPU generation failed: {e}")))?; + .map_err(|e| CliError::ValidationFailed(format!("Generation failed: {e}")))?; Ok(result.text) } -/// Gate 1 for a CPU-only architecture: the same golden cases, run on the CPU -/// forward that serves the model. +/// Gate 1 for an architecture the dense loader refuses: the same golden cases, +/// run through the runtime entry point on whichever backend serves the model. /// /// This gate is the point of #3477: `apr qa` must be able to say PASS about the -/// backend that runs the model. The GPU half is not attempted at all — the GPU -/// declined this architecture (#3090) and its gates skip with that reason. +/// backend that runs the model — and since #3090 that backend is the GPU. #[cfg(feature = "inference")] -fn run_golden_output_gate_cpu_only(path: &Path, config: &QaConfig) -> Result { +fn run_golden_output_gate_runtime(path: &Path, config: &QaConfig) -> Result { let start = Instant::now(); if !config.json && config.verbose { println!( "{}", - "Running golden output test on the CPU forward (#3091)...".yellow() + "Running golden output test through the runtime entry point (GPU #3090 / CPU #3091)..." + .yellow() ); } @@ -587,10 +600,10 @@ fn run_golden_output_gate_cpu_only(path: &Path, config: &QaConfig) -> Result Result Result Result { +fn run_golden_output_gate_runtime(path: &Path, config: &QaConfig) -> Result { let _ = (path, config); Ok(GateResult::skipped( "golden_output", diff --git a/crates/apr-cli/src/commands/parity_refusal.rs b/crates/apr-cli/src/commands/parity_refusal.rs index 61e20b0f40..7d411094b8 100644 --- a/crates/apr-cli/src/commands/parity_refusal.rs +++ b/crates/apr-cli/src/commands/parity_refusal.rs @@ -21,8 +21,12 @@ // MoE parity support is #3367 (`apr chat` had the identical defect, // #3368). // * For Qwen3.5 (Gated DeltaNet + gated full attention), no GPU forward -// exists at all (#3090), so GPU==CPU is unmeasurable BY CONSTRUCTION — no -// run of this command could ever produce a number. +// existed at all when this was written (#3090), so GPU==CPU was +// unmeasurable BY CONSTRUCTION — no run of this command could produce a +// number. #3477 (operator ruling 2026-09-19) built that GPU forward +// (`Qwen35CudaModel`), so the ONE spelling the runtime dispatches is now +// measured; the undispatched spellings still reach no forward and are +// still refused. // // A tool that cannot measure something must say so. Falling through to the // dense loop produced a red row that named the MODEL, which is the exact class @@ -119,10 +123,11 @@ impl ParityRefusal { /// * `unsupported_architecture_reason(..)` — the predicate `apr check` and /// `apr ptx-map` already share so they cannot drift (GH-704 / #2399). /// -/// The Qwen3.5 arm additionally matches the architecture TAG, because a +/// The hybrid arm additionally matches the architecture TAG, because a /// header-only read must refuse even if the tensor scan is inconclusive; the /// spelling set is the one `ArchConstraints` uses -/// (`crates/aprender-serve/src/gguf/arch_constraints_fallback.rs`). +/// (`crates/aprender-serve/src/gguf/arch_constraints_fallback.rs`) MINUS the +/// one spelling the runtime actually dispatches, which #3477 admits. #[cfg(feature = "inference")] #[allow(dead_code)] pub(crate) fn parity_refusal_for<'n>( @@ -150,6 +155,16 @@ pub(crate) fn parity_refusal_for<'n>( }); } + // #3477 (operator ruling 2026-09-19): the architecture the runtime + // dispatches to the hybrid forward now HAS a GPU forward + // (`Qwen35CudaModel`, #3090) as well as the CPU one (#3091), so parity can + // compare the two and must stop refusing it. `hybrid_forward_handles` is + // realizar's own dispatch predicate — the same one `apr qa` and + // `apr ptx-map` ask — so this cannot drift from what `apr run` does. + if realizar::gguf::hybrid_forward_handles(architecture) { + return None; + } + let hybrid_tag = matches!( architecture.to_ascii_lowercase().as_str(), "qwen3_5" | "qwen3.5" | "qwen35" diff --git a/crates/apr-cli/src/commands/parity_refusal_tests.rs b/crates/apr-cli/src/commands/parity_refusal_tests.rs index 817d457332..79e0d3cf3d 100644 --- a/crates/apr-cli/src/commands/parity_refusal_tests.rs +++ b/crates/apr-cli/src/commands/parity_refusal_tests.rs @@ -15,8 +15,9 @@ mod parity_refusal_case_table { // and Qwen3-30B-A3B ship; `qwen3_moe` is the canonical key. Every // spelling `ArchConstraints` calls MoE must be refused, canonical or not. for raw in ["qwen3moe", "qwen3_moe", "qwen3_5_moe", "qwen3_5moe"] { - let r = parity_refusal_for(raw, Vec::<&str>::new()) - .unwrap_or_else(|| panic!("{raw} must be refused: the dense parity loop cannot route it")); + let r = parity_refusal_for(raw, Vec::<&str>::new()).unwrap_or_else(|| { + panic!("{raw} must be refused: the dense parity loop cannot route it") + }); assert_eq!(r.issue, "#3367"); assert!( r.reason.contains("MoE placeholder"), @@ -44,15 +45,32 @@ mod parity_refusal_case_table { ] { let r = parity_refusal_for(raw, Vec::<&str>::new()).expect("refused"); assert_eq!(r.architecture, named, "arch named for raw tag {raw}"); - assert_ne!(r.architecture, "llama", "never launder a refusal as a dense arch"); + assert_ne!( + r.architecture, "llama", + "never launder a refusal as a dense arch" + ); } } + /// #3477 (operator ruling 2026-09-19): `qwen35` — the ONE spelling the + /// runtime dispatches to the hybrid forward — now has a GPU forward + /// (`Qwen35CudaModel`, #3090) as well as the CPU one (#3091), so parity can + /// measure GPU-vs-CPU for it and must stop refusing it. Every undispatched + /// hybrid spelling keeps the refusal: no forward reaches them, so parity + /// would still be comparing nothing. #[test] - fn parity_refusal_qwen35_is_refused_with_issue_3090() { - for raw in ["qwen35", "qwen3.5", "qwen3_5", "QWEN3_5"] { + fn parity_refusal_qwen35_is_admitted_and_undispatched_spellings_are_refused() { + assert!( + parity_refusal_for("qwen35", Vec::<&str>::new()).is_none(), + "the GPU forward exists (#3090) — parity must run, not refuse" + ); + assert!( + parity_refusal_for("qwen35", ["blk.0.ssm_a", "blk.0.attn_qkv.weight"]).is_none(), + "the SSM tensors are exactly what the hybrid forward consumes" + ); + for raw in ["qwen3.5", "qwen3_5", "QWEN3_5"] { let r = parity_refusal_for(raw, Vec::<&str>::new()) - .unwrap_or_else(|| panic!("{raw} must be refused: no GPU forward exists")); + .unwrap_or_else(|| panic!("{raw} reaches no forward: it must be refused")); assert_eq!(r.issue, "#3090"); assert!( r.reason.contains("GPU forward"), @@ -62,17 +80,17 @@ mod parity_refusal_case_table { } } - /// `normalize_architecture("qwen35") == "qwen3"`, so naming the refusal with + /// `normalize_architecture("qwen3_5") == "qwen3"`, so naming the refusal with /// the shared normalizer would print `architecture=qwen3` — a dense arch that /// parity runs happily. The refusal must name the architecture that was /// actually refused. #[test] fn parity_refusal_qwen35_line_does_not_launder_the_arch_as_qwen3() { - let r = parity_refusal_for("qwen35", Vec::<&str>::new()).expect("refused"); - assert_eq!(r.architecture, "qwen35"); + let r = parity_refusal_for("qwen3_5", Vec::<&str>::new()).expect("refused"); + assert_eq!(r.architecture, "qwen3_5"); assert!( !r.line().contains("architecture=qwen3 "), - "the refusal must not report qwen35 as plain qwen3: {}", + "the refusal must not report qwen3_5 as plain qwen3: {}", r.line() ); } @@ -92,7 +110,9 @@ mod parity_refusal_case_table { /// `apr parity` still takes the ordinary CPU-vs-GPU path. #[test] fn parity_refusal_predicate_is_false_for_every_dense_arch() { - for raw in ["qwen2", "qwen2.5", "qwen3", "llama", "mistral", "gemma2", "phi3"] { + for raw in [ + "qwen2", "qwen2.5", "qwen3", "llama", "mistral", "gemma2", "phi3", + ] { assert!( parity_refusal_for(raw, ["blk.0.attn_q.weight", "output.weight"]).is_none(), "{raw} is a dense arch — parity must run it, not refuse it" @@ -136,17 +156,23 @@ mod parity_refusal_case_table { line.starts_with("parity: REFUSED architecture=qwen3_moe — "), "C14 greps this prefix: {line}" ); - assert!(line.ends_with("(#3367)"), "the issue is the last token: {line}"); + assert!( + line.ends_with("(#3367)"), + "the issue is the last token: {line}" + ); assert_eq!(line.lines().count(), 1, "exactly ONE line"); } /// `--json` mode emits the refusal as data, with the same three fields. #[test] fn parity_refusal_json_carries_architecture_reason_and_issue() { - let r = parity_refusal_for("qwen35", Vec::<&str>::new()).expect("refused"); + let r = parity_refusal_for("qwen3_5", Vec::<&str>::new()).expect("refused"); let v = r.json(); let refused = v.get("refused").expect("top-level `refused` key"); - assert_eq!(refused.get("architecture").and_then(|a| a.as_str()), Some("qwen35")); + assert_eq!( + refused.get("architecture").and_then(|a| a.as_str()), + Some("qwen3_5") + ); assert_eq!(refused.get("issue").and_then(|a| a.as_str()), Some("#3090")); assert!(refused .get("reason") diff --git a/crates/apr-cli/src/commands/ptx_map.rs b/crates/apr-cli/src/commands/ptx_map.rs index 6bab271b7d..978f72a124 100644 --- a/crates/apr-cli/src/commands/ptx_map.rs +++ b/crates/apr-cli/src/commands/ptx_map.rs @@ -402,6 +402,63 @@ fn build_prefill_sequence(info: &ModelInfo) -> Vec { ] } +/// #3477: what `ptx-map` prints for a model whose layers this map does not +/// enumerate, or `None` when the dense map is the truthful answer. +/// +/// The Qwen3.5 hybrid is no longer refused by +/// `unsupported_architecture_reason` — it HAS a GPU forward (`Qwen35CudaModel`, +/// #3090). But its Gated DeltaNet layers launch conv1d / per-head L2 / delta +/// rule / gated-RMSNorm kernels that this table has no rows for, so printing +/// the dense 12-step sequence (RoPE + GQA + SwiGLU) for them would be the same +/// fabrication #2399 found, with a different cause. Neither refusing (the +/// model runs) nor inventing a map (it does not run THOSE kernels) is honest, +/// so ptx-map says exactly what it knows. +/// +/// Takes the tensor names so the note is only produced for a file that really +/// carries Gated DeltaNet weights — a dense GGUF mislabelled `qwen35` would +/// still get the dense map it actually runs. +#[cfg(feature = "inference")] +fn hybrid_kernel_map_note<'n>( + architecture: &str, + tensor_names: impl IntoIterator, + num_layers: usize, +) -> Option { + if !realizar::gguf::hybrid_forward_handles(architecture) { + return None; + } + if !tensor_names + .into_iter() + .any(|name| name.contains("ssm_") || name.contains("ssm.")) + { + return None; + } + Some(format!( + " Architecture '{architecture}' is the Qwen3.5 hybrid: {num_layers} layers of Gated DeltaNet \ + (causal conv1d + SiLU, per-head L2 norm, delta-rule recurrence, gated RMSNorm) interleaved \ + with gated full attention.\n\ + \n hybrid layers: kernel map not enumerated yet, #3090.\n\ + \n The GPU forward exists (Qwen35CudaModel, #3090) and the CPU one does too (#3091); what is \ + missing is this table's rows for their kernels. The dense RoPE/GQA/SwiGLU sequence ptx-map \ + prints for a standard transformer is NOT what these layers launch, so it is not printed." + )) +} + +/// The hybrid note for the model at `path`, read from the GGUF header alone. +/// +/// `None` on any read/parse failure: `extract_model_info` is the one place that +/// reports a bad file, and it runs immediately after. +#[cfg(feature = "inference")] +fn hybrid_kernel_map_note_for_model(model_path: &Path) -> Option { + let mapped = + realizar::gguf::MappedGGUFModel::from_path(model_path.to_str().unwrap_or_default()).ok()?; + let config = realizar::gguf::GGUFConfig::from_gguf(&mapped.model).ok()?; + hybrid_kernel_map_note( + &config.architecture, + mapped.model.tensors.iter().map(|t| t.name.as_str()), + config.num_layers, + ) +} + /// Extract model info from GGUF file #[cfg(feature = "inference")] fn extract_model_info(model_path: &Path) -> Result { @@ -519,4 +576,63 @@ fn format_shared(bytes: u32) -> String { } } +#[cfg(all(test, feature = "inference"))] +mod hybrid_note_tests { + use super::hybrid_kernel_map_note; + + /// #3477: `apr ptx-map` used to REFUSE a Qwen3.5 GGUF with "NEITHER the CPU + /// nor the GPU backend implements Gated DeltaNet". Both backends implement + /// it now (#3091 CPU, #3090 GPU), so a refusal is false — and the dense + /// 12-step map is a fabrication for layers that launch conv1d / delta-rule + /// kernels. The honest output names what is missing: the MAP, not the + /// model. + #[test] + fn ptx_map_hybrid_note_is_printed_instead_of_a_dense_map() { + let note = hybrid_kernel_map_note("qwen35", ["token_embd.weight", "blk.0.ssm_a"], 28) + .expect("a hybrid GGUF must get the note"); + assert!( + note.contains("kernel map not enumerated yet") && note.contains("#3090"), + "the note must say what is missing and where it is tracked: {note}" + ); + assert!( + note.contains("28") && note.contains("Gated DeltaNet"), + "the note must report what the file says: {note}" + ); + // The refusal vocabulary must be gone: the model RUNS. + assert!( + !note.contains("NEITHER the CPU nor the GPU"), + "both backends implement it: {note}" + ); + // And nothing from the dense map may be claimed for these layers. + for dense in ["RopeKernel", "BatchedSwigluKernel", "launches/layer"] { + assert!( + !note.contains(dense), + "the note must not fabricate a dense kernel sequence ({dense}): {note}" + ); + } + } + + /// The note is for the hybrid alone: a dense model must still get its real + /// map, and a file carrying no SSM tensors is dense whatever its tag says. + #[test] + fn ptx_map_hybrid_note_is_absent_for_a_dense_model() { + assert_eq!( + hybrid_kernel_map_note("qwen2", ["blk.0.attn_q.weight"], 24), + None, + "a dense transformer's map is real — it must be printed" + ); + assert_eq!( + hybrid_kernel_map_note("qwen35", ["blk.0.attn_q.weight", "output.weight"], 24), + None, + "no Gated DeltaNet tensors: whatever the tag says, this file runs the dense kernels" + ); + assert_eq!( + hybrid_kernel_map_note("mamba", ["blk.0.ssm_a"], 24), + None, + "an architecture with no forward is REFUSED by unsupported_architecture_reason, \ + not given a note" + ); + } +} + include!("ptx_map_print_kernel.rs"); diff --git a/crates/apr-cli/src/commands/ptx_map_print_kernel.rs b/crates/apr-cli/src/commands/ptx_map_print_kernel.rs index 8d5c53275e..919ae6f061 100644 --- a/crates/apr-cli/src/commands/ptx_map_print_kernel.rs +++ b/crates/apr-cli/src/commands/ptx_map_print_kernel.rs @@ -1,4 +1,3 @@ - /// Print the kernel table fn print_kernel_table(steps: &[KernelStep], kernel_filter: Option<&str>) { print_table_header(); @@ -139,7 +138,10 @@ fn print_json(steps: &[KernelStep], info: &ModelInfo, prefill: bool) { /// Main entry point for ptx-map command #[allow(clippy::fn_params_excessive_bools)] -#[provable_contracts_macros::contract("apr-cli-command-safety-v1", equation = "read_only_no_side_effects")] +#[provable_contracts_macros::contract( + "apr-cli-command-safety-v1", + equation = "read_only_no_side_effects" +)] pub fn run( model_path: &Path, kernel_filter: Option<&str>, @@ -159,6 +161,14 @@ pub fn run( eprintln!(); } + // #3477: a hybrid (Gated DeltaNet) model runs on both backends, but its + // kernels have no rows in this table. Say so and stop, rather than + // refusing a model that runs or printing a dense map it never launches. + if let Some(note) = hybrid_kernel_map_note_for_model(model_path) { + println!("{note}"); + return Ok(()); + } + let info = extract_model_info(model_path)?; let steps = if prefill { diff --git a/crates/apr-cli/src/commands/qa_capability.rs b/crates/apr-cli/src/commands/qa_capability.rs index ddb7762db8..2b66110a7e 100644 --- a/crates/apr-cli/src/commands/qa_capability.rs +++ b/crates/apr-cli/src/commands/qa_capability.rs @@ -81,8 +81,8 @@ pub fn run_capability_gate(path: &Path, config: &QaConfig) -> Result )); } - // Parse GGUF to get architecture string and tensor names - let Some((arch, tensor_names)) = extract_gguf_arch_and_tensors(&data) else { + // Parse GGUF to get architecture string and tensor (name, GGML type) pairs + let Some((arch, tensors)) = extract_gguf_arch_and_tensors(&data) else { let duration = start.elapsed(); return Ok(GateResult::passed( "capability_match", @@ -101,12 +101,19 @@ pub fn run_capability_gate(path: &Path, config: &QaConfig) -> Result // `ssm_gate_result`. #[cfg(feature = "inference")] { - let tensor_refs: Vec<&str> = tensor_names.iter().map(|s| s.as_str()).collect(); + let tensor_refs: Vec<&str> = tensors.iter().map(|(n, _)| n.as_str()).collect(); let backend = if cfg!(feature = "cuda") { Backend::Gpu } else { Backend::Cpu }; + // #3477 / PMAT-781/783/785: the hybrid's Gated DeltaNet projections are + // invisible to the dense construction-time quant gate (the qwen35 base + // model carries an empty layer list), so the GPU rung judges them here, + // from the header, before anything is uploaded. + if let Some(result) = hybrid_quant_gate_result(&arch, &tensors, backend, start.elapsed()) { + return Ok(result); + } if let Some(result) = ssm_gate_result(&arch, tensor_refs, backend, start.elapsed()) { return Ok(result); } @@ -200,8 +207,8 @@ pub(crate) enum CapabilityVerdict { Declined(String), } -/// Is this the architecture string the CPU runtime dispatches to the Qwen3.5 -/// (Gated DeltaNet) forward? +/// Is this the architecture string the runtime dispatches to the Qwen3.5 +/// (Gated DeltaNet) hybrid forward? /// /// #3091: `infer/inference_result.rs` compares the GGUF architecture to the /// literal `"qwen35"` and, on a match, builds the model with @@ -209,46 +216,59 @@ pub(crate) enum CapabilityVerdict { /// mirrors that comparison exactly: admitting a spelling the runtime does not /// dispatch (`qwen3_5`, `qwen3.5`) would have `apr qa` promise a load that /// `apr run` refuses — the very drift #3091 is about, with the sign flipped. +/// +/// #3477: the comparison is no longer COPIED here. `realizar` exposes it as +/// `gguf::hybrid_forward_handles`, and `apr parity` / `apr ptx-map` ask the same +/// function, so the three tools cannot drift from the runtime or from each +/// other. #[cfg(feature = "inference")] pub(crate) fn cpu_forward_handles(arch: &str) -> bool { - arch == "qwen35" + realizar::gguf::hybrid_forward_handles(arch) } /// Verdict for a GGUF carrying SSM / Gated DeltaNet tensors. /// /// `realizar::gguf::unsupported_architecture_reason` stays the single predicate -/// for "does this file carry SSM tensors" (`apr ptx-map` and the parity refusal -/// depend on it answering yes for qwen35, #3090). What changed in #3091 is the -/// CPU half of its verdict: the CPU forward now runs Gated DeltaNet, so this -/// gate re-judges that one case per backend instead of repeating the -/// "NEITHER backend" sentence the runtime has outgrown. +/// for "is there any forward for this SSM file", and since #3477 it answers NO +/// REFUSAL for the hybrid the runtime dispatches — both backends implement it +/// (CPU #3091, GPU #3090). The per-backend judgement left here is about THIS +/// BINARY: a build without the `cuda` feature has no GPU backend linked in at +/// all, so it cannot certify one. #[cfg(feature = "inference")] pub(crate) fn hybrid_ssm_verdict<'n>( arch: &str, tensor_names: impl IntoIterator, backend: Backend, ) -> CapabilityVerdict { + // #3477: the hybrid is judged FIRST, because the shared predicate no longer + // refuses it — falling through would admit it on a non-cuda build's GPU + // backend, which does not exist. + if cpu_forward_handles(arch) { + return match backend { + Backend::Cpu => CapabilityVerdict::Admitted, + Backend::Gpu if cfg!(feature = "cuda") => CapabilityVerdict::Admitted, + Backend::Gpu => CapabilityVerdict::Declined(format!( + "Architecture '{arch}': the Gated DeltaNet GPU forward exists (#3090), but this \ + binary has no CUDA backend — it was built without `--features cuda`. The CPU \ + forward runs it here (#3091)." + )), + }; + } let Some(reason) = realizar::gguf::unsupported_architecture_reason(arch, tensor_names) else { return CapabilityVerdict::Admitted; }; - if !cpu_forward_handles(arch) { - return CapabilityVerdict::Declined(reason); - } - match backend { - Backend::Cpu => CapabilityVerdict::Admitted, - Backend::Gpu => CapabilityVerdict::Declined(format!( - "Architecture '{arch}': Gated DeltaNet runs on the CPU only — the GPU backend has no \ - SSM kernels (#3090). Run it with `apr run --no-gpu` (CPU forward: #3091)." - )), - } + CapabilityVerdict::Declined(reason) } /// The Capability Match message for an architecture only the CPU rung certifies. +/// +/// #3477: reachable only on a build WITHOUT `cuda` — the GPU forward for the +/// hybrid exists (#3090); what a non-cuda binary lacks is the backend itself. #[cfg(feature = "inference")] pub(crate) fn cpu_only_pass_message(arch: &str) -> String { format!( - "Architecture '{arch}': CPU-only — Gated DeltaNet runs on the CPU (#3091); the GPU \ - backend has no SSM kernels (#3090), GPU gates skipped" + "Architecture '{arch}': CPU-only in this binary — Gated DeltaNet runs on the CPU (#3091) \ + and on the GPU (#3090), but this build has no CUDA backend, GPU gates skipped" ) } @@ -292,6 +312,39 @@ pub(crate) fn ssm_gate_result<'n>( )) } +/// The Capability Match FAIL for a hybrid GGUF whose Gated DeltaNet weights +/// carry a quant type with no GPU GEMV kernel, or `None`. +/// +/// PMAT-781/783/785: the GPU weight upload decodes an unrecognized GGML type as +/// Q4_K, so such a file would produce garbage logits rather than an error. The +/// dense construction-time gate (`OwnedQuantizedModel::has_gpu_unsupported_quant`) +/// cannot catch it for the hybrid — the `OwnedQuantizedModel` built for `qwen35` +/// has an EMPTY layer list, the DeltaNet weights living in `Qwen35Model` — so +/// the verdict is taken from the header here. Only the GPU rung asks: the CPU +/// forward decodes every quant type correctly. +#[cfg(feature = "inference")] +pub(crate) fn hybrid_quant_gate_result( + arch: &str, + tensors: &[(String, u32)], + backend: Backend, + duration: Duration, +) -> Option { + if backend != Backend::Gpu { + return None; + } + let reason = realizar::gguf::hybrid_gpu_quant_refusal( + arch, + tensors.iter().map(|(n, q)| (n.as_str(), *q)), + )?; + Some(GateResult::failed( + "capability_match", + &reason, + Some(1.0), + Some(0.0), + duration, + )) +} + /// Does the GPU decline this architecture while the CPU forward runs it? /// /// This is the predicate the `apr qa` driver asks to decide which gates it can @@ -327,10 +380,10 @@ pub(crate) fn cpu_only_architecture(path: &Path) -> bool { if data.len() < 4 || &data[0..4] != b"GGUF" { return false; } - let Some((arch, tensor_names)) = extract_gguf_arch_and_tensors(&data) else { + let Some((arch, tensors)) = extract_gguf_arch_and_tensors(&data) else { return false; }; - is_cpu_only_architecture(&arch, tensor_names.iter().map(String::as_str)) + is_cpu_only_architecture(&arch, tensors.iter().map(|(n, _)| n.as_str())) } /// Without the `inference` feature there is no runtime to speak for. @@ -339,13 +392,49 @@ pub(crate) fn cpu_only_architecture(_path: &Path) -> bool { false } -/// Extract the architecture string from GGUF metadata. +/// Is the model at `path` a hybrid the DENSE GGUF loader refuses? +/// +/// #3477: `cpu_only_architecture` answers "which backend", which for `qwen35` on +/// a cuda build is now BOTH (#3090 GPU, #3091 CPU). A separate question survives +/// it: a gate built on `OwnedQuantizedModel::from_mapped` — Ollama parity, +/// cross-format parity, GPU speedup, GPU state isolation — still cannot run, +/// because that loader has no hybrid path on either rung. Those gates SKIP with +/// that reason; they must not FAIL, and they must not be answered with a number +/// measured on some other model shape. +#[cfg(feature = "inference")] +pub(crate) fn hybrid_loader_architecture(path: &Path) -> bool { + let Ok(data) = std::fs::read(path) else { + return false; + }; + if data.len() < 4 || &data[0..4] != b"GGUF" { + return false; + } + let Some((arch, _)) = extract_gguf_arch_and_tensors(&data) else { + return false; + }; + cpu_forward_handles(&arch) +} + +/// Without `inference` there is no loader to speak for. +#[cfg(not(feature = "inference"))] +pub(crate) fn hybrid_loader_architecture(_path: &Path) -> bool { + false +} + +/// Extract the architecture string and the tensor table from GGUF metadata. /// /// Uses aprender's GGUF reader to parse metadata without loading tensors. -fn extract_gguf_arch_and_tensors(data: &[u8]) -> Option<(String, Vec)> { +/// #3477: the GGML type travels with the name, because the hybrid GPU quant +/// gate judges `(name, type)` pairs — reading the names alone was what let an +/// unsupported-quant DeltaNet tensor through. +fn extract_gguf_arch_and_tensors(data: &[u8]) -> Option<(String, Vec<(String, u32)>)> { let reader = aprender::format::gguf::reader::GgufReader::from_bytes(data.to_vec()).ok()?; let arch = reader.architecture()?; - let tensors = reader.tensors.into_iter().map(|t| t.name).collect(); + let tensors = reader + .tensors + .into_iter() + .map(|t| (t.name, t.dtype)) + .collect(); Some((arch, tensors)) } @@ -369,27 +458,96 @@ mod qa_capability_ssm_tests { ); } - // The GPU has no Gated DeltaNet kernels (#3090) — that refusal stands, and - // must not be re-worded into the old "NEITHER backend" claim. + // #3477 (operator ruling 2026-09-19): the GPU forward for the hybrid exists + // (`Qwen35CudaModel`, #3090), so a binary built WITH `cuda` must admit it + // on the GPU backend. A binary built WITHOUT `cuda` has no GPU backend + // linked in at all — that is the only thing left to decline, and the reason + // must say so rather than claiming the kernels do not exist. #[test] - fn qwen35_ssm_model_is_declined_on_gpu() { - let CapabilityVerdict::Declined(reason) = - hybrid_ssm_verdict("qwen35", ["blk.0.ssm_a"], Backend::Gpu) - else { - panic!("the GPU has no Gated DeltaNet kernels (#3090) — it must decline"); + fn qwen35_ssm_model_is_admitted_on_gpu_under_cuda() { + let verdict = hybrid_ssm_verdict("qwen35", ["blk.0.ssm_a"], Backend::Gpu); + if cfg!(feature = "cuda") { + assert_eq!( + verdict, + CapabilityVerdict::Admitted, + "the GPU runs qwen35 since #3090 — the capability gate must admit it" + ); + return; + } + let CapabilityVerdict::Declined(reason) = verdict else { + panic!("a binary without the cuda feature has no GPU backend to certify"); }; assert!( - reason.contains("#3090"), - "must cite the GPU issue: {reason}" + reason.contains("no CUDA backend"), + "the reason must be this BINARY's missing backend, not missing kernels: {reason}" ); assert!( - reason.contains("CPU"), - "must point the user at the backend that does run it: {reason}" + reason.contains("#3090"), + "must cite the GPU issue: {reason}" ); assert!( !reason.contains("NEITHER the CPU nor the GPU"), "the CPU claim is false since #3091: {reason}" ); + assert!( + !reason.contains("the GPU backend has no"), + "the GPU backend HAS Gated DeltaNet kernels since #3090: {reason}" + ); + } + + // #3477 / PMAT-781/783/785: a Gated DeltaNet projection carrying a quant + // type with no GPU GEMV kernel must FAIL the GPU rung loudly. The dense + // construction-time gate cannot see these tensors at all (the qwen35 base + // model has an empty layer list), so before this the file went to the GPU + // and was decoded as Q4_K. + #[test] + fn qwen35_with_a_gpu_unsupported_quant_fails_the_gpu_rung() { + let tensors = vec![ + ("token_embd.weight".to_string(), 12u32), + ("blk.0.ssm_beta.weight".to_string(), 11), // Q3_K: no GPU kernel + ]; + let result = super::hybrid_quant_gate_result( + "qwen35", + &tensors, + Backend::Gpu, + std::time::Duration::from_millis(1), + ) + .expect("a Q3_K DeltaNet projection has no GPU GEMV kernel"); + assert!( + !result.passed, + "the GPU rung must FAIL, not warn: {result:?}" + ); + assert!( + result.message.contains("blk.0.ssm_beta.weight"), + "the message must name the tensor: {}", + result.message + ); + // The CPU decodes every quant type correctly — it must not be failed. + assert!( + super::hybrid_quant_gate_result( + "qwen35", + &tensors, + Backend::Cpu, + std::time::Duration::from_millis(1) + ) + .is_none(), + "the CPU forward runs any quant type — only the GPU rung asks" + ); + // A hybrid whose projections are all GPU-eligible passes through. + let ok = vec![ + ("blk.0.attn_qkv.weight".to_string(), 8u32), + ("blk.0.ssm_out.weight".to_string(), 12), + ]; + assert!( + super::hybrid_quant_gate_result( + "qwen35", + &ok, + Backend::Gpu, + std::time::Duration::from_millis(1) + ) + .is_none(), + "Q8_0/Q4_K have verified GPU kernels — this model is GPU-eligible" + ); } // The admission mirrors the runtime dispatch, which compares the GGUF @@ -405,17 +563,19 @@ mod qa_capability_ssm_tests { panic!("'{arch}' is not dispatched to forward_qwen35 — it must be declined"); }; assert!( - reason.contains("NEITHER the CPU nor the GPU"), - "an undispatched hybrid keeps the original refusal: {reason}" + reason.contains("no CPU or GPU forward for architecture"), + "an undispatched hybrid keeps the refusal: {reason}" ); } } - // #3477: the GPU declining is not the end of the gate. The runtime serves - // this model on the CPU (`apr run` → `forward_qwen35`), so `apr qa` must be - // able to say PASS about the backend that runs it and skip the GPU gates — - // the FAIL turned 8 of 12 gates into "Skipped due to capability match - // failure" and exited 5 on a model that generates correct text. + // #3477: on a cuda build the gate no longer stops here at all — the GPU + // forward exists (#3090), so the hybrid falls through to the ordinary + // per-op GPU check like any other architecture. On a build without `cuda` + // there is no GPU backend to certify, and the CPU rung still carries the + // report (the earlier FAIL turned 8 of 12 gates into "Skipped due to + // capability match failure" and exited 5 on a model that generates correct + // text). #[test] fn qwen35_on_the_gpu_backend_is_a_cpu_only_pass() { let result = super::ssm_gate_result( @@ -423,8 +583,15 @@ mod qa_capability_ssm_tests { ["token_embd.weight", "blk.0.ssm_a"], Backend::Gpu, std::time::Duration::from_millis(1), - ) - .expect("a hybrid SSM model is never simply admitted on the GPU"); + ); + if cfg!(feature = "cuda") { + assert!( + result.is_none(), + "the GPU runs qwen35 (#3090) — the per-op GPU check must run, not a CPU-only PASS: {result:?}" + ); + return; + } + let result = result.expect("without cuda there is no GPU backend to certify"); assert!( result.passed && !result.skipped, "the CPU rung certifies this model, so the gate PASSES: {result:?}" @@ -436,7 +603,7 @@ mod qa_capability_ssm_tests { ); assert!( result.message.contains("#3091") && result.message.contains("#3090"), - "the message must cite the CPU forward and the missing GPU kernels: {}", + "the message must cite both forwards: {}", result.message ); assert!( @@ -474,9 +641,11 @@ mod qa_capability_ssm_tests { // must answer no, or a GPU-capable model would lose its GPU gates. #[test] fn only_a_gpu_declined_cpu_running_architecture_is_cpu_only() { - assert!( + assert_eq!( super::is_cpu_only_architecture("qwen35", ["blk.0.ssm_a"]), - "the GPU declines qwen35 (#3090) and the CPU runs it (#3091)" + !cfg!(feature = "cuda"), + "#3477: with cuda the GPU runs qwen35 (#3090) so its GPU gates must NOT be skipped; \ + without cuda this binary has no GPU backend and the CPU rung carries the report" ); assert!( !super::is_cpu_only_architecture("qwen2", ["blk.0.attn_q.weight"]), diff --git a/crates/apr-cli/src/commands/qa_gguf.rs b/crates/apr-cli/src/commands/qa_gguf.rs index 7422d6834e..a9e727b00f 100644 --- a/crates/apr-cli/src/commands/qa_gguf.rs +++ b/crates/apr-cli/src/commands/qa_gguf.rs @@ -1,4 +1,3 @@ - /// Check if model is GGUF format (for Ollama parity gate). fn is_gguf_format(path: &Path) -> bool { #[cfg(feature = "inference")] @@ -21,12 +20,26 @@ fn is_gguf_format(path: &Path) -> bool { } /// Skip reason for a gate that needs the GPU on an architecture the GPU declined. -const GPU_DECLINED_SKIP: &str = "GPU declined for this architecture (#3090)"; +/// +/// #3477: the `(#3090)` citation is gone. #3090 is the Gated DeltaNet GPU +/// forward, which now EXISTS — citing it as the reason a GPU gate skipped +/// survived as a stale claim long after the refusal it named was lifted. This +/// arm is reached only when `is_cpu_only_architecture` finds a real (CPU runs, +/// GPU declines) pair, so it must speak for that pair and nothing else. +const GPU_DECLINED_SKIP: &str = + "The GPU backend declined this architecture; the CPU forward carries the report"; /// Skip reason for a gate built on the dense GGUF loader /// (`OwnedQuantizedModel::from_mapped`), which refuses Gated DeltaNet. +/// +/// #3477: this string used to end "GPU declined (#3090)", which is now FALSE — +/// the hybrid has a GPU forward and the Golden Output and Throughput gates above +/// are measured on it. The reason a dense-loader gate still cannot run is the +/// loader, on BOTH rungs; saying anything about the GPU's capability here is the +/// withdrawn claim this ticket exists to delete. const DENSE_LOADER_SKIP: &str = - "Dense-loader gate: Gated DeltaNet has its own CPU forward (#3091), GPU declined (#3090)"; + "Dense-loader gate: Gated DeltaNet runs on the CPU (#3091) and the GPU (#3090), but this \ + gate measures through OwnedQuantizedModel::from_mapped, which has no hybrid path"; /// Skip decision for a GPU-dependent gate (GPU speedup, PTX parity, GPU state /// isolation). @@ -55,20 +68,50 @@ fn dense_gate_skip(cpu_only: bool, skip: bool, reason: &'static str) -> (bool, & } } -/// Golden output for this architecture: the CPU-only rung goes through the -/// runtime entry point, everything else through the existing gate. -fn golden_gate_for(cpu_only: bool, path: &Path, config: &QaConfig) -> Result { - if cpu_only { - run_golden_output_gate_cpu_only(path, config) +/// Skip reason for a GPU gate whose measurement is taken through the dense +/// `OwnedQuantizedModelCuda`, which has no hybrid path. +/// +/// #3477: NOT "the GPU declined" — the GPU runs this architecture (#3090) and +/// the Golden Output gate above judges its output. What is missing is this +/// gate's own instrument. +const HYBRID_DENSE_GPU_SKIP: &str = + "Gated DeltaNet runs on the GPU (#3090), but this gate measures through the dense \ + OwnedQuantizedModelCuda, which has no hybrid path"; + +/// Skip decision for a GPU gate on a hybrid file. +fn hybrid_gpu_gate_skip(hybrid: bool, skip: bool, reason: &'static str) -> (bool, &'static str) { + if hybrid { + (true, HYBRID_DENSE_GPU_SKIP) + } else { + (skip, reason) + } +} + +/// Golden output for this architecture: a file the dense loader refuses goes +/// through the runtime entry point — which since #3477 serves `qwen35` on the +/// GPU (#3090) when one is present, so this gate judges the GPU's output. +fn golden_gate_for( + cpu_only: bool, + hybrid: bool, + path: &Path, + config: &QaConfig, +) -> Result { + if cpu_only || hybrid { + run_golden_output_gate_runtime(path, config) } else { run_golden_output_gate(path, config) } } /// Throughput for this architecture, same split as `golden_gate_for`. -fn throughput_gate_for(cpu_only: bool, path: &Path, config: &QaConfig) -> Result { - if cpu_only { - run_throughput_gate_cpu_only(path, config) +fn throughput_gate_for( + cpu_only: bool, + hybrid: bool, + path: &Path, + config: &QaConfig, +) -> Result { + if cpu_only || hybrid { + run_throughput_gate_runtime(path, config) } else { run_throughput_gate(path, config) } @@ -160,6 +203,9 @@ fn run_qa(path: &Path, config: &QaConfig) -> Result { // (qwen35, #3091) certifies on the CPU rung — the CPU gates run for real and // only the GPU/dense-loader gates skip, each with its own reason. let cpu_only = super::qa_capability::cpu_only_architecture(path); + // #3477: and independently of which backend runs it — the dense loader has + // no hybrid path, so every gate built on it skips with that reason. + let hybrid = super::qa_capability::hybrid_loader_architecture(path); let get_skip = |skip: bool, reason: &'static str| -> (bool, &'static str) { if capability_match_failed { @@ -172,10 +218,14 @@ fn run_qa(path: &Path, config: &QaConfig) -> Result { // The CPU gates run for real on a CPU-only architecture — through the same // entry point `apr run` uses, because the dense loader refuses the model. let (s, r) = get_skip(config.skip_golden, "Skipped by --skip-golden"); - dispatch_gate(&mut gates, config.json, s, "golden_output", r, || golden_gate_for(cpu_only, path, config))?; + dispatch_gate(&mut gates, config.json, s, "golden_output", r, || { + golden_gate_for(cpu_only, hybrid, path, config) + })?; let (s, r) = get_skip(config.skip_throughput, "Skipped by --skip-throughput"); - dispatch_gate(&mut gates, config.json, s, "throughput", r, || throughput_gate_for(cpu_only, path, config))?; + dispatch_gate(&mut gates, config.json, s, "throughput", r, || { + throughput_gate_for(cpu_only, hybrid, path, config) + })?; let is_ollama_fmt = is_gguf_format(path); let orig_skip_ollama = config.skip_ollama || !is_ollama_fmt; @@ -185,25 +235,39 @@ fn run_qa(path: &Path, config: &QaConfig) -> Result { "Skipped by --skip-ollama" }; let (s, r) = get_skip(orig_skip_ollama, orig_reason_ollama); - let (s, r) = dense_gate_skip(cpu_only, s, r); - dispatch_gate(&mut gates, config.json, s, "ollama_parity", r, || run_ollama_parity_gate(path, config))?; + let (s, r) = dense_gate_skip(cpu_only || hybrid, s, r); + dispatch_gate(&mut gates, config.json, s, "ollama_parity", r, || { + run_ollama_parity_gate(path, config) + })?; let (s, r) = get_skip(config.skip_gpu_speedup, "Skipped by --skip-gpu-speedup"); let (s, r) = gpu_gate_skip(cpu_only, s, r); - dispatch_gate(&mut gates, config.json, s, "gpu_speedup", r, || run_gpu_speedup_gate(path, config))?; + let (s, r) = hybrid_gpu_gate_skip(hybrid, s, r); + dispatch_gate(&mut gates, config.json, s, "gpu_speedup", r, || { + run_gpu_speedup_gate(path, config) + })?; // get_skip expects &'static str, but the format-parity reason is &str, so // the three-way decision lives in its own function. - let (s, r) = format_parity_skip(capability_match_failed, cpu_only, config); - dispatch_gate(&mut gates, config.json, s, "format_parity", r, || run_format_parity_gate(path, config))?; + let (s, r) = format_parity_skip(capability_match_failed, cpu_only || hybrid, config); + dispatch_gate(&mut gates, config.json, s, "format_parity", r, || { + run_format_parity_gate(path, config) + })?; + // PTX parity reads the GGUF config only — no dense loader, so the hybrid + // runs it for real (#3477). let (s, r) = get_skip(config.skip_ptx_parity, "Skipped by --skip-ptx-parity"); let (s, r) = gpu_gate_skip(cpu_only, s, r); - dispatch_gate(&mut gates, config.json, s, "ptx_parity", r, || run_ptx_parity_gate(path, config))?; + dispatch_gate(&mut gates, config.json, s, "ptx_parity", r, || { + run_ptx_parity_gate(path, config) + })?; let (s, r) = get_skip(config.skip_gpu_state, "Skipped by --skip-gpu-state"); let (s, r) = gpu_gate_skip(cpu_only, s, r); - dispatch_gate(&mut gates, config.json, s, "gpu_state_isolation", r, || run_gpu_state_isolation_gate(path, config))?; + let (s, r) = hybrid_gpu_gate_skip(hybrid, s, r); + dispatch_gate(&mut gates, config.json, s, "gpu_state_isolation", r, || { + run_gpu_state_isolation_gate(path, config) + })?; // Gate 9: Performance regression detection (auto-discovers previous report) dispatch_regression_gate(path, &mut gates, config)?; @@ -511,12 +575,14 @@ fn run_classifier_head_gate(path: &Path, config: &QaConfig) -> Result Result { } /// #3477: measure decode throughput for an architecture the dense loader -/// refuses but the runtime serves on the CPU (Qwen3.5 Gated DeltaNet, #3091). +/// refuses but the runtime serves (Qwen3.5 Gated DeltaNet — GPU #3090, CPU #3091). /// /// `throughput_gguf` builds the model with `OwnedQuantizedModel::from_mapped`, -/// which refuses `qwen35`. This measures the path `apr run` actually takes. +/// which refuses `qwen35`. This measures the path `apr run` actually takes — +/// including its BACKEND: the GPU is not disabled here, so on a cuda build with +/// a device this is the hybrid's GPU decode rate, the number a user sees. /// `inference_ms` excludes model load, matching what `measure_generate_throughput` /// times on the dense path (the model is built outside the timed closure there). /// @@ -117,19 +119,18 @@ fn run_throughput_gate(path: &Path, config: &QaConfig) -> Result { /// The threshold is untouched (10 tok/s for unasserted GGUF); what changes is /// that the number the gate compares is decode throughput. #[cfg(feature = "inference")] -fn throughput_cpu_runtime(path: &Path, config: &QaConfig) -> Result { +fn throughput_runtime(path: &Path, config: &QaConfig) -> Result { use realizar::{run_inference, InferenceConfig}; let infer_config = InferenceConfig::new(path) .with_prompt("Write a hello world program in Python:") .with_max_tokens(config.max_tokens.max(128)) .with_temperature(0.0) - .with_top_k(1) - .without_gpu(); + .with_top_k(1); let run = || { run_inference(&infer_config) - .map_err(|e| CliError::ValidationFailed(format!("CPU generation failed: {e}"))) + .map_err(|e| CliError::ValidationFailed(format!("Generation failed: {e}"))) }; for _ in 0..config.warmup { @@ -151,25 +152,32 @@ fn throughput_cpu_runtime(path: &Path, config: &QaConfig) -> Result { }) } -/// Gate 2 for a CPU-only architecture: the same falsifiable throughput floor, -/// measured on the CPU forward that serves the model. -fn run_throughput_gate_cpu_only(path: &Path, config: &QaConfig) -> Result { +/// Gate 2 for an architecture the dense loader refuses: the same falsifiable +/// throughput floor, measured through the runtime entry point on whichever +/// backend serves the model (GPU #3090 on a cuda build, else CPU #3091). +fn run_throughput_gate_runtime(path: &Path, config: &QaConfig) -> Result { let start = Instant::now(); if !config.json && config.verbose { println!( "{}", - "Running throughput benchmark on the CPU forward (#3091)...".yellow() + "Running throughput benchmark through the runtime entry point (#3090/#3091)..." + .yellow() ); } #[cfg(feature = "inference")] { - let tps = throughput_cpu_runtime(path, config)?; + let tps = throughput_runtime(path, config)?; let threshold = throughput_threshold(config.min_tps, realizar::format::ModelFormat::Gguf); let duration = start.elapsed(); + let backend = if cfg!(feature = "cuda") { + "hybrid forward, GPU #3090" + } else { + "hybrid forward, CPU #3091" + }; let message = format!( - "{tps:.1} tok/s {} {threshold:.0} tok/s threshold (CPU forward, #3091)", + "{tps:.1} tok/s {} {threshold:.0} tok/s threshold ({backend})", if tps >= threshold { ">=" } else { "<" } ); if tps >= threshold { @@ -232,14 +240,9 @@ fn ollama_parity_grade(ratio: f64) -> &'static str { /// decode-only throughput (eval_count/eval_duration), so short runs /// unfairly penalize our measurement. #[cfg(feature = "inference")] -fn measure_our_gguf_tps( - path: &Path, - config: &QaConfig, - tracer: &TracerImpl, -) -> Result { +fn measure_our_gguf_tps(path: &Path, config: &QaConfig, tracer: &TracerImpl) -> Result { use realizar::gguf::{ - GGUFModel, MappedGGUFModel, OwnedQuantizedModel, - QuantizedGenerateConfig, + GGUFModel, MappedGGUFModel, OwnedQuantizedModel, QuantizedGenerateConfig, }; let model_bytes = std::fs::read(path) @@ -407,11 +410,7 @@ fn run_ollama_parity_gate(path: &Path, config: &QaConfig) -> Result /// Measure GPU and CPU throughput for a GGUF model, returning (cpu_tps, gpu_tps). #[cfg_attr(coverage_nightly, coverage(off))] #[cfg(all(feature = "inference", feature = "cuda"))] -fn measure_gpu_cpu_tps( - path: &Path, - config: &QaConfig, - tracer: &TracerImpl, -) -> Result<(f64, f64)> { +fn measure_gpu_cpu_tps(path: &Path, config: &QaConfig, tracer: &TracerImpl) -> Result<(f64, f64)> { use realizar::gguf::{ GGUFModel, MappedGGUFModel, OwnedQuantizedModel, OwnedQuantizedModelCuda, QuantizedGenerateConfig, diff --git a/crates/aprender-gpu/src/kernels/gdn/causal_conv1d.rs b/crates/aprender-gpu/src/kernels/gdn/causal_conv1d.rs new file mode 100644 index 0000000000..dceaa3928b --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/causal_conv1d.rs @@ -0,0 +1,303 @@ +//! PMAT-3477: fused causal depthwise conv1d + SiLU for Gated `DeltaNet`. +//! +//! One decode step of `causal_conv1d` followed by the SiLU loop that +//! `forward_deltanet` applies to its output: +//! +//! ```text +//! for c in 0..channels: +//! sum = sum_{k Self { + assert!(kernel_size >= 1, "kernel_size must be >= 1"); + Self { + channels, + kernel_size, + } + } + + /// Launch grid for `channels` channels. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.channels.div_ceil(ELEMENTWISE_BLOCK), 1, 1) + } + + /// Launch block (one thread per channel). + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (ELEMENTWISE_BLOCK, 1, 1) + } +} + +impl Kernel for CausalConv1dSiluKernel { + fn name(&self) -> &str { + "gdn_causal_conv1d_silu" + } + + fn build_ptx(&self) -> PtxKernel { + let channels = self.channels; + let k = self.kernel_size; + let state_len = k - 1; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "input_ptr") // [channels] + .param(PtxType::U64, "state_ptr") // [channels][kernel_size - 1], updated in place + .param(PtxType::U64, "weight_ptr") // [channels][kernel_size] + .param(PtxType::U64, "output_ptr") // [channels] + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let cta = ctx.special_reg(PtxReg::CtaIdX); + let block = ctx.mov_u32_imm(ELEMENTWISE_BLOCK); + let chan = ctx.mad_lo_u32(cta, block, tid); + + let channels_r = ctx.mov_u32_imm(channels); + let in_bounds = ctx.setp_lt_u32(chan, channels_r); + ctx.branch_if_not(in_bounds, "gdn_conv_exit"); + + let input_ptr = ctx.load_param_u64("input_ptr"); + let state_ptr = ctx.load_param_u64("state_ptr"); + let weight_ptr = ctx.load_param_u64("weight_ptr"); + let output_ptr = ctx.load_param_u64("output_ptr"); + + let four = ctx.mov_u32_imm(4); + let chan_bytes = ctx.mul_wide_u32_reg(chan, four); + + // input[c] — read before the window is rewritten. + let in_addr = ctx.add_u64(input_ptr, chan_bytes); + let x = ctx.ld_global_f32(in_addr); + + // state row base: c * (K - 1) * 4, weight row base: c * K * 4 + let s_stride = ctx.mov_u32_imm(state_len * 4); + let s_off = ctx.mul_wide_u32_reg(chan, s_stride); + let s_base = ctx.add_u64(state_ptr, s_off); + let w_stride = ctx.mov_u32_imm(k * 4); + let w_off = ctx.mul_wide_u32_reg(chan, w_stride); + let w_base = ctx.add_u64(weight_ptr, w_off); + + // Load the whole window first: the shift below overwrites it, and the + // CPU reference reads every past sample before it moves any of them. + let state_addrs: Vec<_> = (0..state_len) + .map(|i| { + let off = ctx.mov_u64_imm(u64::from(i * 4)); + ctx.add_u64(s_base, off) + }) + .collect(); + let state_vals: Vec<_> = state_addrs + .iter() + .map(|&addr| ctx.ld_global_f32(addr)) + .collect(); + + // sum = sum_k state[k] * w[k] + x * w[K-1], in the CPU's order and with + // the CPU's separate multiply and add (no FMA contraction). + let sum = ctx.mov_f32_imm(0.0); + for i in 0..state_len { + let off = ctx.mov_u64_imm(u64::from(i * 4)); + let w_addr = ctx.add_u64(w_base, off); + let w_val = ctx.ld_global_f32(w_addr); + let prod = ctx.mul_f32(state_vals[i as usize], w_val); + ctx.add_f32_inplace(sum, prod); + } + let last_off = ctx.mov_u64_imm(u64::from((k - 1) * 4)); + let w_last_addr = ctx.add_u64(w_base, last_off); + let w_last = ctx.ld_global_f32(w_last_addr); + let prod_last = ctx.mul_f32(x, w_last); + ctx.add_f32_inplace(sum, prod_last); + + // Shift the window left and append the new sample. + for i in 0..state_len.saturating_sub(1) { + ctx.st_global_f32(state_addrs[i as usize], state_vals[i as usize + 1]); + } + if state_len >= 1 { + ctx.st_global_f32(state_addrs[state_len as usize - 1], x); + } + + let activated = emit_silu_f32(ctx, sum); + let out_addr = ctx.add_u64(output_ptr, chan_bytes); + ctx.st_global_f32(out_addr, activated); + + ctx.label("gdn_conv_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_conv1d_ptx_shape() { + let kernel = CausalConv1dSiluKernel::new(6144, 4); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_causal_conv1d_silu"), "{ptx}"); + // 3 window loads + 4 weight loads + 1 input load + assert_eq!(ptx.matches("ld.global.f32").count(), 8, "{ptx}"); + // 2 shifted window stores + 1 appended sample + 1 output + assert_eq!(ptx.matches("st.global.f32").count(), 4, "{ptx}"); + assert_eq!(kernel.grid(), (24, 1, 1)); + assert_eq!(kernel.block(), (256, 1, 1)); + } +} + +/// Device parity against a verbatim port of `causal_conv1d` + the SiLU loop. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_conv1d_device_tests { + use super::CausalConv1dSiluKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{ + assert_close, run_kernel, Lcg, CONV_CHANNELS, CONV_KERNEL, + }; + + /// Verbatim port of `aprender-serve`'s `causal_conv1d` (forward_qwen35.rs:85). + fn causal_conv1d( + input: &[f32], + state: &mut [f32], + weight: &[f32], + kernel_size: usize, + channels: usize, + output: &mut [f32], + ) { + for c in 0..channels { + let mut sum = 0.0; + let s_offset = c * (kernel_size - 1); + let w_offset = c * kernel_size; + + for k in 0..(kernel_size - 1) { + sum += state[s_offset + k] * weight[w_offset + k]; + } + sum += input[c] * weight[w_offset + kernel_size - 1]; + + for k in 0..(kernel_size - 2) { + state[s_offset + k] = state[s_offset + k + 1]; + } + if kernel_size > 1 { + state[s_offset + kernel_size - 2] = input[c]; + } + + output[c] = sum; + } + } + + /// The SiLU loop `forward_deltanet` applies to the conv output, once. + fn silu_in_place(x: &mut [f32]) { + for v in x.iter_mut() { + *v = *v / (1.0 + (-*v).exp()); + } + } + + fn reference( + input: &[f32], + state: &mut [f32], + weight: &[f32], + kernel_size: usize, + channels: usize, + ) -> Vec { + let mut out = vec![0.0; channels]; + causal_conv1d(input, state, weight, kernel_size, channels, &mut out); + silu_in_place(&mut out); + out + } + + #[test] + fn gdn_causal_conv1d_silu_matches_cpu_reference_three_steps() { + let Ok(ctx) = CudaContext::new(0) else { + println!( + "gdn_causal_conv1d_silu: no CUDA device — SKIPPED. gdn_conv1d_ptx_shape \ + still guards the codegen." + ); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let mut rng = Lcg::new(0x3477_0001); + let weight = rng.vec(CONV_CHANNELS * CONV_KERNEL, 0.5); + let mut host_state = rng.vec(CONV_CHANNELS * (CONV_KERNEL - 1), 1.0); + let steps: Vec> = (0..3).map(|_| rng.vec(CONV_CHANNELS, 1.0)).collect(); + + let weight_buf = GpuBuffer::from_host(&ctx, &weight).expect("weight"); + let state_buf = GpuBuffer::from_host(&ctx, &host_state).expect("state"); + let out_buf = GpuBuffer::::new(&ctx, CONV_CHANNELS).expect("out"); + + let kernel = CausalConv1dSiluKernel::new(CONV_CHANNELS as u32, CONV_KERNEL as u32); + + // Three sequential steps: the window must carry over on the device exactly as + // it does on the host, or step 2 and 3 diverge even if step 1 matched. + for (step, input) in steps.iter().enumerate() { + let want = reference(input, &mut host_state, &weight, CONV_KERNEL, CONV_CHANNELS); + + let in_buf = GpuBuffer::from_host(&ctx, input).expect("input"); + let mut args = [ + in_buf.as_ptr(), + state_buf.as_ptr(), + weight_buf.as_ptr(), + out_buf.as_ptr(), + ]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; CONV_CHANNELS]; + out_buf.copy_to_host(&mut got).expect("download output"); + assert_close( + &got, + &want, + 1e-3, + &format!("conv1d+silu output, step {step}"), + ); + + let mut got_state = vec![0.0f32; CONV_CHANNELS * (CONV_KERNEL - 1)]; + state_buf + .copy_to_host(&mut got_state) + .expect("download state"); + assert_close( + &got_state, + &host_state, + 1e-6, + &format!("conv1d window state after step {step}"), + ); + } + + // Keep the buffers alive to the end of the test. + drop(out_buf); + drop(state_buf); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/decode_attention.rs b/crates/aprender-gpu/src/kernels/gdn/decode_attention.rs new file mode 100644 index 0000000000..49818d5640 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/decode_attention.rs @@ -0,0 +1,607 @@ +//! PMAT-3477 / aprender#3090: single-query decode attention for `head_dim = 256`. +//! +//! The attention half of `forward_attention` in the CPU reference +//! (forward_qwen35.rs:1052), over a resident KV cache: +//! +//! ```text +//! group_size = num_heads / num_kv_heads +//! for h in 0..num_heads: +//! kv_h = h / group_size +//! for p in 0..seq_len: +//! scores[p] = dot(q_h, k_cache[p][kv_h]) / sqrt(head_dim) +//! softmax(scores) // max-subtracted, fp32 +//! out_h = sum_p scores[p] * v_cache[p][kv_h] +//! ``` +//! +//! ## Why a new kernel +//! +//! Every attention kernel under `kernels/attention/` hard-codes `head_dim <= 128` +//! (four floats per lane times a 32-lane warp: `incremental.rs:210`, +//! `flash_decoding/chunk_kernel.rs:90`) and **silently truncates** a 256-wide head. +//! Qwen3.5's gated full-attention layers are `head_dim = 256`, so they cannot use +//! them at all. +//! +//! ## Shape +//! +//! Grid `(num_heads, 1, 1)`, block `(256, 1, 1)`. `k_cache` and `v_cache` are +//! `[max_len][num_kv_heads * head_dim]` with row stride `num_kv_heads * head_dim`, +//! exactly as `Qwen35State::kv_cache` stores them; `seq_len` (the number of valid +//! positions, i.e. `position + 1`) is a kernel parameter, so one compiled module +//! serves the whole decode. `head_dim` must be at most the 256-thread block. +//! +//! ## Positions per pass +//! +//! The scores of a pass live in shared memory, so a pass covers at most +//! [`DEFAULT_MAX_POSITIONS_PER_PASS`] = 4096 positions (16 KiB of the 48 KiB static +//! shared budget). A longer context is processed in several passes with the running +//! max/sum rescaling of flash decoding — mathematically the same softmax, and the +//! pass loop is exercised by a test that sets the cap to 8 with `seq_len = 37`. + +use crate::kernels::gdn::emit_exp_f32; +use crate::kernels::Kernel; +use crate::ptx::builder::{KernelBuilder, PtxArithmetic, PtxComparison, PtxControl, PtxMemory}; +use crate::ptx::{PtxKernel, PtxReg, PtxType, VirtualReg}; + +/// Threads per block. +const BLOCK: u32 = 256; +/// Warps per block. +const WARPS: u32 = BLOCK / 32; +/// Positions whose scores fit in shared memory in one pass (16 KiB). +pub const DEFAULT_MAX_POSITIONS_PER_PASS: u32 = 4096; + +/// Which reduction a [`emit_block_reduce`] call performs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReduceOp { + Max, + Sum, +} + +/// Single-query decode attention over a resident KV cache, `head_dim` up to 256. +/// +/// For Qwen3.5-0.8B: `num_heads = 16`, `num_kv_heads = 2` or `4`, `head_dim = 256`. +#[derive(Debug, Clone, Copy)] +pub struct DecodeAttention256Kernel { + /// Query heads (`config.num_heads`). + pub num_heads: u32, + /// Key/value heads (`config.num_kv_heads`); `num_heads` must be a multiple of it. + pub num_kv_heads: u32, + /// Width of one head (256 for Qwen3.5); must be `<= 256`. + pub head_dim: u32, + /// Positions scored per pass — the shared scores buffer holds this many floats. + pub max_positions_per_pass: u32, +} + +impl DecodeAttention256Kernel { + /// Create the kernel with the default 4096-position pass. + /// + /// # Panics + /// If `head_dim` exceeds the 256-thread block or `num_kv_heads` does not divide + /// `num_heads` — both would silently produce wrong output. + #[must_use] + pub fn new(num_heads: u32, num_kv_heads: u32, head_dim: u32) -> Self { + assert!( + head_dim <= BLOCK, + "head_dim {head_dim} exceeds the {BLOCK}-thread block; one thread owns one output element" + ); + assert!( + num_kv_heads > 0 && num_heads % num_kv_heads == 0, + "num_heads {num_heads} must be a multiple of num_kv_heads {num_kv_heads}" + ); + Self { + num_heads, + num_kv_heads, + head_dim, + max_positions_per_pass: DEFAULT_MAX_POSITIONS_PER_PASS, + } + } + + /// Override the positions-per-pass cap (the shared scores buffer size). + #[must_use] + pub const fn with_max_positions_per_pass(mut self, cap: u32) -> Self { + self.max_positions_per_pass = cap; + self + } + + /// GQA group size — how many query heads share one KV head. + #[must_use] + pub const fn group_size(&self) -> u32 { + self.num_heads / self.num_kv_heads + } + + /// Static shared memory: the scores buffer plus two reduction scratch rows. + #[must_use] + pub const fn shared_bytes(&self) -> usize { + (self.max_positions_per_pass * 4 + WARPS * 4 * 2) as usize + } + + /// Launch grid — one block per query head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads, 1, 1) + } + + /// Launch block — 256 threads. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (BLOCK, 1, 1) + } +} + +/// Block-wide max or sum of `val`, broadcast to every thread. +/// +/// Warp shuffles, then one f32 per warp in shared memory at `scratch_off`, then every +/// thread folds the `WARPS` partials itself — which avoids a second barrier and a +/// broadcast slot. `label_prefix` must be unique per call site: PTX labels are flat. +fn emit_block_reduce( + ctx: &mut KernelBuilder<'_>, + val: VirtualReg, + lane: VirtualReg, + warp: VirtualReg, + scratch_off: u32, + op: ReduceOp, + label_prefix: &str, +) -> VirtualReg { + let mut v = val; + for offset in [16u32, 8, 4, 2, 1] { + let shuffled = ctx.shfl_down_f32(v, offset, 0xFFFF_FFFF); + v = match op { + ReduceOp::Max => ctx.max_f32(v, shuffled), + ReduceOp::Sum => ctx.add_f32(v, shuffled), + }; + } + + let zero = ctx.mov_u32_imm(0); + let is_lane0 = ctx.setp_eq_u32(lane, zero); + let skip = format!("{label_prefix}_skip_partial_store"); + ctx.branch_if_not(is_lane0, &skip); + let warp_bytes = ctx.mul_u32(warp, 4); + let slot = ctx.add_u32(warp_bytes, scratch_off); + let slot64 = ctx.cvt_u64_u32(slot); + ctx.st_shared_f32(slot64, v); + ctx.label(&skip); + ctx.bar_sync(0); + + let mut acc: Option = None; + for w in 0..WARPS { + let off = ctx.mov_u32_imm(scratch_off + w * 4); + let off64 = ctx.cvt_u64_u32(off); + let partial = ctx.ld_shared_f32(off64); + acc = Some(match acc { + None => partial, + Some(a) => match op { + ReduceOp::Max => ctx.max_f32(a, partial), + ReduceOp::Sum => ctx.add_f32(a, partial), + }, + }); + } + acc.expect("WARPS > 0") +} + +impl Kernel for DecodeAttention256Kernel { + fn name(&self) -> &str { + "gdn_decode_attention" + } + + fn build_ptx(&self) -> PtxKernel { + let head_dim = self.head_dim; + let group_size = self.group_size(); + let cap = self.max_positions_per_pass; + let row_stride_bytes = self.num_kv_heads * head_dim * 4; + // The CPU divides by sqrt(head_dim); the same division, not a reciprocal. + let sqrt_head_dim = (head_dim as f32).sqrt(); + let scratch_max = cap * 4; + let scratch_sum = cap * 4 + WARPS * 4; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "q_ptr") // [num_heads * head_dim] + .param(PtxType::U64, "k_cache_ptr") // [max_len][num_kv_heads * head_dim] + .param(PtxType::U64, "v_cache_ptr") // [max_len][num_kv_heads * head_dim] + .param(PtxType::U64, "out_ptr") // [num_heads * head_dim] + .param(PtxType::U32, "seq_len") // valid positions, = position + 1 + .shared_memory(self.shared_bytes()) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let h = ctx.special_reg(PtxReg::CtaIdX); + let lane = ctx.and_u32_imm(tid, 31); + let warp = ctx.shr_u32_imm(tid, 5); + + let q_ptr = ctx.load_param_u64("q_ptr"); + let k_ptr = ctx.load_param_u64("k_cache_ptr"); + let v_ptr = ctx.load_param_u64("v_cache_ptr"); + let out_ptr = ctx.load_param_u64("out_ptr"); + let seq_len = ctx.load_param_u32("seq_len"); + + // An empty cache has no softmax: leave the output alone. + let zero_u32 = ctx.mov_u32_imm(0); + let has_positions = ctx.setp_lt_u32(zero_u32, seq_len); + ctx.branch_if_not(has_positions, "gdn_attn_exit"); + + // kv head of this query head, and its byte offset inside a cache row. + let kv_h = ctx.div_u32(h, group_size); + let kv_off = ctx.mul_wide_u32(kv_h, head_dim * 4); + let head_off = ctx.mul_wide_u32(h, head_dim * 4); + let q_base = ctx.add_u64(q_ptr, head_off); + let out_base = ctx.add_u64(out_ptr, head_off); + let k_head = ctx.add_u64(k_ptr, kv_off); + let v_head = ctx.add_u64(v_ptr, kv_off); + + let head_dim_r = ctx.mov_u32_imm(head_dim); + let cap_r = ctx.mov_u32_imm(cap); + let sqrt_hd = ctx.mov_f32_imm(sqrt_head_dim); + let four = ctx.mov_u32_imm(4); + let out_elem_off = ctx.mul_wide_u32_reg(tid, four); + + // Running softmax state, carried across passes (flash decoding). + let running_max = ctx.mov_f32_imm(f32::NEG_INFINITY); + let running_sum = ctx.mov_f32_imm(0.0); + let acc = ctx.mov_f32_imm(0.0); + let chunk = ctx.mov_u32_imm(0); + + ctx.label("gdn_attn_chunk_loop"); + let more_chunks = ctx.setp_lt_u32(chunk, seq_len); + ctx.branch_if_not(more_chunks, "gdn_attn_chunk_end"); + // n_c = min(cap, seq_len - chunk) + let remaining = ctx.sub_u32_reg(seq_len, chunk); + let n_c = ctx.min_u32(remaining, cap_r); + + // ---- Phase 1: scores for this pass, one thread per few positions. + let local_max = ctx.mov_f32_imm(f32::NEG_INFINITY); + let pp = ctx.add_u32(tid, 0); + ctx.label("gdn_attn_score_loop"); + let score_go = ctx.setp_lt_u32(pp, n_c); + ctx.branch_if_not(score_go, "gdn_attn_score_end"); + let p = ctx.add_u32_reg(chunk, pp); + let row_off = ctx.mul_wide_u32(p, row_stride_bytes); + let k_row = ctx.add_u64(k_head, row_off); + + let dot = ctx.mov_f32_imm(0.0); + let i = ctx.mov_u32_imm(0); + ctx.label("gdn_attn_dot_loop"); + let dot_go = ctx.setp_lt_u32(i, head_dim_r); + ctx.branch_if_not(dot_go, "gdn_attn_dot_end"); + let elem_off = ctx.mul_wide_u32_reg(i, four); + let q_addr = ctx.add_u64(q_base, elem_off); + let k_addr = ctx.add_u64(k_row, elem_off); + let q_val = ctx.ld_global_f32(q_addr); + let k_val = ctx.ld_global_f32(k_addr); + // mul then add, like the CPU's `dot += q_h[i] * k_p[i]` — Rust does + // not contract into an fma, so neither do we. + let prod = ctx.mul_f32(q_val, k_val); + ctx.add_f32_inplace(dot, prod); + ctx.add_u32_inplace(i, 1); + ctx.branch("gdn_attn_dot_loop"); + ctx.label("gdn_attn_dot_end"); + + let score = ctx.div_f32(dot, sqrt_hd); + let score_slot = ctx.mul_u32(pp, 4); + let score_addr = ctx.cvt_u64_u32(score_slot); + ctx.st_shared_f32(score_addr, score); + ctx.max_f32_inplace(local_max, score); + ctx.add_u32_inplace(pp, BLOCK); + ctx.branch("gdn_attn_score_loop"); + ctx.label("gdn_attn_score_end"); + ctx.bar_sync(0); + + // ---- Phase 2: softmax over the pass, folded into the running state. + let chunk_max = + emit_block_reduce(ctx, local_max, lane, warp, scratch_max, ReduceOp::Max, "gdn_attn_max"); + let new_max = ctx.max_f32(running_max, chunk_max); + // correction = exp(old_max - new_max); on the first pass old_max is + // -inf, so this is 0 and the (still zero) accumulators are unaffected. + let max_delta = ctx.sub_f32(running_max, new_max); + let correction = emit_exp_f32(ctx, max_delta); + ctx.mov_f32_reg(running_max, new_max); + + let local_sum = ctx.mov_f32_imm(0.0); + let wp = ctx.add_u32(tid, 0); + ctx.label("gdn_attn_weight_loop"); + let weight_go = ctx.setp_lt_u32(wp, n_c); + ctx.branch_if_not(weight_go, "gdn_attn_weight_end"); + let w_slot = ctx.mul_u32(wp, 4); + let w_addr = ctx.cvt_u64_u32(w_slot); + let raw = ctx.ld_shared_f32(w_addr); + let shifted = ctx.sub_f32(raw, running_max); + let weight = emit_exp_f32(ctx, shifted); + ctx.st_shared_f32(w_addr, weight); + ctx.add_f32_inplace(local_sum, weight); + ctx.add_u32_inplace(wp, BLOCK); + ctx.branch("gdn_attn_weight_loop"); + ctx.label("gdn_attn_weight_end"); + ctx.bar_sync(0); + + let chunk_sum = + emit_block_reduce(ctx, local_sum, lane, warp, scratch_sum, ReduceOp::Sum, "gdn_attn_sum"); + ctx.mul_f32_inplace(running_sum, correction); + ctx.add_f32_inplace(running_sum, chunk_sum); + ctx.mul_f32_inplace(acc, correction); + + // ---- Phase 3: thread `tid` accumulates output element `tid`, + // positions ascending, exactly the CPU's accumulation order. + let in_head = ctx.setp_lt_u32(tid, head_dim_r); + ctx.branch_if_not(in_head, "gdn_attn_value_skip"); + let vp = ctx.mov_u32_imm(0); + ctx.label("gdn_attn_value_loop"); + let value_go = ctx.setp_lt_u32(vp, n_c); + ctx.branch_if_not(value_go, "gdn_attn_value_end"); + let v_slot = ctx.mul_u32(vp, 4); + let v_slot64 = ctx.cvt_u64_u32(v_slot); + let w_val = ctx.ld_shared_f32(v_slot64); + let vpos = ctx.add_u32_reg(chunk, vp); + let v_row_off = ctx.mul_wide_u32(vpos, row_stride_bytes); + let v_row = ctx.add_u64(v_head, v_row_off); + let v_addr = ctx.add_u64(v_row, out_elem_off); + let v_val = ctx.ld_global_f32(v_addr); + let contrib = ctx.mul_f32(w_val, v_val); + ctx.add_f32_inplace(acc, contrib); + ctx.add_u32_inplace(vp, 1); + ctx.branch("gdn_attn_value_loop"); + ctx.label("gdn_attn_value_end"); + ctx.label("gdn_attn_value_skip"); + + // The next pass overwrites the scores buffer. + ctx.bar_sync(0); + ctx.add_u32_reg_inplace(chunk, cap_r); + ctx.branch("gdn_attn_chunk_loop"); + ctx.label("gdn_attn_chunk_end"); + + // out = sum_p exp(s_p - max) * v_p / sum_p exp(s_p - max) + let in_head_out = ctx.setp_lt_u32(tid, head_dim_r); + ctx.branch_if_not(in_head_out, "gdn_attn_exit"); + let result = ctx.div_f32(acc, running_sum); + let out_addr = ctx.add_u64(out_base, out_elem_off); + ctx.st_global_f32(out_addr, result); + + ctx.label("gdn_attn_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_decode_attention_ptx_shape() { + let kernel = DecodeAttention256Kernel::new(16, 2, 256); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_decode_attention"), "{ptx}"); + // The 256-wide dot is a loop over head_dim, not four floats per lane: the + // truncation defect in kernels/attention is exactly a fixed lane budget. + assert!(ptx.contains("gdn_attn_dot_loop"), "{ptx}"); + assert!( + ptx.contains("bar.sync"), + "a block-wide softmax needs barriers:\n{ptx}" + ); + assert!(ptx.contains("shfl.sync.down.b32"), "{ptx}"); + assert_eq!(kernel.grid(), (16, 1, 1)); + assert_eq!(kernel.block(), (256, 1, 1)); + assert_eq!(kernel.group_size(), 8); + // 4096 scores + 8 warp maxes + 8 warp sums. + assert_eq!(kernel.shared_bytes(), 4096 * 4 + 64); + } + + #[test] + fn gdn_decode_attention_head_dim_over_block_is_refused() { + // A kernel that quietly handled only the first 256 elements would be the + // very defect this ticket exists to fix. + let refused = std::panic::catch_unwind(|| DecodeAttention256Kernel::new(16, 2, 512)); + assert!(refused.is_err(), "head_dim 512 must not be accepted"); + let refused_gqa = std::panic::catch_unwind(|| DecodeAttention256Kernel::new(16, 5, 256)); + assert!(refused_gqa.is_err(), "num_kv_heads must divide num_heads"); + } +} + +/// Device parity against a verbatim port of `forward_attention`'s attention block. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_decode_attention_device_tests { + use super::DecodeAttention256Kernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg}; + + /// `config.num_heads` of the Qwen3.5-0.8B attention layers. + const NUM_HEADS: usize = 16; + /// `attn_q_norm.len()`. + const HEAD_DIM: usize = 256; + /// Positions the test cache holds. + const MAX_LEN: usize = 64; + + /// Verbatim port of `aprender-serve`'s `gguf::ops::softmax` (ops.rs:348). + fn softmax(logits: &mut [f32]) { + let max_val = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let mut sum = 0.0f32; + for x in logits.iter_mut() { + *x = (*x - max_val).exp(); + sum += *x; + } + let inv_sum = 1.0 / sum; + for x in logits.iter_mut() { + *x *= inv_sum; + } + } + + /// Verbatim port of the attention block of `forward_attention` + /// (forward_qwen35.rs:1052), with `position = seq_len - 1`. + fn decode_attention( + q: &[f32], + k_cache: &[f32], + v_cache: &[f32], + seq_len: usize, + num_heads: usize, + num_kv_heads: usize, + head_dim: usize, + ) -> Vec { + let position = seq_len - 1; + let group_size = num_heads / num_kv_heads; + let mut attn_out_in = vec![0.0; num_heads * head_dim]; + for h in 0..num_heads { + let kv_h = h / group_size; + let q_h = &q[h * head_dim..(h + 1) * head_dim]; + + let mut scores = vec![0.0; position + 1]; + for (p, score) in scores.iter_mut().enumerate() { + let mut dot = 0.0; + let k_p = &k_cache[p * (num_kv_heads * head_dim) + kv_h * head_dim + ..p * (num_kv_heads * head_dim) + (kv_h + 1) * head_dim]; + for i in 0..head_dim { + dot += q_h[i] * k_p[i]; + } + *score = dot / (head_dim as f32).sqrt(); + } + softmax(&mut scores); + + let out_h = &mut attn_out_in[h * head_dim..(h + 1) * head_dim]; + for (p, &w) in scores.iter().enumerate() { + let v_p = &v_cache[p * (num_kv_heads * head_dim) + kv_h * head_dim + ..p * (num_kv_heads * head_dim) + (kv_h + 1) * head_dim]; + for i in 0..head_dim { + out_h[i] += w * v_p[i]; + } + } + } + attn_out_in + } + + /// One device run of `kernel` for `seq_len`, returned as a host vector. + fn run( + ctx: &CudaContext, + stream: &CudaStream, + kernel: &DecodeAttention256Kernel, + q: &[f32], + k_cache: &[f32], + v_cache: &[f32], + seq_len: usize, + ) -> Vec { + let q_buf = GpuBuffer::from_host(ctx, q).expect("q"); + let k_buf = GpuBuffer::from_host(ctx, k_cache).expect("k"); + let v_buf = GpuBuffer::from_host(ctx, v_cache).expect("v"); + let out_buf = GpuBuffer::::new(ctx, NUM_HEADS * HEAD_DIM).expect("out"); + let mut args = [ + q_buf.as_ptr(), + k_buf.as_ptr(), + v_buf.as_ptr(), + out_buf.as_ptr(), + seq_len as u64, + ]; + run_kernel( + ctx, + stream, + kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + let mut got = vec![0.0f32; NUM_HEADS * HEAD_DIM]; + out_buf.copy_to_host(&mut got).expect("download"); + got + } + + /// q, and a KV cache whose rows differ in scale so that a kernel reading the + /// wrong position or the wrong KV head lands somewhere else. + fn fixture(num_kv_heads: usize, seed: u32) -> (Vec, Vec, Vec) { + let mut rng = Lcg::new(seed); + let q = rng.vec(NUM_HEADS * HEAD_DIM, 0.1); + let row = num_kv_heads * HEAD_DIM; + let mut k = Vec::with_capacity(MAX_LEN * row); + let mut v = Vec::with_capacity(MAX_LEN * row); + for p in 0..MAX_LEN { + let scale = 0.05f32.mul_add(p as f32, 0.5); + for _ in 0..row { + k.push(rng.next_scaled(scale)); + } + for _ in 0..row { + v.push(rng.next_scaled(scale)); + } + } + (q, k, v) + } + + #[test] + fn gdn_decode_attention_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_decode_attention: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + for num_kv_heads in [2usize, 4] { + let (q, k, v) = fixture(num_kv_heads, 0x3477_0030 + num_kv_heads as u32); + let kernel = DecodeAttention256Kernel::new( + NUM_HEADS as u32, + num_kv_heads as u32, + HEAD_DIM as u32, + ); + for seq_len in [1usize, 5, 37] { + let want = decode_attention(&q, &k, &v, seq_len, NUM_HEADS, num_kv_heads, HEAD_DIM); + let got = run(&ctx, &stream, &kernel, &q, &k, &v, seq_len); + assert_close( + &got, + &want, + 1e-3, + &format!("decode attention, num_kv_heads {num_kv_heads}, seq_len {seq_len}"), + ); + } + } + } + + #[test] + fn gdn_decode_attention_multi_pass_matches_single_pass() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_decode_attention multi-pass: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let num_kv_heads = 2usize; + let (q, k, v) = fixture(num_kv_heads, 0x3477_0033); + let seq_len = 37; + let want = decode_attention(&q, &k, &v, seq_len, NUM_HEADS, num_kv_heads, HEAD_DIM); + + // A cap of 8 forces 5 passes over the same 37 positions, so the running + // max/sum rescaling is what is being measured, not the single-pass path. + let kernel = + DecodeAttention256Kernel::new(NUM_HEADS as u32, num_kv_heads as u32, HEAD_DIM as u32) + .with_max_positions_per_pass(8); + assert_eq!(kernel.shared_bytes(), 8 * 4 + 64); + let got = run(&ctx, &stream, &kernel, &q, &k, &v, seq_len); + assert_close(&got, &want, 1e-3, "decode attention across 5 passes"); + } + + #[test] + fn gdn_decode_attention_reads_the_right_kv_head() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_decode_attention kv head: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + // Two KV heads, one all +1 and one all -1 in V: with a single position the + // softmax weight is exactly 1, so out_h must be the sign of its own KV head. + let num_kv_heads = 2usize; + let row = num_kv_heads * HEAD_DIM; + let mut rng = Lcg::new(0x3477_0034); + let q = rng.vec(NUM_HEADS * HEAD_DIM, 0.1); + let k = vec![0.01f32; MAX_LEN * row]; + let mut v = vec![0.0f32; MAX_LEN * row]; + for p in 0..MAX_LEN { + for i in 0..HEAD_DIM { + v[p * row + i] = 1.0; + v[p * row + HEAD_DIM + i] = -1.0; + } + } + let kernel = + DecodeAttention256Kernel::new(NUM_HEADS as u32, num_kv_heads as u32, HEAD_DIM as u32); + let got = run(&ctx, &stream, &kernel, &q, &k, &v, 1); + for h in 0..NUM_HEADS { + let expected = if h < NUM_HEADS / 2 { 1.0 } else { -1.0 }; + for i in 0..HEAD_DIM { + assert!( + (got[h * HEAD_DIM + i] - expected).abs() < 1e-5, + "head {h} element {i} = {} but its KV head is {expected}", + got[h * HEAD_DIM + i] + ); + } + } + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/delta_rule.rs b/crates/aprender-gpu/src/kernels/gdn/delta_rule.rs new file mode 100644 index 0000000000..10ebae3c0c --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/delta_rule.rs @@ -0,0 +1,367 @@ +//! PMAT-3477: the gated delta-rule recurrence, one token. +//! +//! `delta_rule_recurrence` in the CPU reference, per value head `h` with the state +//! `S_h` stored transposed (`s_h[j * D + i] == S[i][j]`, so memory row `j` is column +//! `j` of `S`): +//! +//! ```text +//! 1. s_h *= exp(gate[h]) +//! 2. delta[j] = (v[j] - sum_i s_h[j*D+i] * k[i]) * beta[h] +//! 3. s_h[j*D+i] += k[i] * delta[j] +//! 4. out[j] = (sum_i s_h[j*D+i] * q[i]) * D^-0.5 +//! ``` +//! +//! Grid: `(num_v_heads, 1, 1)`, Block: `(head_v_dim, 1, 1)`. Thread `j` owns memory +//! row `j` of `S_h` **exclusively**: every one of steps 1–4 touches only row `j` for +//! output `j`, so the whole recurrence runs with no barrier and no cross-thread +//! dependency. Steps 1 and 2 are fused into one ascending pass over `i` and steps 3 +//! and 4 into a second, which keeps the fp32 accumulation order identical to the CPU +//! loop (`i = 0..D`) — the per-layer L∞ ≤ 1e-3 parity contract depends on it. + +use crate::kernels::gdn::emit_exp_f32; +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Gated delta-rule recurrence for a single token. +/// +/// For Qwen3.5-0.8B: `num_v_heads = 16`, `head_v_dim = 128`. +#[derive(Debug, Clone, Copy)] +pub struct DeltaRuleRecurrenceKernel { + /// Number of value heads. + pub num_v_heads: u32, + /// Value head width `D` (also the state's row and column count). + pub head_v_dim: u32, +} + +impl DeltaRuleRecurrenceKernel { + /// Create the kernel. + #[must_use] + pub const fn new(num_v_heads: u32, head_v_dim: u32) -> Self { + Self { + num_v_heads, + head_v_dim, + } + } + + /// Launch grid — one block per value head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_v_heads, 1, 1) + } + + /// Launch block — one thread per state row. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (self.head_v_dim, 1, 1) + } +} + +impl Kernel for DeltaRuleRecurrenceKernel { + fn name(&self) -> &str { + "gdn_delta_rule_recurrence" + } + + fn build_ptx(&self) -> PtxKernel { + let d = self.head_v_dim; + // The CPU computes `1.0 / (head_v_dim as f32).sqrt()` once; the same value is + // folded in here as an immediate so no rsqrt approximation enters the output. + let scale = 1.0 / (d as f32).sqrt(); + + PtxKernel::new(self.name()) + .param(PtxType::U64, "q_ptr") // [num_v_heads * D] + .param(PtxType::U64, "k_ptr") // [num_v_heads * D] + .param(PtxType::U64, "v_ptr") // [num_v_heads * D] + .param(PtxType::U64, "beta_ptr") // [num_v_heads] + .param(PtxType::U64, "gate_ptr") // [num_v_heads] (dt) + .param(PtxType::U64, "state_ptr") // [num_v_heads * D * D], updated in place + .param(PtxType::U64, "output_ptr") // [num_v_heads * D] + .shared_memory(0) + .build(|ctx| { + let j = ctx.special_reg(PtxReg::TidX); + let h = ctx.special_reg(PtxReg::CtaIdX); + + let d_r = ctx.mov_u32_imm(d); + let row_in_bounds = ctx.setp_lt_u32(j, d_r); + ctx.branch_if_not(row_in_bounds, "gdn_dr_exit"); + + let q_ptr = ctx.load_param_u64("q_ptr"); + let k_ptr = ctx.load_param_u64("k_ptr"); + let v_ptr = ctx.load_param_u64("v_ptr"); + let beta_ptr = ctx.load_param_u64("beta_ptr"); + let gate_ptr = ctx.load_param_u64("gate_ptr"); + let state_ptr = ctx.load_param_u64("state_ptr"); + let output_ptr = ctx.load_param_u64("output_ptr"); + + let four = ctx.mov_u32_imm(4); + let d_bytes = ctx.mov_u32_imm(d * 4); + + // q/k/v/out head base: h * D * 4 + let head_off = ctx.mul_wide_u32_reg(h, d_bytes); + let q_base = ctx.add_u64(q_ptr, head_off); + let k_base = ctx.add_u64(k_ptr, head_off); + let v_base = ctx.add_u64(v_ptr, head_off); + let out_base = ctx.add_u64(output_ptr, head_off); + + // state row base: (h * D * D + j * D) * 4 + let state_head_bytes = ctx.mov_u32_imm(d * d * 4); + let state_head_off = ctx.mul_wide_u32_reg(h, state_head_bytes); + let state_head = ctx.add_u64(state_ptr, state_head_off); + let row_off = ctx.mul_wide_u32_reg(j, d_bytes); + let s_row = ctx.add_u64(state_head, row_off); + + // Per-head scalars, read once. + let scalar_off = ctx.mul_wide_u32_reg(h, four); + let beta_addr = ctx.add_u64(beta_ptr, scalar_off); + let gate_addr = ctx.add_u64(gate_ptr, scalar_off); + let beta = ctx.ld_global_f32(beta_addr); + let gate = ctx.ld_global_f32(gate_addr); + let exp_gate = emit_exp_f32(ctx, gate); + + // Steps 1 + 2: decay row j in place and dot it with k, i ascending. + let sum = ctx.mov_f32_imm(0.0); + let i = ctx.mov_u32_imm(0); + ctx.label("gdn_dr_decay_loop"); + let go = ctx.setp_lt_u32(i, d_r); + ctx.branch_if_not(go, "gdn_dr_decay_end"); + let off = ctx.mul_wide_u32_reg(i, four); + let s_addr = ctx.add_u64(s_row, off); + let k_addr = ctx.add_u64(k_base, off); + let s_val = ctx.ld_global_f32(s_addr); + let s_scaled = ctx.mul_f32(s_val, exp_gate); + ctx.st_global_f32(s_addr, s_scaled); + let k_val = ctx.ld_global_f32(k_addr); + let prod = ctx.mul_f32(s_scaled, k_val); + ctx.add_f32_inplace(sum, prod); + ctx.add_u32_inplace(i, 1); + ctx.branch("gdn_dr_decay_loop"); + ctx.label("gdn_dr_decay_end"); + + // delta[j] = (v[j] - sum) * beta + let v_off = ctx.mul_wide_u32_reg(j, four); + let v_addr = ctx.add_u64(v_base, v_off); + let v_j = ctx.ld_global_f32(v_addr); + let diff = ctx.sub_f32(v_j, sum); + let delta = ctx.mul_f32(diff, beta); + + // Steps 3 + 4: update row j and dot it with q, i ascending. + let out_sum = ctx.mov_f32_imm(0.0); + let i2 = ctx.mov_u32_imm(0); + ctx.label("gdn_dr_update_loop"); + let go2 = ctx.setp_lt_u32(i2, d_r); + ctx.branch_if_not(go2, "gdn_dr_update_end"); + let off2 = ctx.mul_wide_u32_reg(i2, four); + let s_addr2 = ctx.add_u64(s_row, off2); + let k_addr2 = ctx.add_u64(k_base, off2); + let q_addr2 = ctx.add_u64(q_base, off2); + let s_old = ctx.ld_global_f32(s_addr2); + let k_val2 = ctx.ld_global_f32(k_addr2); + let upd = ctx.mul_f32(k_val2, delta); + let s_new = ctx.add_f32(s_old, upd); + ctx.st_global_f32(s_addr2, s_new); + let q_val = ctx.ld_global_f32(q_addr2); + let prod2 = ctx.mul_f32(s_new, q_val); + ctx.add_f32_inplace(out_sum, prod2); + ctx.add_u32_inplace(i2, 1); + ctx.branch("gdn_dr_update_loop"); + ctx.label("gdn_dr_update_end"); + + let scale_r = ctx.mov_f32_imm(scale); + let result = ctx.mul_f32(out_sum, scale_r); + let out_addr = ctx.add_u64(out_base, v_off); + ctx.st_global_f32(out_addr, result); + + ctx.label("gdn_dr_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_delta_rule_ptx_shape() { + let kernel = DeltaRuleRecurrenceKernel::new(16, 128); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_delta_rule_recurrence"), "{ptx}"); + // exp(gate) is the only transcendental in the recurrence. + assert_eq!(ptx.matches("ex2.approx.f32").count(), 1, "{ptx}"); + // No barrier: thread j owns state row j exclusively. + assert!(!ptx.contains("bar.sync"), "{ptx}"); + // D^-0.5 is a host-computed immediate, not an rsqrt. + assert!(!ptx.contains("rsqrt"), "{ptx}"); + assert_eq!(kernel.grid(), (16, 1, 1)); + assert_eq!(kernel.block(), (128, 1, 1)); + } +} + +/// Device parity against a verbatim port of `delta_rule_recurrence`. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_delta_rule_device_tests { + use super::DeltaRuleRecurrenceKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg, HEAD_DIM, NUM_V_HEADS}; + + /// Verbatim port of `aprender-serve`'s `delta_rule_recurrence` + /// (forward_qwen35.rs:121). + #[allow(clippy::too_many_arguments)] + fn delta_rule_recurrence( + q: &[f32], + k: &[f32], + v: &[f32], + beta: &[f32], + gate: &[f32], + state: &mut [f32], + output: &mut [f32], + num_v_heads: usize, + head_v_dim: usize, + ) { + let scale = 1.0 / (head_v_dim as f32).sqrt(); + + for h in 0..num_v_heads { + let q_h = &q[h * head_v_dim..(h + 1) * head_v_dim]; + let k_h = &k[h * head_v_dim..(h + 1) * head_v_dim]; + let v_h = &v[h * head_v_dim..(h + 1) * head_v_dim]; + let beta_val = beta[h]; + let gate_val = gate[h]; + + let state_offset = h * head_v_dim * head_v_dim; + let s_h = &mut state[state_offset..state_offset + head_v_dim * head_v_dim]; + + let exp_gate = gate_val.exp(); + for s in s_h.iter_mut() { + *s *= exp_gate; + } + + let mut delta = vec![0.0; head_v_dim]; + for j in 0..head_v_dim { + let row_j = &s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let mut sum = 0.0; + for i in 0..head_v_dim { + sum += row_j[i] * k_h[i]; + } + delta[j] = (v_h[j] - sum) * beta_val; + } + + for j in 0..head_v_dim { + let row_j = &mut s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let d_j = delta[j]; + for i in 0..head_v_dim { + row_j[i] += k_h[i] * d_j; + } + } + + for j in 0..head_v_dim { + let row_j = &s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let mut sum = 0.0; + for i in 0..head_v_dim { + sum += row_j[i] * q_h[i]; + } + output[h * head_v_dim + j] = sum * scale; + } + } + } + + /// q and k arrive L2-normalised per head, so reproduce that here. + fn l2_norm_per_head(x: &mut [f32], head_dim: usize) { + for head in x.chunks_exact_mut(head_dim) { + let sq: f32 = head.iter().map(|v| v * v).sum(); + let s = 1.0 / (sq + 1e-6).sqrt(); + for v in head.iter_mut() { + *v *= s; + } + } + } + + #[test] + fn gdn_delta_rule_recurrence_matches_cpu_reference_three_steps() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_delta_rule_recurrence: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let n = NUM_V_HEADS * HEAD_DIM; + let mut rng = Lcg::new(0x3477_0004); + let mut host_state = rng.vec(NUM_V_HEADS * HEAD_DIM * HEAD_DIM, 0.05); + let state_buf = GpuBuffer::from_host(&ctx, &host_state).expect("state"); + let out_buf = GpuBuffer::::new(&ctx, n).expect("out"); + let kernel = DeltaRuleRecurrenceKernel::new(NUM_V_HEADS as u32, HEAD_DIM as u32); + + for step in 0..3 { + let mut q = rng.vec(n, 1.0); + let mut k = rng.vec(n, 1.0); + l2_norm_per_head(&mut q, HEAD_DIM); + l2_norm_per_head(&mut k, HEAD_DIM); + let v = rng.vec(n, 1.0); + let beta: Vec = (0..NUM_V_HEADS).map(|_| 0.5 + 0.25 * rng.next()).collect(); + // dt = softplus(...) * ssm_a, with ssm_a negative: the state decays. + let gate: Vec = (0..NUM_V_HEADS) + .map(|_| -0.1 - 0.3 * rng.next().abs()) + .collect(); + + let mut want = vec![0.0f32; n]; + delta_rule_recurrence( + &q, + &k, + &v, + &beta, + &gate, + &mut host_state, + &mut want, + NUM_V_HEADS, + HEAD_DIM, + ); + + let q_buf = GpuBuffer::from_host(&ctx, &q).expect("q"); + let k_buf = GpuBuffer::from_host(&ctx, &k).expect("k"); + let v_buf = GpuBuffer::from_host(&ctx, &v).expect("v"); + let beta_buf = GpuBuffer::from_host(&ctx, &beta).expect("beta"); + let gate_buf = GpuBuffer::from_host(&ctx, &gate).expect("gate"); + let mut args = [ + q_buf.as_ptr(), + k_buf.as_ptr(), + v_buf.as_ptr(), + beta_buf.as_ptr(), + gate_buf.as_ptr(), + state_buf.as_ptr(), + out_buf.as_ptr(), + ]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; n]; + out_buf.copy_to_host(&mut got).expect("download output"); + assert_close( + &got, + &want, + 1e-3, + &format!("delta-rule output, step {step}"), + ); + + let mut got_state = vec![0.0f32; host_state.len()]; + state_buf + .copy_to_host(&mut got_state) + .expect("download state"); + assert_close( + &got_state, + &host_state, + 1e-3, + &format!("delta-rule state after step {step}"), + ); + + // The fixture must actually move the state, or the carry-over is untested. + let moved = got_state.iter().map(|s| s.abs()).fold(0.0f32, f32::max); + assert!(moved > 1e-3, "state is inert at step {step}"); + } + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/gated_rmsnorm.rs b/crates/aprender-gpu/src/kernels/gdn/gated_rmsnorm.rs new file mode 100644 index 0000000000..266df52143 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/gated_rmsnorm.rs @@ -0,0 +1,263 @@ +//! PMAT-3477: gated RMSNorm — the Gated `DeltaNet` output norm. +//! +//! `gated_rmsnorm` in the CPU reference. The **input** is normalised, the gate is +//! not: +//! +//! ```text +//! for each head chunk of head_dim: +//! rms_scale = 1 / sqrt(sum(input^2) / head_dim + eps) +//! out[i] = (input[i] * rms_scale * weight[i]) * silu(gate[i]) +//! ``` +//! +//! `weight` is `ssm_norm` and is `[head_dim]` — shared across heads, no head stride. +//! `gate` is `attn_gate . x` and *is* indexed per head. +//! +//! Grid: `(num_heads, 1, 1)`, Block: `(32, 1, 1)` — one warp per head. + +use crate::kernels::gdn::emit_silu_f32; +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Gated RMSNorm over `num_heads` chunks of `head_dim`. +/// +/// For Qwen3.5-0.8B: `head_dim = 128` (`head_v_dim`), `num_heads = 16`, `eps = 1e-6`. +#[derive(Debug, Clone, Copy)] +pub struct GatedRmsNormKernel { + /// Elements per head (`head_v_dim`). + pub head_dim: u32, + /// Number of heads (`num_v_heads`). + pub num_heads: u32, + /// Epsilon, added to the mean square. + pub eps: f32, +} + +impl GatedRmsNormKernel { + /// Create the kernel. + #[must_use] + pub const fn new(head_dim: u32, num_heads: u32, eps: f32) -> Self { + Self { + head_dim, + num_heads, + eps, + } + } + + /// Launch grid — one block per head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads, 1, 1) + } + + /// Launch block — one warp. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (32, 1, 1) + } +} + +impl Kernel for GatedRmsNormKernel { + fn name(&self) -> &str { + "gdn_gated_rmsnorm" + } + + fn build_ptx(&self) -> PtxKernel { + let head_dim = self.head_dim; + let eps = self.eps; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "input_ptr") // [num_heads * head_dim] + .param(PtxType::U64, "gate_ptr") // [num_heads * head_dim] + .param(PtxType::U64, "weight_ptr") // [head_dim], shared across heads + .param(PtxType::U64, "output_ptr") // [num_heads * head_dim] + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let head_idx = ctx.special_reg(PtxReg::CtaIdX); + + let input_ptr = ctx.load_param_u64("input_ptr"); + let gate_ptr = ctx.load_param_u64("gate_ptr"); + let weight_ptr = ctx.load_param_u64("weight_ptr"); + let output_ptr = ctx.load_param_u64("output_ptr"); + + let head_dim_r = ctx.mov_u32_imm(head_dim); + let four = ctx.mov_u32_imm(4); + let head_elems = ctx.mul_u32_reg(head_idx, head_dim_r); + let head_bytes = ctx.mul_wide_u32_reg(head_elems, four); + let in_base = ctx.add_u64(input_ptr, head_bytes); + let gate_base = ctx.add_u64(gate_ptr, head_bytes); + let out_base = ctx.add_u64(output_ptr, head_bytes); + + // Pass 1: sum of squares of the INPUT (the gate is not normalised). + let sq_sum = ctx.mov_f32_imm(0.0); + let idx = ctx.mov_u32_imm(0); + ctx.label("gdn_grms_sum_loop"); + let i = ctx.add_u32_reg(idx, tid); + let go = ctx.setp_lt_u32(i, head_dim_r); + ctx.branch_if_not(go, "gdn_grms_sum_end"); + let off = ctx.mul_wide_u32_reg(i, four); + let addr = ctx.add_u64(in_base, off); + let val = ctx.ld_global_f32(addr); + ctx.fma_f32_inplace(sq_sum, val, val); + ctx.add_u32_inplace(idx, 32); + ctx.branch("gdn_grms_sum_loop"); + ctx.label("gdn_grms_sum_end"); + + let s16 = ctx.shfl_down_f32(sq_sum, 16, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s16); + let s8 = ctx.shfl_down_f32(sq_sum, 8, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s8); + let s4 = ctx.shfl_down_f32(sq_sum, 4, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s4); + let s2 = ctx.shfl_down_f32(sq_sum, 2, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s2); + let s1 = ctx.shfl_down_f32(sq_sum, 1, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s1); + let total = ctx.shfl_idx_f32(sq_sum, 0, 0xFFFF_FFFF); + + // rms_scale = 1 / sqrt(sum / head_dim + eps) + let head_dim_f = ctx.cvt_f32_u32(head_dim_r); + let mean_sq = ctx.div_f32(total, head_dim_f); + let eps_r = ctx.mov_f32_imm(eps); + let denom = ctx.add_f32(mean_sq, eps_r); + let rms_scale = ctx.rsqrt_f32(denom); + + // Pass 2: out[i] = input[i] * rms_scale * weight[i] * silu(gate[i]) + let idx2 = ctx.mov_u32_imm(0); + ctx.label("gdn_grms_out_loop"); + let i2 = ctx.add_u32_reg(idx2, tid); + let go2 = ctx.setp_lt_u32(i2, head_dim_r); + ctx.branch_if_not(go2, "gdn_grms_exit"); + let off2 = ctx.mul_wide_u32_reg(i2, four); + let in_addr = ctx.add_u64(in_base, off2); + let gate_addr = ctx.add_u64(gate_base, off2); + // weight is [head_dim] — indexed by position within the head only. + let w_addr = ctx.add_u64(weight_ptr, off2); + let out_addr = ctx.add_u64(out_base, off2); + let x = ctx.ld_global_f32(in_addr); + let w = ctx.ld_global_f32(w_addr); + let g = ctx.ld_global_f32(gate_addr); + let normed = ctx.mul_f32(x, rms_scale); + let norm_v = ctx.mul_f32(normed, w); + let gate_act = emit_silu_f32(ctx, g); + let result = ctx.mul_f32(norm_v, gate_act); + ctx.st_global_f32(out_addr, result); + ctx.add_u32_inplace(idx2, 32); + ctx.branch("gdn_grms_out_loop"); + + ctx.label("gdn_grms_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_gated_rmsnorm_ptx_normalises_the_input_only() { + let kernel = GatedRmsNormKernel::new(128, 16, 1e-6); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_gated_rmsnorm"), "{ptx}"); + // Exactly one squared-accumulate site: the gate must not enter the norm. + assert_eq!(ptx.matches("fma.rn.f32").count(), 1, "{ptx}"); + // RMSNorm divides by head_dim before the rsqrt; plus silu's division. + assert_eq!(ptx.matches("div.rn.f32").count(), 2, "{ptx}"); + assert_eq!(ptx.matches("ex2.approx.f32").count(), 1, "{ptx}"); + assert_eq!(kernel.grid(), (16, 1, 1)); + } +} + +/// Device parity against a verbatim port of `gated_rmsnorm`. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_gated_rmsnorm_device_tests { + use super::GatedRmsNormKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{ + assert_close, run_kernel, Lcg, EPS, HEAD_DIM, NUM_V_HEADS, + }; + + /// Verbatim port of `silu` (forward_qwen35.rs:7). + fn silu(x: f32) -> f32 { + x / (1.0 + (-x).exp()) + } + + /// Verbatim port of `gated_rmsnorm` (forward_qwen35.rs:52). + fn gated_rmsnorm( + input: &[f32], + gate: &[f32], + weight: &[f32], + eps: f32, + head_v_dim: usize, + output: &mut [f32], + ) { + for (chunk_in, (chunk_gate, chunk_out)) in input.chunks_exact(head_v_dim).zip( + gate.chunks_exact(head_v_dim) + .zip(output.chunks_exact_mut(head_v_dim)), + ) { + let mut sq_sum = 0.0; + for &v in chunk_in { + sq_sum += v * v; + } + let rms_scale = 1.0 / ((sq_sum / head_v_dim as f32) + eps).sqrt(); + + for i in 0..head_v_dim { + let norm_v = chunk_in[i] * rms_scale * weight[i]; + chunk_out[i] = norm_v * silu(chunk_gate[i]); + } + } + } + + #[test] + fn gdn_gated_rmsnorm_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_gated_rmsnorm: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let n = NUM_V_HEADS * HEAD_DIM; + let mut rng = Lcg::new(0x3477_0005); + // Per-head magnitudes differ, so a norm taken over the whole vector instead + // of per head lands somewhere else. + let mut input = Vec::with_capacity(n); + for h in 0..NUM_V_HEADS { + let s = 0.2 * (h as f32 + 1.0); + for _ in 0..HEAD_DIM { + input.push(rng.next_scaled(s)); + } + } + let gate = rng.vec(n, 2.0); + let weight = rng.vec(HEAD_DIM, 1.0); + + let mut want = vec![0.0f32; n]; + gated_rmsnorm(&input, &gate, &weight, EPS, HEAD_DIM, &mut want); + + let in_buf = GpuBuffer::from_host(&ctx, &input).expect("input"); + let gate_buf = GpuBuffer::from_host(&ctx, &gate).expect("gate"); + let w_buf = GpuBuffer::from_host(&ctx, &weight).expect("weight"); + let out_buf = GpuBuffer::::new(&ctx, n).expect("out"); + + let kernel = GatedRmsNormKernel::new(HEAD_DIM as u32, NUM_V_HEADS as u32, EPS); + let mut args = [ + in_buf.as_ptr(), + gate_buf.as_ptr(), + w_buf.as_ptr(), + out_buf.as_ptr(), + ]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; n]; + out_buf.copy_to_host(&mut got).expect("download"); + assert_close(&got, &want, 1e-3, "gated RMSNorm"); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/gdn_gates.rs b/crates/aprender-gpu/src/kernels/gdn/gdn_gates.rs new file mode 100644 index 0000000000..f50c461c00 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/gdn_gates.rs @@ -0,0 +1,209 @@ +//! PMAT-3477: the Gated `DeltaNet` per-head gates. +//! +//! `forward_deltanet` computes, for each of the `num_v_heads` heads: +//! +//! ```text +//! dt[h] = softplus(alpha_raw[h] + dt_bias[h]) * a[h] +//! beta[h] = sigmoid(beta_raw[h]) +//! ``` +//! +//! `alpha_raw` is `ssm_alpha . x`, `beta_raw` is `ssm_beta . x`, `dt_bias` is +//! `ssm_dt.bias` and `a` is `ssm_a`. `dt` is the gate the delta-rule recurrence +//! exponentiates; `beta` is its update rate. +//! +//! Grid: `(ceil(num_heads / 256), 1, 1)`, Block: `(256, 1, 1)` — one thread per head +//! (16 heads for Qwen3.5-0.8B, so a single partly-idle block: this runs once per layer +//! per token and is latency-, not occupancy-bound). + +use crate::kernels::gdn::{emit_sigmoid_f32, emit_softplus_f32, ELEMENTWISE_BLOCK}; +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Per-head `dt` and `beta` gates for one Gated `DeltaNet` layer. +#[derive(Debug, Clone, Copy)] +pub struct GdnGatesKernel { + /// Number of value heads (`num_v_heads`, 16 for Qwen3.5-0.8B). + pub num_heads: u32, +} + +impl GdnGatesKernel { + /// Create the kernel. + #[must_use] + pub const fn new(num_heads: u32) -> Self { + Self { num_heads } + } + + /// Launch grid. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads.div_ceil(ELEMENTWISE_BLOCK), 1, 1) + } + + /// Launch block. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (ELEMENTWISE_BLOCK, 1, 1) + } +} + +impl Kernel for GdnGatesKernel { + fn name(&self) -> &str { + "gdn_gates" + } + + fn build_ptx(&self) -> PtxKernel { + let num_heads = self.num_heads; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "alpha_ptr") // [num_heads] ssm_alpha . x + .param(PtxType::U64, "dt_bias_ptr") // [num_heads] ssm_dt.bias + .param(PtxType::U64, "a_ptr") // [num_heads] ssm_a + .param(PtxType::U64, "beta_raw_ptr") // [num_heads] ssm_beta . x + .param(PtxType::U64, "dt_ptr") // [num_heads] out + .param(PtxType::U64, "beta_ptr") // [num_heads] out + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let cta = ctx.special_reg(PtxReg::CtaIdX); + let block = ctx.mov_u32_imm(ELEMENTWISE_BLOCK); + let h = ctx.mad_lo_u32(cta, block, tid); + + let n = ctx.mov_u32_imm(num_heads); + let in_bounds = ctx.setp_lt_u32(h, n); + ctx.branch_if_not(in_bounds, "gdn_gates_exit"); + + let alpha_ptr = ctx.load_param_u64("alpha_ptr"); + let dt_bias_ptr = ctx.load_param_u64("dt_bias_ptr"); + let a_ptr = ctx.load_param_u64("a_ptr"); + let beta_raw_ptr = ctx.load_param_u64("beta_raw_ptr"); + let dt_ptr = ctx.load_param_u64("dt_ptr"); + let beta_ptr = ctx.load_param_u64("beta_ptr"); + + let four = ctx.mov_u32_imm(4); + let off = ctx.mul_wide_u32_reg(h, four); + + // dt[h] = softplus(alpha[h] + dt_bias[h]) * a[h] + let alpha_addr = ctx.add_u64(alpha_ptr, off); + let bias_addr = ctx.add_u64(dt_bias_ptr, off); + let a_addr = ctx.add_u64(a_ptr, off); + let alpha = ctx.ld_global_f32(alpha_addr); + let bias = ctx.ld_global_f32(bias_addr); + let a = ctx.ld_global_f32(a_addr); + let pre = ctx.add_f32(alpha, bias); + let sp = emit_softplus_f32(ctx, pre, "gdn_gates_dt"); + let dt = ctx.mul_f32(sp, a); + let dt_addr = ctx.add_u64(dt_ptr, off); + ctx.st_global_f32(dt_addr, dt); + + // beta[h] = sigmoid(beta_raw[h]) + let beta_raw_addr = ctx.add_u64(beta_raw_ptr, off); + let beta_raw = ctx.ld_global_f32(beta_raw_addr); + let beta = emit_sigmoid_f32(ctx, beta_raw); + let beta_addr = ctx.add_u64(beta_ptr, off); + ctx.st_global_f32(beta_addr, beta); + + ctx.label("gdn_gates_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_gates_ptx_has_softplus_branch_and_sigmoid() { + let ptx = GdnGatesKernel::new(16).emit_ptx(); + assert!(ptx.contains(".entry gdn_gates"), "{ptx}"); + assert!(ptx.contains("gdn_gates_dt_softplus_big:"), "{ptx}"); + // one ex2 for softplus, one for the sigmoid + assert_eq!(ptx.matches("ex2.approx.f32").count(), 2, "{ptx}"); + assert_eq!(ptx.matches("lg2.approx.f32").count(), 1, "{ptx}"); + assert_eq!(ptx.matches("st.global.f32").count(), 2, "{ptx}"); + } +} + +/// Device parity against a verbatim port of `forward_deltanet`'s gate arithmetic. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_gates_device_tests { + use super::GdnGatesKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg, NUM_V_HEADS}; + + /// Verbatim port of `fn_softplus` (forward_qwen35.rs:864, == `softplus` at :12). + fn softplus(x: f32) -> f32 { + if x > 20.0 { + x + } else { + (1.0 + x.exp()).ln() + } + } + + fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + (-x).exp()) + } + + #[test] + fn gdn_gates_match_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_gates: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let mut rng = Lcg::new(0x3477_0003); + let mut alpha = rng.vec(NUM_V_HEADS, 3.0); + let dt_bias = rng.vec(NUM_V_HEADS, 1.0); + // ssm_a is negative in the real model (the state decays); keep that sign. + let a: Vec = (0..NUM_V_HEADS).map(|_| -rng.next().abs() - 0.05).collect(); + let beta_raw = rng.vec(NUM_V_HEADS, 4.0); + // Exercise BOTH arms of the softplus branch: head 0 takes the x > 20 shortcut. + alpha[0] = 40.0; + alpha[1] = -30.0; + + let want_dt: Vec = (0..NUM_V_HEADS) + .map(|h| softplus(alpha[h] + dt_bias[h]) * a[h]) + .collect(); + let want_beta: Vec = beta_raw.iter().map(|&b| sigmoid(b)).collect(); + + let alpha_buf = GpuBuffer::from_host(&ctx, &alpha).expect("alpha"); + let bias_buf = GpuBuffer::from_host(&ctx, &dt_bias).expect("dt_bias"); + let a_buf = GpuBuffer::from_host(&ctx, &a).expect("a"); + let beta_raw_buf = GpuBuffer::from_host(&ctx, &beta_raw).expect("beta_raw"); + let dt_buf = GpuBuffer::::new(&ctx, NUM_V_HEADS).expect("dt"); + let beta_buf = GpuBuffer::::new(&ctx, NUM_V_HEADS).expect("beta"); + + let kernel = GdnGatesKernel::new(NUM_V_HEADS as u32); + let mut args = [ + alpha_buf.as_ptr(), + bias_buf.as_ptr(), + a_buf.as_ptr(), + beta_raw_buf.as_ptr(), + dt_buf.as_ptr(), + beta_buf.as_ptr(), + ]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got_dt = vec![0.0f32; NUM_V_HEADS]; + let mut got_beta = vec![0.0f32; NUM_V_HEADS]; + dt_buf.copy_to_host(&mut got_dt).expect("download dt"); + beta_buf.copy_to_host(&mut got_beta).expect("download beta"); + + assert_close(&got_dt, &want_dt, 1e-3, "dt = softplus(alpha + bias) * a"); + assert_close(&got_beta, &want_beta, 1e-3, "beta = sigmoid(beta_raw)"); + assert!( + want_dt[0].is_finite() && (got_dt[0] - want_dt[0]).abs() <= 1e-3, + "the x > 20 shortcut arm of softplus must be taken for head 0" + ); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/l2_norm.rs b/crates/aprender-gpu/src/kernels/gdn/l2_norm.rs new file mode 100644 index 0000000000..116727938b --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/l2_norm.rs @@ -0,0 +1,223 @@ +//! PMAT-3477: per-head L2 normalisation for Gated `DeltaNet` q/k. +//! +//! `l2_norm_per_head` in the CPU reference: each `head_dim`-wide head is scaled on +//! its own by `1 / sqrt(sum(x_h^2) + eps)`. The epsilon is added to the *sum of +//! squares*, not to a mean — this is llama.cpp's `build_gdn_l2_norm`, not RMSNorm. +//! One norm over all heads would scale every head by the other heads' magnitudes. +//! +//! Grid: `(num_heads, 1, 1)`, Block: `(32, 1, 1)` — one warp per head, in place. + +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Per-head L2 normalisation, in place. +/// +/// For Qwen3.5-0.8B: `head_dim = 128`, `num_heads = 16` (q and k alike), `eps = 1e-6`. +#[derive(Debug, Clone, Copy)] +pub struct PerHeadL2NormKernel { + /// Elements per head (`head_k_dim`). + pub head_dim: u32, + /// Number of heads (`num_k_heads`). + pub num_heads: u32, + /// Epsilon, added to the sum of squares. + pub eps: f32, +} + +impl PerHeadL2NormKernel { + /// Create the kernel. + #[must_use] + pub const fn new(head_dim: u32, num_heads: u32, eps: f32) -> Self { + Self { + head_dim, + num_heads, + eps, + } + } + + /// Launch grid — one block per head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads, 1, 1) + } + + /// Launch block — one warp. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (32, 1, 1) + } +} + +impl Kernel for PerHeadL2NormKernel { + fn name(&self) -> &str { + "gdn_per_head_l2_norm" + } + + fn build_ptx(&self) -> PtxKernel { + let head_dim = self.head_dim; + let eps = self.eps; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "x_ptr") // [num_heads * head_dim], normalised in place + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let head_idx = ctx.special_reg(PtxReg::CtaIdX); + let x_ptr = ctx.load_param_u64("x_ptr"); + + let head_dim_r = ctx.mov_u32_imm(head_dim); + let four = ctx.mov_u32_imm(4); + let head_elems = ctx.mul_u32_reg(head_idx, head_dim_r); + let head_bytes = ctx.mul_wide_u32_reg(head_elems, four); + let head_base = ctx.add_u64(x_ptr, head_bytes); + + // Pass 1: sum of squares over this head, warp-strided. + let sq_sum = ctx.mov_f32_imm(0.0); + let idx = ctx.mov_u32_imm(0); + ctx.label("gdn_l2_sum_loop"); + let i = ctx.add_u32_reg(idx, tid); + let in_bounds = ctx.setp_lt_u32(i, head_dim_r); + ctx.branch_if_not(in_bounds, "gdn_l2_sum_end"); + let off = ctx.mul_wide_u32_reg(i, four); + let addr = ctx.add_u64(head_base, off); + let val = ctx.ld_global_f32(addr); + ctx.fma_f32_inplace(sq_sum, val, val); + ctx.add_u32_inplace(idx, 32); + ctx.branch("gdn_l2_sum_loop"); + ctx.label("gdn_l2_sum_end"); + + let s16 = ctx.shfl_down_f32(sq_sum, 16, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s16); + let s8 = ctx.shfl_down_f32(sq_sum, 8, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s8); + let s4 = ctx.shfl_down_f32(sq_sum, 4, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s4); + let s2 = ctx.shfl_down_f32(sq_sum, 2, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s2); + let s1 = ctx.shfl_down_f32(sq_sum, 1, 0xFFFF_FFFF); + ctx.add_f32_inplace(sq_sum, s1); + let total = ctx.shfl_idx_f32(sq_sum, 0, 0xFFFF_FFFF); + + // scale = 1 / sqrt(sum + eps) — eps on the SUM, as l2_norm does. + let eps_r = ctx.mov_f32_imm(eps); + let denom = ctx.add_f32(total, eps_r); + let scale = ctx.rsqrt_f32(denom); + + // Pass 2: scale in place. + let idx2 = ctx.mov_u32_imm(0); + ctx.label("gdn_l2_scale_loop"); + let i2 = ctx.add_u32_reg(idx2, tid); + let in_bounds2 = ctx.setp_lt_u32(i2, head_dim_r); + ctx.branch_if_not(in_bounds2, "gdn_l2_exit"); + let off2 = ctx.mul_wide_u32_reg(i2, four); + let addr2 = ctx.add_u64(head_base, off2); + let v = ctx.ld_global_f32(addr2); + let scaled = ctx.mul_f32(v, scale); + ctx.st_global_f32(addr2, scaled); + ctx.add_u32_inplace(idx2, 32); + ctx.branch("gdn_l2_scale_loop"); + + ctx.label("gdn_l2_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_l2_norm_ptx_divides_by_no_count() { + let ptx = PerHeadL2NormKernel::new(128, 16, 1e-6).emit_ptx(); + assert!(ptx.contains(".entry gdn_per_head_l2_norm"), "{ptx}"); + // L2 norm, not RMSNorm: no division by head_dim before the rsqrt. + assert!( + !ptx.contains("div.rn.f32"), + "eps goes on the sum of squares; a mean divide would be RMSNorm:\n{ptx}" + ); + assert!(ptx.contains("rsqrt.approx.f32"), "{ptx}"); + } +} + +/// Device parity against a verbatim port of `l2_norm_per_head`. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_l2_norm_device_tests { + use super::PerHeadL2NormKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{ + assert_close, run_kernel, Lcg, EPS, HEAD_DIM, NUM_K_HEADS, + }; + + /// Verbatim port of `aprender-serve`'s `l2_norm` (forward_qwen35.rs:21). + fn l2_norm(x: &mut [f32], eps: f32) { + let mut sq_sum = 0.0; + for &v in x.iter() { + sq_sum += v * v; + } + let scale = 1.0 / (sq_sum + eps).sqrt(); + for v in x.iter_mut() { + *v *= scale; + } + } + + /// Verbatim port of `l2_norm_per_head` (forward_qwen35.rs:36). + fn l2_norm_per_head(x: &mut [f32], head_dim: usize, eps: f32) { + for head in x.chunks_exact_mut(head_dim) { + l2_norm(head, eps); + } + } + + #[test] + fn gdn_per_head_l2_norm_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_per_head_l2_norm: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let mut rng = Lcg::new(0x3477_0002); + // Heads deliberately differ in magnitude: a single norm over the whole + // vector (the defect this kernel exists to avoid) lands elsewhere. + let mut host: Vec = Vec::with_capacity(HEAD_DIM * NUM_K_HEADS); + for h in 0..NUM_K_HEADS { + let scale = 0.25 * (h as f32 + 1.0); + for _ in 0..HEAD_DIM { + host.push(rng.next_scaled(scale)); + } + } + let mut want = host.clone(); + l2_norm_per_head(&mut want, HEAD_DIM, EPS); + + let buf = GpuBuffer::from_host(&ctx, &host).expect("x"); + let kernel = PerHeadL2NormKernel::new(HEAD_DIM as u32, NUM_K_HEADS as u32, EPS); + let mut args = [buf.as_ptr()]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; host.len()]; + buf.copy_to_host(&mut got).expect("download"); + assert_close(&got, &want, 1e-4, "per-head L2 norm"); + + // Each head must be unit length on its own. + for h in 0..NUM_K_HEADS { + let norm: f32 = got[h * HEAD_DIM..(h + 1) * HEAD_DIM] + .iter() + .map(|v| v * v) + .sum::() + .sqrt(); + assert!( + (norm - 1.0).abs() < 1e-4, + "head {h} has L2 norm {norm}, not 1 — a global norm would leave the \ + heads at different lengths" + ); + } + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/mod.rs b/crates/aprender-gpu/src/kernels/gdn/mod.rs new file mode 100644 index 0000000000..9afd1eb664 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/mod.rs @@ -0,0 +1,184 @@ +//! PMAT-3477 / aprender#3090: device kernels for Qwen3.5's Gated `DeltaNet` block. +//! +//! The CPU reference in `aprender-serve` +//! (`gguf/inference/forward/forward_qwen35.rs`) is the specification; every kernel +//! here reproduces one of its functions element for element, and every test in this +//! module compares against a *verbatim port* of that function (copied, not imported — +//! `aprender-gpu` does not depend on `aprender-serve`). +//! +//! | kernel | CPU function it implements | +//! |--------|----------------------------| +//! | [`CausalConv1dSiluKernel`] | `causal_conv1d` + the `silu` loop that immediately follows it | +//! | [`PerHeadL2NormKernel`] | `l2_norm_per_head` | +//! | [`GdnGatesKernel`] | `dt[h] = softplus(alpha[h] + dt_bias[h]) * a[h]`, `beta[h] = sigmoid(beta_raw[h])` | +//! | [`DeltaRuleRecurrenceKernel`] | `delta_rule_recurrence` | +//! | [`GatedRmsNormKernel`] | `gated_rmsnorm` | +//! | [`SigmoidGateKernel`] | `apply_sigmoid_gate` | +//! +//! Qwen3.5 interleaves those Gated `DeltaNet` layers with *gated full-attention* +//! layers, whose 256-wide heads no attention kernel in this crate could serve +//! (`kernels/attention` truncates at `head_dim = 128`). Their three kernels live here +//! too, against the same CPU reference (`forward_attention`, forward_qwen35.rs:985): +//! +//! | kernel | CPU function it implements | +//! |--------|----------------------------| +//! | [`SplitInterleavedKernel`] | the `q` / `gate` de-interleave of the joint `attn_q` projection | +//! | [`PartialNeoxRopeKernel`] | `apply_partial_neox_rope` | +//! | [`DecodeAttention256Kernel`] | the scores / `softmax` / value accumulation block | +//! +//! ## Shapes (Qwen3.5-0.8B) +//! +//! `head_k_dim = head_v_dim = 128`, `num_k_heads = 16`, `num_v_heads = 16`, +//! `conv_kernel = 4`, so the conv channel count is +//! `head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads = 6144` (`3 * inner_size`), +//! exactly as `forward_deltanet` computes `conv_dim`. +//! +//! ## Numerics +//! +//! `exp` is `ex2.approx.f32` after a `log2(e)` scale and `ln` is `lg2.approx.f32` +//! after a `ln(2)` scale; both carry ~2 ulp of error, which is why every kernel that +//! goes through them (conv+SiLU, gates, recurrence, gated RMSNorm, sigmoid gate) is +//! checked at 1e-3 instead of 1e-4. Reduction order inside a head is kept ascending +//! (`i = 0..D`) to match the CPU loop, so the per-layer L∞ ≤ 1e-3 parity contract for +//! the Gated `DeltaNet` block holds against the reference implementation. + +mod causal_conv1d; +mod decode_attention; +mod delta_rule; +mod gated_rmsnorm; +mod gdn_gates; +mod l2_norm; +mod partial_rope; +mod sigmoid_gate; +mod split_interleave; + +#[cfg(test)] +mod test_support; + +pub use causal_conv1d::CausalConv1dSiluKernel; +pub use decode_attention::{DecodeAttention256Kernel, DEFAULT_MAX_POSITIONS_PER_PASS}; +pub use delta_rule::DeltaRuleRecurrenceKernel; +pub use gated_rmsnorm::GatedRmsNormKernel; +pub use gdn_gates::GdnGatesKernel; +pub use l2_norm::PerHeadL2NormKernel; +pub use partial_rope::PartialNeoxRopeKernel; +pub use sigmoid_gate::SigmoidGateKernel; +pub use split_interleave::SplitInterleavedKernel; + +use crate::ptx::builder::{KernelBuilder, PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::VirtualReg; + +/// Threads per block for the elementwise kernels in this module. +pub(crate) const ELEMENTWISE_BLOCK: u32 = 256; + +/// `exp(x)` as `ex2.approx.f32(x * log2(e))`. +/// +/// ~2 ulp. The CPU reference uses `f32::exp`; the difference is well inside the +/// 1e-3 tolerance the Gated `DeltaNet` parity contract allows. +pub(crate) fn emit_exp_f32(ctx: &mut KernelBuilder<'_>, x: VirtualReg) -> VirtualReg { + let log2e = ctx.mov_f32_imm(std::f32::consts::LOG2_E); + let scaled = ctx.mul_f32(x, log2e); + ctx.ex2_f32(scaled) +} + +/// `sigmoid(x) = 1 / (1 + exp(-x))` — the CPU expression, operation for operation. +pub(crate) fn emit_sigmoid_f32(ctx: &mut KernelBuilder<'_>, x: VirtualReg) -> VirtualReg { + let neg = ctx.neg_f32(x); + let e = emit_exp_f32(ctx, neg); + let one = ctx.mov_f32_imm(1.0); + let den = ctx.add_f32(one, e); + ctx.div_f32(one, den) +} + +/// `silu(x) = x / (1 + exp(-x))` — the CPU expression, operation for operation. +pub(crate) fn emit_silu_f32(ctx: &mut KernelBuilder<'_>, x: VirtualReg) -> VirtualReg { + let neg = ctx.neg_f32(x); + let e = emit_exp_f32(ctx, neg); + let one = ctx.mov_f32_imm(1.0); + let den = ctx.add_f32(one, e); + ctx.div_f32(x, den) +} + +/// `softplus(x) = if x > 20 { x } else { ln(1 + exp(x)) }`, branch and threshold +/// exactly as the CPU reference (`forward_qwen35.rs`'s `fn_softplus`). +/// +/// `ln(y)` is `lg2.approx.f32(y) * ln(2)`. `label_prefix` must be unique within the +/// kernel: the builder emits flat PTX labels, so two softplus sites in one kernel +/// would otherwise collide. +pub(crate) fn emit_softplus_f32( + ctx: &mut KernelBuilder<'_>, + x: VirtualReg, + label_prefix: &str, +) -> VirtualReg { + let big_label = format!("{label_prefix}_softplus_big"); + let end_label = format!("{label_prefix}_softplus_end"); + + let out = ctx.mov_f32_imm(0.0); + let threshold = ctx.mov_f32_imm(20.0); + let is_big = ctx.setp_gt_f32(x, threshold); + ctx.branch_if(is_big, &big_label); + + // ln(1 + exp(x)) + let e = emit_exp_f32(ctx, x); + let one = ctx.mov_f32_imm(1.0); + let arg = ctx.add_f32(one, e); + let log2_arg = ctx.lg2_f32(arg); + let ln2 = ctx.mov_f32_imm(std::f32::consts::LN_2); + let small = ctx.mul_f32(log2_arg, ln2); + ctx.mov_f32_reg(out, small); + ctx.branch(&end_label); + + ctx.label(&big_label); + ctx.mov_f32_reg(out, x); + + ctx.label(&end_label); + out +} + +#[cfg(test)] +mod helper_ptx_tests { + use super::*; + use crate::kernels::Kernel; + + /// A throwaway kernel that exercises every helper, so the PTX shape of the + /// transcendentals is asserted even on a host with no CUDA device. + struct HelperProbe; + + impl Kernel for HelperProbe { + fn name(&self) -> &str { + "gdn_helper_probe" + } + + fn build_ptx(&self) -> crate::ptx::PtxKernel { + crate::ptx::PtxKernel::new(self.name()) + .param(crate::ptx::PtxType::U64, "x_ptr") + .shared_memory(0) + .build(|ctx| { + let ptr = ctx.load_param_u64("x_ptr"); + let x = ctx.ld_global_f32(ptr); + let s = emit_silu_f32(ctx, x); + let g = emit_sigmoid_f32(ctx, s); + let p = emit_softplus_f32(ctx, g, "probe"); + ctx.st_global_f32(ptr, p); + ctx.ret(); + }) + } + } + + #[test] + fn gdn_helpers_emit_ex2_and_lg2() { + let ptx = HelperProbe.emit_ptx(); + assert!( + ptx.contains("ex2.approx.f32"), + "exp must lower to ex2.approx.f32:\n{ptx}" + ); + assert!( + ptx.contains("lg2.approx.f32"), + "ln must lower to lg2.approx.f32:\n{ptx}" + ); + // softplus branches on x > 20 and both arms write the same register. + assert!(ptx.contains("probe_softplus_big:"), "{ptx}"); + assert!(ptx.contains("probe_softplus_end:"), "{ptx}"); + assert!(ptx.contains("setp.gt.f32"), "{ptx}"); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/partial_rope.rs b/crates/aprender-gpu/src/kernels/gdn/partial_rope.rs new file mode 100644 index 0000000000..81d81bf11d --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/partial_rope.rs @@ -0,0 +1,399 @@ +//! PMAT-3477 / aprender#3090: partial NEOX RoPE for the Qwen3.5 attention heads. +//! +//! `apply_partial_neox_rope` in the CPU reference (forward_qwen35.rs:191), which is +//! llama.cpp's `ggml_rope_multi` for text input: +//! +//! ```text +//! half = n_rot / 2; theta_scale = freq_base^(-2/n_rot) +//! for h in heads: +//! theta = pos +//! for j in 0..half: +//! (sin, cos) = theta.sin_cos() +//! (a, b) = (x[h*head_dim + j], x[h*head_dim + j + half]) +//! x[h*head_dim + j] = a * cos - b * sin +//! x[h*head_dim + j + half] = a * sin + b * cos +//! theta *= theta_scale +//! ``` +//! +//! Dimensions `n_rot..head_dim` are **not** rotated. The existing +//! `elementwise/rope/neox.rs` rotates the whole head, which is the defect this kernel +//! exists to avoid: for Qwen3.5 `n_rot = 2 * sum(rope.dimension_sections) = 128` of a +//! 256-wide head, so a full rotation corrupts the upper half. +//! +//! Grid: `(num_heads, 1, 1)`, Block: `(half, 1, 1)` — one thread per (head, pair), in +//! place. +//! +//! ## Numerics: why `theta_scale`, and not `freq_base`, is the parameter +//! +//! Thread `j` reproduces the CPU's *iterative* `theta` by multiplying `theta_scale` +//! into `pos` exactly `j` times, in the CPU's order. That is bit-identical to the +//! reference **only if `theta_scale` itself is bit-identical**, and it has to be: +//! the error in `theta` is amplified by `j * theta_j`, which peaks at 2556 for +//! `pos = 1000, freq_base = 1e4`. So a single ulp (6e-8 relative) in `theta_scale` +//! is already 1.5e-4 rad of `theta` — more than the entire 1.4e-4 tolerance. This +//! was measured, not assumed: computing `freq_base^(-2/n_rot)` on the device from +//! `lg2.approx` + a polynomial `exp2` (the best the ISA offers, ~1e-7 relative) +//! missed the CPU by 2.3e-4 at that position. +//! +//! `freq_base^(-2/n_rot)` is one scalar per layer, and the CPU computes it once with +//! `f32::powf`. So the kernel takes that scalar; [`PartialNeoxRopeKernel::theta_scale`] +//! computes it the CPU's way, and callers must use it rather than rolling their own. +//! +//! What remains is `sin`/`cos`. `sin.approx.f32`/`cos.approx.f32` are accurate to +//! ~2^-21 **only for arguments in `[-pi, pi]`**, and `theta` reaches `pos` +//! (thousands). A Cody-Waite reduction with a two-word `2*pi` (`6.28125` plus the +//! remainder — the head is exact for any `n < 2^16`, i.e. positions below ~411k) +//! brings the argument into range with ~3e-8 rad of error before the hardware sees +//! it, leaving ~2e-7 in the rotated values. + +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType, VirtualReg}; + +/// High word of `2*pi`: exactly representable with 8 significant bits, so `n * HI` is +/// exact for every `n < 2^16`. +const TWO_PI_HI: f32 = 6.281_25; +/// Low word of `2*pi`: `2*pi - TWO_PI_HI`, rounded to f32. +const TWO_PI_LO: f32 = 0.001_935_307_2; + +/// Partial NEOX RoPE over the first `n_rot` dimensions of each head, in place. +/// +/// For Qwen3.5-0.8B: `num_heads = 16` (or `num_kv_heads` when rotating K), +/// `head_dim = 256`, `n_rot = 128`. +#[derive(Debug, Clone, Copy)] +pub struct PartialNeoxRopeKernel { + /// Number of heads in the buffer (`num_heads` for Q, `num_kv_heads` for K). + pub num_heads: u32, + /// Width of one head (256 for Qwen3.5). + pub head_dim: u32, + /// Rotated prefix of each head: `2 * sum(rope.dimension_sections)` (128). + pub n_rot: u32, +} + +impl PartialNeoxRopeKernel { + /// Create the kernel. + #[must_use] + pub const fn new(num_heads: u32, head_dim: u32, n_rot: u32) -> Self { + Self { + num_heads, + head_dim, + n_rot, + } + } + + /// Number of rotated pairs per head (`n_rot / 2`). + #[must_use] + pub const fn half(&self) -> u32 { + self.n_rot / 2 + } + + /// The `theta_scale` kernel argument for a given `freq_base` (`config.rope_theta`). + /// + /// `freq_base.powf(-2.0 / n_rot as f32)`, the CPU reference's expression + /// character for character. It must be *this* value: see the module docs — one + /// ulp here is more than the whole parity budget at position 1000. + #[must_use] + pub fn theta_scale(&self, freq_base: f32) -> f32 { + freq_base.powf(-2.0 / self.n_rot as f32) + } + + /// Launch grid — one block per head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads, 1, 1) + } + + /// Launch block — one thread per rotated pair. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (self.half(), 1, 1) + } +} + +/// `(sin(theta), cos(theta))` with a Cody-Waite reduction of `theta` into `[-pi, pi]` +/// before the hardware approximations, which are only accurate in that range. +fn emit_sin_cos( + ctx: &mut crate::ptx::builder::KernelBuilder<'_>, + theta: VirtualReg, +) -> (VirtualReg, VirtualReg) { + let inv_two_pi = ctx.mov_f32_imm(1.0 / std::f32::consts::TAU); + let half = ctx.mov_f32_imm(0.5); + let scaled = ctx.mul_f32(theta, inv_two_pi); + let biased = ctx.add_f32(scaled, half); + let n = ctx.floor_f32(biased); + + let neg_hi = ctx.mov_f32_imm(-TWO_PI_HI); + let neg_lo = ctx.mov_f32_imm(-TWO_PI_LO); + let t1 = ctx.fma_f32(n, neg_hi, theta); + let reduced = ctx.fma_f32(n, neg_lo, t1); + + let sin = ctx.sin_f32(reduced); + let cos = ctx.cos_f32(reduced); + (sin, cos) +} + +impl Kernel for PartialNeoxRopeKernel { + fn name(&self) -> &str { + "gdn_partial_neox_rope" + } + + fn build_ptx(&self) -> PtxKernel { + let head_dim = self.head_dim; + let half = self.half(); + + PtxKernel::new(self.name()) + .param(PtxType::U64, "x_ptr") // [num_heads * head_dim], rotated in place + .param(PtxType::U32, "position") + .param(PtxType::F32, "theta_scale") // freq_base^(-2/n_rot), from the host + .shared_memory(0) + .build(|ctx| { + let j = ctx.special_reg(PtxReg::TidX); + let h = ctx.special_reg(PtxReg::CtaIdX); + + let half_r = ctx.mov_u32_imm(half); + let in_bounds = ctx.setp_lt_u32(j, half_r); + ctx.branch_if_not(in_bounds, "gdn_rope_exit"); + + let x_ptr = ctx.load_param_u64("x_ptr"); + let position = ctx.load_param_u32("position"); + let theta_scale = ctx.load_param_f32("theta_scale"); + + // theta = pos * theta_scale^j, by j multiplications in the CPU's order. + let theta = ctx.cvt_f32_u32(position); + let step = ctx.mov_u32_imm(0); + ctx.label("gdn_rope_theta_loop"); + let more = ctx.setp_lt_u32(step, j); + ctx.branch_if_not(more, "gdn_rope_theta_end"); + ctx.mul_f32_inplace(theta, theta_scale); + ctx.add_u32_inplace(step, 1); + ctx.branch("gdn_rope_theta_loop"); + ctx.label("gdn_rope_theta_end"); + + let (sin, cos) = emit_sin_cos(ctx, theta); + + // a at h*head_dim + j, b at h*head_dim + j + half. + let head_off = ctx.mul_wide_u32(h, head_dim * 4); + let head_base = ctx.add_u64(x_ptr, head_off); + let a_off = ctx.mul_wide_u32(j, 4); + let a_addr = ctx.add_u64(head_base, a_off); + let b_delta = ctx.mov_u64_imm(u64::from(half) * 4); + let b_addr = ctx.add_u64(a_addr, b_delta); + let a = ctx.ld_global_f32(a_addr); + let b = ctx.ld_global_f32(b_addr); + + // Exactly the CPU's two expressions: separate mul/mul/sub and + // mul/mul/add, not fma, so the rounding matches term for term. + let a_cos = ctx.mul_f32(a, cos); + let b_sin = ctx.mul_f32(b, sin); + let rot_a = ctx.sub_f32(a_cos, b_sin); + let a_sin = ctx.mul_f32(a, sin); + let b_cos = ctx.mul_f32(b, cos); + let rot_b = ctx.add_f32(a_sin, b_cos); + ctx.st_global_f32(a_addr, rot_a); + ctx.st_global_f32(b_addr, rot_b); + + ctx.label("gdn_rope_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_partial_rope_ptx_reduces_before_the_hardware_sine() { + let kernel = PartialNeoxRopeKernel::new(16, 256, 128); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_partial_neox_rope"), "{ptx}"); + assert!(ptx.contains("sin.approx.f32"), "{ptx}"); + assert!(ptx.contains("cos.approx.f32"), "{ptx}"); + // Cody-Waite: the reduction is two fma steps against a split 2*pi, so a + // theta of a few thousand never reaches sin.approx unreduced. + assert!( + ptx.contains("cvt.rmi.f32.f32"), + "range reduction floor:\n{ptx}" + ); + assert_eq!( + ptx.matches("fma.rn.f32").count(), + 2, + "two-word reduction, and nothing else:\n{ptx}" + ); + // theta_scale arrives from the host: no device pow, which cannot be made + // sub-ulp here and an ulp is the whole budget (see the module docs). + assert!(!ptx.contains("lg2.approx.f32"), "{ptx}"); + assert!(ptx.contains("theta_scale"), "{ptx}"); + assert_eq!(kernel.grid(), (16, 1, 1)); + assert_eq!(kernel.block(), (64, 1, 1)); + } + + #[test] + fn gdn_partial_rope_theta_scale_is_the_cpu_expression() { + let kernel = PartialNeoxRopeKernel::new(16, 256, 128); + for freq_base in [10_000.0f32, 1_000_000.0f32] { + assert_eq!( + kernel.theta_scale(freq_base).to_bits(), + freq_base.powf(-2.0 / 128.0f32).to_bits(), + "theta_scale must be bit-identical to the CPU reference" + ); + } + } + + #[test] + fn gdn_partial_rope_two_pi_split_is_exact() { + // The reduction is only sound if TWO_PI_HI is exact with few enough bits and + // the two words really sum to 2*pi. + let sum = f64::from(TWO_PI_HI) + f64::from(TWO_PI_LO); + assert!( + (sum - std::f64::consts::TAU).abs() < 1e-10, + "hi+lo must reconstruct 2*pi, got {sum}" + ); + // n * TWO_PI_HI must be exact for n up to 2^16: 6.28125 = 201/32. + assert!((f64::from(TWO_PI_HI) * 32.0 - 201.0).abs() < f64::EPSILON); + } +} + +/// Device parity against a verbatim port of `apply_partial_neox_rope`. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_partial_rope_device_tests { + use super::PartialNeoxRopeKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg}; + + /// `config.num_heads` of the Qwen3.5-0.8B attention layers. + const NUM_HEADS: usize = 16; + /// `attn_q_norm.len()`. + const HEAD_DIM: usize = 256; + /// `2 * sum(rope.dimension_sections)` = `2 * (16 + 24 + 24 + 0)`. + const N_ROT: usize = 128; + + /// Verbatim port of `aprender-serve`'s `apply_partial_neox_rope` + /// (forward_qwen35.rs:191). + fn apply_partial_neox_rope( + x: &mut [f32], + num_heads: usize, + head_dim: usize, + n_rot: usize, + pos: usize, + freq_base: f32, + ) { + let half = n_rot / 2; + let theta_scale = freq_base.powf(-2.0 / n_rot as f32); + for h in 0..num_heads { + let base = h * head_dim; + let mut theta = pos as f32; + for j in 0..half { + let (sin, cos) = theta.sin_cos(); + let (a, b) = (x[base + j], x[base + j + half]); + x[base + j] = a * cos - b * sin; + x[base + j + half] = a * sin + b * cos; + theta *= theta_scale; + } + } + } + + #[test] + fn gdn_partial_neox_rope_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_partial_neox_rope: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + let kernel = PartialNeoxRopeKernel::new(NUM_HEADS as u32, HEAD_DIM as u32, N_ROT as u32); + + let mut rng = Lcg::new(0x3477_0020); + let host = rng.vec(NUM_HEADS * HEAD_DIM, 1.0); + + for freq_base in [10_000.0f32, 1_000_000.0f32] { + for pos in [0usize, 1, 17, 1000] { + let mut want = host.clone(); + apply_partial_neox_rope(&mut want, NUM_HEADS, HEAD_DIM, N_ROT, pos, freq_base); + + let buf = GpuBuffer::from_host(&ctx, &host).expect("x"); + // The f32 parameter travels in the low half of its argument slot. + let mut args = [ + buf.as_ptr(), + u64::try_from(pos).expect("pos"), + u64::from(kernel.theta_scale(freq_base).to_bits()), + ]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; host.len()]; + buf.copy_to_host(&mut got).expect("download"); + assert_close( + &got, + &want, + 1e-4, + &format!("partial NEOX RoPE, pos {pos}, freq_base {freq_base}"), + ); + + // The tail past n_rot must be untouched — the whole point of the + // "partial" in partial NEOX RoPE. + for h in 0..NUM_HEADS { + let base = h * HEAD_DIM; + assert_eq!( + &got[base + N_ROT..base + HEAD_DIM], + &host[base + N_ROT..base + HEAD_DIM], + "head {h} was rotated past n_rot at pos {pos}" + ); + } + } + } + } + + #[test] + fn gdn_partial_neox_rope_at_position_zero_is_the_identity() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_partial_neox_rope identity: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + let kernel = PartialNeoxRopeKernel::new(NUM_HEADS as u32, HEAD_DIM as u32, N_ROT as u32); + + let mut rng = Lcg::new(0x3477_0021); + let host = rng.vec(NUM_HEADS * HEAD_DIM, 1.0); + let buf = GpuBuffer::from_host(&ctx, &host).expect("x"); + let scale = u64::from(kernel.theta_scale(1_000_000.0).to_bits()); + let mut args = [buf.as_ptr(), 0u64, scale]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + let mut got = vec![0.0f32; host.len()]; + buf.copy_to_host(&mut got).expect("download"); + // theta = 0 for every pair, so sin = 0, cos = 1: bit-identical passthrough. + assert_eq!(got, host, "position 0 must not move a single element"); + + // A kernel that did nothing at all would also pass the assertion above, so + // the same buffer is rotated at a position that must move it. + let moving = GpuBuffer::from_host(&ctx, &host).expect("x"); + let mut args2 = [moving.as_ptr(), 3u64, scale]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args2, + ); + let mut moved = vec![0.0f32; host.len()]; + moving.copy_to_host(&mut moved).expect("download"); + assert!( + moved != host, + "position 3 must rotate the first n_rot dimensions" + ); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/sigmoid_gate.rs b/crates/aprender-gpu/src/kernels/gdn/sigmoid_gate.rs new file mode 100644 index 0000000000..d65760fd19 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/sigmoid_gate.rs @@ -0,0 +1,148 @@ +//! PMAT-3477: the output sigmoid gate of Qwen3.5's full-attention layers. +//! +//! `apply_sigmoid_gate` in the CPU reference: `x[i] *= sigmoid(gate[i])`, applied to +//! the attention output before the output projection (llama.cpp's `attn_gated`). +//! +//! Grid: `(ceil(n / 256), 1, 1)`, Block: `(256, 1, 1)`. + +use crate::kernels::gdn::{emit_sigmoid_f32, ELEMENTWISE_BLOCK}; +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Elementwise `x *= sigmoid(gate)`, in place. +#[derive(Debug, Clone, Copy)] +pub struct SigmoidGateKernel { + /// Number of elements. + pub n: u32, +} + +impl SigmoidGateKernel { + /// Create the kernel. + #[must_use] + pub const fn new(n: u32) -> Self { + Self { n } + } + + /// Launch grid. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.n.div_ceil(ELEMENTWISE_BLOCK), 1, 1) + } + + /// Launch block. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (ELEMENTWISE_BLOCK, 1, 1) + } +} + +impl Kernel for SigmoidGateKernel { + fn name(&self) -> &str { + "gdn_sigmoid_gate" + } + + fn build_ptx(&self) -> PtxKernel { + let n = self.n; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "x_ptr") // [n], scaled in place + .param(PtxType::U64, "gate_ptr") // [n] + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let cta = ctx.special_reg(PtxReg::CtaIdX); + let block = ctx.mov_u32_imm(ELEMENTWISE_BLOCK); + let i = ctx.mad_lo_u32(cta, block, tid); + + let n_r = ctx.mov_u32_imm(n); + let go = ctx.setp_lt_u32(i, n_r); + ctx.branch_if_not(go, "gdn_sigmoid_gate_exit"); + + let x_ptr = ctx.load_param_u64("x_ptr"); + let gate_ptr = ctx.load_param_u64("gate_ptr"); + let four = ctx.mov_u32_imm(4); + let off = ctx.mul_wide_u32_reg(i, four); + let x_addr = ctx.add_u64(x_ptr, off); + let gate_addr = ctx.add_u64(gate_ptr, off); + + let x = ctx.ld_global_f32(x_addr); + let g = ctx.ld_global_f32(gate_addr); + let s = emit_sigmoid_f32(ctx, g); + let result = ctx.mul_f32(x, s); + ctx.st_global_f32(x_addr, result); + + ctx.label("gdn_sigmoid_gate_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_sigmoid_gate_ptx_shape() { + let kernel = SigmoidGateKernel::new(2048); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_sigmoid_gate"), "{ptx}"); + assert_eq!(ptx.matches("ex2.approx.f32").count(), 1, "{ptx}"); + assert_eq!(ptx.matches("st.global.f32").count(), 1, "{ptx}"); + assert_eq!(kernel.grid(), (8, 1, 1)); + } +} + +/// Device parity against a verbatim port of `apply_sigmoid_gate`. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_sigmoid_gate_device_tests { + use super::SigmoidGateKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg, HEAD_DIM, NUM_K_HEADS}; + + /// Verbatim port of `apply_sigmoid_gate` (forward_qwen35.rs:44). + fn apply_sigmoid_gate(x: &mut [f32], gate: &[f32]) { + for (o, g) in x.iter_mut().zip(gate) { + *o *= 1.0 / (1.0 + (-*g).exp()); + } + } + + #[test] + fn gdn_sigmoid_gate_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_sigmoid_gate: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + // A full-attention layer's attn_output width for Qwen3.5-0.8B. + let n = HEAD_DIM * NUM_K_HEADS; + let mut rng = Lcg::new(0x3477_0006); + let host = rng.vec(n, 2.0); + // Gate values spanning the saturating tails as well as the linear region. + let gate: Vec = (0..n) + .map(|i| rng.next_scaled(6.0) + (i % 3) as f32 - 1.0) + .collect(); + + let mut want = host.clone(); + apply_sigmoid_gate(&mut want, &gate); + + let x_buf = GpuBuffer::from_host(&ctx, &host).expect("x"); + let gate_buf = GpuBuffer::from_host(&ctx, &gate).expect("gate"); + let kernel = SigmoidGateKernel::new(n as u32); + let mut args = [x_buf.as_ptr(), gate_buf.as_ptr()]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got = vec![0.0f32; n]; + x_buf.copy_to_host(&mut got).expect("download"); + assert_close(&got, &want, 1e-3, "x *= sigmoid(gate)"); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/split_interleave.rs b/crates/aprender-gpu/src/kernels/gdn/split_interleave.rs new file mode 100644 index 0000000000..7919f8f689 --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/split_interleave.rs @@ -0,0 +1,208 @@ +//! PMAT-3477 / aprender#3090: de-interleave Qwen3.5's joint Q|gate projection. +//! +//! `forward_attention` in the CPU reference reads `attn_q` as one GEMV of width +//! `num_heads * 2 * head_dim`, laid out **per head** as `[q(head_dim) | gate(head_dim)]`, +//! and splits it with two `copy_from_slice`s: +//! +//! ```text +//! q [h * head_dim .. +head_dim] = q_full[h * head_dim * 2 .. +head_dim] +//! gate[h * head_dim .. +head_dim] = q_full[h * head_dim * 2 + head_dim .. +head_dim] +//! ``` +//! +//! Grid: `(num_heads, 1, 1)`, Block: `(256, 1, 1)` with a strided loop, so the same +//! kernel serves any `head_dim`. Pure data movement — no arithmetic, so the device +//! result must be **bit-identical** to the host split. + +use crate::kernels::Kernel; +use crate::ptx::builder::{PtxArithmetic, PtxComparison, PtxControl}; +use crate::ptx::{PtxKernel, PtxReg, PtxType}; + +/// Threads per block for the split. +const BLOCK: u32 = 256; + +/// Split an interleaved `[q | gate]` attention projection into two contiguous buffers. +/// +/// For Qwen3.5-0.8B: `num_heads = 16`, `head_dim = 256`, so the source is +/// `16 * 2 * 256 = 8192` wide and each destination is `4096` wide. +#[derive(Debug, Clone, Copy)] +pub struct SplitInterleavedKernel { + /// Number of query heads (`config.num_heads`). + pub num_heads: u32, + /// Width of one head (`attn_q_norm.len()`, 256 for Qwen3.5). + pub head_dim: u32, +} + +impl SplitInterleavedKernel { + /// Create the kernel. + #[must_use] + pub const fn new(num_heads: u32, head_dim: u32) -> Self { + Self { + num_heads, + head_dim, + } + } + + /// Launch grid — one block per head. + #[must_use] + pub const fn grid(&self) -> (u32, u32, u32) { + (self.num_heads, 1, 1) + } + + /// Launch block — 256 threads striding over `head_dim`. + #[must_use] + pub const fn block(&self) -> (u32, u32, u32) { + (BLOCK, 1, 1) + } +} + +impl Kernel for SplitInterleavedKernel { + fn name(&self) -> &str { + "gdn_split_interleaved_q_gate" + } + + fn build_ptx(&self) -> PtxKernel { + let head_dim = self.head_dim; + + PtxKernel::new(self.name()) + .param(PtxType::U64, "src_ptr") // [num_heads * 2 * head_dim] + .param(PtxType::U64, "q_ptr") // [num_heads * head_dim] + .param(PtxType::U64, "gate_ptr") // [num_heads * head_dim] + .shared_memory(0) + .build(|ctx| { + let tid = ctx.special_reg(PtxReg::TidX); + let h = ctx.special_reg(PtxReg::CtaIdX); + + let src_ptr = ctx.load_param_u64("src_ptr"); + let q_ptr = ctx.load_param_u64("q_ptr"); + let gate_ptr = ctx.load_param_u64("gate_ptr"); + + // src head base: h * 2 * head_dim * 4 bytes; dst head base: h * head_dim * 4. + let src_head_off = ctx.mul_wide_u32(h, head_dim * 2 * 4); + let dst_head_off = ctx.mul_wide_u32(h, head_dim * 4); + let src_head = ctx.add_u64(src_ptr, src_head_off); + let q_head = ctx.add_u64(q_ptr, dst_head_off); + let gate_head = ctx.add_u64(gate_ptr, dst_head_off); + + // The gate half starts head_dim floats into the head's source block. + let gate_src_delta = ctx.mov_u64_imm(u64::from(head_dim) * 4); + let gate_src_head = ctx.add_u64(src_head, gate_src_delta); + + let head_dim_r = ctx.mov_u32_imm(head_dim); + let i = ctx.add_u32(tid, 0); // copy tid into a mutable counter + ctx.label("gdn_split_loop"); + let go = ctx.setp_lt_u32(i, head_dim_r); + ctx.branch_if_not(go, "gdn_split_exit"); + + let off = ctx.mul_wide_u32(i, 4); + let q_src = ctx.add_u64(src_head, off); + let g_src = ctx.add_u64(gate_src_head, off); + let q_dst = ctx.add_u64(q_head, off); + let g_dst = ctx.add_u64(gate_head, off); + let q_val = ctx.ld_global_f32(q_src); + let g_val = ctx.ld_global_f32(g_src); + ctx.st_global_f32(q_dst, q_val); + ctx.st_global_f32(g_dst, g_val); + + ctx.add_u32_inplace(i, BLOCK); + ctx.branch("gdn_split_loop"); + + ctx.label("gdn_split_exit"); + ctx.ret(); + }) + } +} + +#[cfg(test)] +mod ptx_tests { + use super::*; + + #[test] + fn gdn_split_interleaved_ptx_is_pure_data_movement() { + let kernel = SplitInterleavedKernel::new(16, 256); + let ptx = kernel.emit_ptx(); + assert!(ptx.contains(".entry gdn_split_interleaved_q_gate"), "{ptx}"); + // A de-interleave is loads and stores only: any float arithmetic here would + // mean the kernel is transforming what it is supposed to copy. + for op in ["add.f32", "mul.f32", "fma.rn.f32", "ex2.approx.f32"] { + assert!(!ptx.contains(op), "unexpected {op} in a pure copy:\n{ptx}"); + } + // Two loads and two stores per iteration: q and gate. + assert_eq!(ptx.matches("ld.global.f32").count(), 2, "{ptx}"); + assert_eq!(ptx.matches("st.global.f32").count(), 2, "{ptx}"); + assert_eq!(kernel.grid(), (16, 1, 1)); + assert_eq!(kernel.block(), (256, 1, 1)); + } +} + +/// Device parity against a verbatim port of `forward_attention`'s split. +#[cfg(test)] +#[cfg(feature = "cuda")] +mod gdn_split_interleave_device_tests { + use super::SplitInterleavedKernel; + use crate::driver::{CudaContext, CudaStream, GpuBuffer}; + use crate::kernels::gdn::test_support::{assert_close, run_kernel, Lcg}; + + /// `num_heads` of the Qwen3.5-0.8B attention layers (`config.num_heads`). + const NUM_HEADS: usize = 16; + /// `head_dim` of the Qwen3.5-0.8B attention layers (`attn_q_norm.len()`). + const HEAD_DIM: usize = 256; + + /// Verbatim port of the `q` / `gate` split in `forward_attention` + /// (forward_qwen35.rs:1008). + fn split_q_gate(q_full: &[f32], num_heads: usize, head_dim: usize) -> (Vec, Vec) { + let mut q = vec![0.0; num_heads * head_dim]; + let mut gate = vec![0.0; num_heads * head_dim]; + for h in 0..num_heads { + let offset_q_full = h * head_dim * 2; + let offset_q = h * head_dim; + q[offset_q..offset_q + head_dim] + .copy_from_slice(&q_full[offset_q_full..offset_q_full + head_dim]); + gate[offset_q..offset_q + head_dim] + .copy_from_slice(&q_full[offset_q_full + head_dim..offset_q_full + head_dim * 2]); + } + (q, gate) + } + + #[test] + fn gdn_split_interleaved_matches_cpu_reference() { + let Ok(ctx) = CudaContext::new(0) else { + println!("gdn_split_interleaved: no CUDA device — SKIPPED."); + return; + }; + let stream = CudaStream::new(&ctx).expect("stream"); + + let mut rng = Lcg::new(0x3477_0010); + let src = rng.vec(NUM_HEADS * 2 * HEAD_DIM, 1.0); + let (want_q, want_gate) = split_q_gate(&src, NUM_HEADS, HEAD_DIM); + + let src_buf = GpuBuffer::from_host(&ctx, &src).expect("src"); + let q_buf = GpuBuffer::::new(&ctx, NUM_HEADS * HEAD_DIM).expect("q"); + let gate_buf = GpuBuffer::::new(&ctx, NUM_HEADS * HEAD_DIM).expect("gate"); + let kernel = SplitInterleavedKernel::new(NUM_HEADS as u32, HEAD_DIM as u32); + let mut args = [src_buf.as_ptr(), q_buf.as_ptr(), gate_buf.as_ptr()]; + run_kernel( + &ctx, + &stream, + &kernel, + kernel.grid(), + kernel.block(), + &mut args, + ); + + let mut got_q = vec![0.0f32; NUM_HEADS * HEAD_DIM]; + let mut got_gate = vec![0.0f32; NUM_HEADS * HEAD_DIM]; + q_buf.copy_to_host(&mut got_q).expect("download q"); + gate_buf.copy_to_host(&mut got_gate).expect("download gate"); + + // tol 0: a copy that is off by anything at all is a wrong copy. + assert_close(&got_q, &want_q, 0.0, "split q"); + assert_close(&got_gate, &want_gate, 0.0, "split gate"); + + // A kernel that ignored the per-head stride (e.g. treating the source as + // [all q | all gate]) would still be bit-exact on head 0 alone. + assert!( + want_q[HEAD_DIM..] != want_gate[HEAD_DIM..], + "fixture must distinguish the halves beyond head 0" + ); + } +} diff --git a/crates/aprender-gpu/src/kernels/gdn/test_support.rs b/crates/aprender-gpu/src/kernels/gdn/test_support.rs new file mode 100644 index 0000000000..3fa693ad9f --- /dev/null +++ b/crates/aprender-gpu/src/kernels/gdn/test_support.rs @@ -0,0 +1,151 @@ +//! Shared fixtures for the Gated `DeltaNet` device tests (PMAT-3477). +//! +//! Deterministic inputs (a seeded LCG — no `rand` dependency), the real +//! Qwen3.5-0.8B shapes, and one launch helper so each kernel's test is only its +//! host reference plus its assertion. + +#![cfg(test)] + +/// `head_k_dim` / `head_v_dim` for Qwen3.5-0.8B (`qwen2.ssm.state_size`). +pub(crate) const HEAD_DIM: usize = 128; +/// `num_k_heads` for Qwen3.5-0.8B (`qwen2.ssm.group_count`). +pub(crate) const NUM_K_HEADS: usize = 16; +/// `num_v_heads` for Qwen3.5-0.8B (`qwen2.ssm.time_step_rank`). +pub(crate) const NUM_V_HEADS: usize = 16; +/// `conv_kernel` for Qwen3.5-0.8B (`qwen2.ssm.conv_kernel`). +pub(crate) const CONV_KERNEL: usize = 4; +/// `conv_dim` as `forward_deltanet` computes it: `k_dim * 2 + v_dim` = 6144. +pub(crate) const CONV_CHANNELS: usize = HEAD_DIM * NUM_K_HEADS * 2 + HEAD_DIM * NUM_V_HEADS; +/// `config.eps` of the 0.8B model. +pub(crate) const EPS: f32 = 1e-6; + +/// A seeded LCG producing values in roughly `[-1, 1)`. +pub(crate) struct Lcg(u32); + +impl Lcg { + pub(crate) const fn new(seed: u32) -> Self { + Self(seed) + } + + /// Next value in `[-1, 1)`. + pub(crate) fn next(&mut self) -> f32 { + self.0 = self.0.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + f32::from(((self.0 >> 16) & 0xFFFF) as u16) / 32768.0 - 1.0 + } + + /// Next value in `[-scale, scale)`. + pub(crate) fn next_scaled(&mut self, scale: f32) -> f32 { + self.next() * scale + } + + /// `n` values in `[-scale, scale)`. + pub(crate) fn vec(&mut self, n: usize, scale: f32) -> Vec { + (0..n).map(|_| self.next_scaled(scale)).collect() + } +} + +#[cfg(feature = "cuda")] +mod device { + use crate::driver::{CudaContext, CudaModule, CudaStream, LaunchConfig}; + use crate::kernels::Kernel; + + /// Compile and launch `kernel` with raw device pointers as arguments. + /// + /// `sm_70` PTX is used deliberately: everything these kernels emit + /// (`shfl.sync`, `ex2.approx`, `lg2.approx`) is Volta-era, so the same module + /// JITs on every supported device. + pub(crate) fn run_kernel( + ctx: &CudaContext, + stream: &CudaStream, + kernel: &K, + grid: (u32, u32, u32), + block: (u32, u32, u32), + args: &mut [u64], + ) { + let ptx = kernel.emit_ptx(); + let mut module = CudaModule::from_ptx(ctx, &ptx).expect("module from ptx"); + let config = LaunchConfig { + grid, + block, + shared_mem: 0, + }; + let mut raw: Vec<*mut std::ffi::c_void> = args + .iter_mut() + .map(|a| std::ptr::from_mut(a).cast()) + .collect(); + // SAFETY: every entry of `args` is a live device allocation sized by the + // caller to the extent the kernel indexes, and grid/block are the kernel's + // documented launch shape. + unsafe { + stream + .launch_kernel(&mut module, kernel.name(), &config, &mut raw) + .expect("kernel launch"); + } + stream.synchronize().expect("stream synchronize"); + } +} + +#[cfg(feature = "cuda")] +pub(crate) use device::run_kernel; + +/// Assert two buffers agree, at `tol` **relative to the reference's own scale**. +/// +/// The limit is `tol * max|want|`, not `tol` — an absolute tolerance is vacuous here. +/// The delta-rule outputs land near 5e-3 (`q`/`k` are unit-norm and the recurrence +/// scales by `D^-0.5`), so an absolute 1e-3 accepted a reference mutated by 1%: +/// measured, not assumed. A reference that is all ~zero fails outright, because then +/// nothing about the kernel has been checked. +pub(crate) fn assert_close(got: &[f32], want: &[f32], tol: f32, what: &str) { + assert_eq!(got.len(), want.len(), "{what}: length mismatch"); + let scale = want.iter().fold(0.0f32, |m, v| m.max(v.abs())); + assert!( + scale > 1e-4, + "{what}: the CPU reference is all ~zero (max |cpu| = {scale}); this comparison \ + would pass however wrong the kernel is" + ); + let limit = tol * scale; + let mut worst = 0.0f32; + let mut worst_i = 0usize; + for (i, (g, w)) in got.iter().zip(want).enumerate() { + let d = (g - w).abs(); + if d > worst { + worst = d; + worst_i = i; + } + } + assert!( + worst <= limit, + "{what}: worst |gpu - cpu| = {worst} at index {worst_i} \ + (gpu {gpu}, cpu {cpu}), limit {limit} = {tol} * max|cpu| {scale}. \ + The CPU side here is a verbatim \ + port of aprender-serve's forward_qwen35.rs, which is the specification for \ + the Gated DeltaNet block (aprender#3090).", + gpu = got[worst_i], + cpu = want[worst_i], + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gdn_lcg_is_deterministic_and_bounded() { + let a = Lcg::new(7).vec(64, 1.0); + let b = Lcg::new(7).vec(64, 1.0); + assert_eq!(a, b, "the fixture must be reproducible"); + assert!(a.iter().all(|v| (-1.0..1.0).contains(v)), "{a:?}"); + // Not a constant stream — a fixture of one repeated value would make most + // of these kernels' index arithmetic unfalsifiable. + assert!(a.windows(2).any(|w| w[0] != w[1])); + } + + #[test] + fn gdn_shapes_match_qwen35_0_8b() { + // group_count * state_size == inner_size, and conv_dim == 3 * inner_size. + assert_eq!(NUM_K_HEADS * HEAD_DIM, 2048); + assert_eq!(CONV_CHANNELS, 3 * 2048); + assert_eq!(CONV_KERNEL, 4); + assert_eq!(NUM_V_HEADS, 16); + } +} diff --git a/crates/aprender-gpu/src/kernels/mod.rs b/crates/aprender-gpu/src/kernels/mod.rs index 775de5fa2d..2747b43a6e 100644 --- a/crates/aprender-gpu/src/kernels/mod.rs +++ b/crates/aprender-gpu/src/kernels/mod.rs @@ -26,6 +26,7 @@ mod bias_activation; mod conv1d; mod elementwise; mod fused; +pub mod gdn; pub(crate) mod gemm; mod gemv; mod layernorm; @@ -82,6 +83,11 @@ pub use elementwise::{ TransposeKernel, // WAPR-PERF-004 }; pub use fused::{FusedGateUpKernel, FusedGemmBiasGeluKernel, FusedQKVKernel}; +pub use gdn::{ + CausalConv1dSiluKernel, DecodeAttention256Kernel, DeltaRuleRecurrenceKernel, + GatedRmsNormKernel, GdnGatesKernel, PartialNeoxRopeKernel, PerHeadL2NormKernel, + SigmoidGateKernel, SplitInterleavedKernel, DEFAULT_MAX_POSITIONS_PER_PASS, +}; pub use gemm::basic::tensor_core::cta64_wmma::build_cta64x128_mma_pipeline_fp16; pub use gemm::{ Batched4DGemmConfig, Batched4DGemmKernel, BatchedGemmConfig, BatchedGemmKernel, GemmConfig, diff --git a/crates/aprender-serve/src/capability.rs b/crates/aprender-serve/src/capability.rs index 2b27d5fe26..b1e810b1c4 100644 --- a/crates/aprender-serve/src/capability.rs +++ b/crates/aprender-serve/src/capability.rs @@ -299,6 +299,37 @@ mod tests { assert!(check_capability(&required, &supported).is_ok()); } + /// #3477 (operator ruling 2026-09-19): the capability gate must ADMIT the + /// Qwen3.5 hybrid now that `Qwen35CudaModel` gives it a GPU forward (#3090). + /// + /// `ArchConstraints` declares no Gated DeltaNet operation — the struct has + /// no field for one, and the qwen3_5 row is an ordinary + /// RoPE/RMSNorm/SwiGLU/no-QK-norm row — so `required_ops` derives only ops + /// `gpu_supported_ops` already contains. That is why `check_gpu_capability` + /// admits this architecture without a new `RequiredOp`: the discriminator + /// for "can the GPU run the hybrid layers" is the hybrid forward's own + /// existence (`gguf::hybrid_forward_handles`), not this op set. This test + /// pins the admission so a future op added to the qwen3_5 row cannot + /// silently re-refuse it. + #[test] + fn qwen35_hybrid_is_admitted_by_the_gpu_capability_gate() { + for arch in ["qwen35", "qwen3_5", "qwen3.5"] { + let constraints = ArchConstraints::from_architecture(arch); + let required = required_ops_for_model(&constraints, arch); + let supported = gpu_supported_ops(); + assert!( + check_capability(&required, &supported).is_ok(), + "{arch}: the GPU forward exists (#3090) — the capability gate must not refuse it, \ + missing {:?}", + check_capability(&required, &supported) + ); + assert!( + crate::gguf::hybrid_forward_handles("qwen35"), + "the runtime-dispatch predicate is what decides the hybrid layers" + ); + } + } + #[test] fn test_required_op_display() { assert_eq!(RequiredOp::QkNorm.to_string(), "QkNorm"); diff --git a/crates/aprender-serve/src/cuda/executor/gdn_ops.rs b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs new file mode 100644 index 0000000000..1409e583cd --- /dev/null +++ b/crates/aprender-serve/src/cuda/executor/gdn_ops.rs @@ -0,0 +1,489 @@ +//! PMAT-3477 / aprender#3090: executor wrappers for the six Gated `DeltaNet` +//! device kernels in `aprender-gpu/src/kernels/gdn/`. +//! +//! Each wrapper mirrors [`CudaExecutor::per_head_rmsnorm_into`] exactly: +//! `KernelType` arm -> kernel name -> PTX compiled once per shape and cached by +//! key -> `LaunchConfig` from the kernel's own `grid()`/`block()` -> launch -> +//! **the `graph_recording` push**. The push is not optional: the manual decode +//! graph is rebuilt ONLY from `graph_recorded_kernels`, so a kernel that does not +//! record itself is silently dropped from every replayed step (#3413 was exactly +//! that bug, for QK-norm). +//! +//! The CPU reference these reproduce is +//! `gguf/inference/forward/forward_qwen35.rs`; the per-layer parity contract +//! (relative L∞ <= 1e-3 against `forward_deltanet`) is proven in +//! `gguf/cuda/forward_qwen35_cuda_tests.rs`. + +use super::*; + +impl CudaExecutor { + /// Compile (once) and fetch the module for a Gated `DeltaNet` kernel. + /// + /// Returns the kernel's entry name; the module lives in `self.modules` + /// under `cache_key`. + fn gdn_prepare( + &mut self, + kernel_type: &KernelType, + cache_key: &str, + ) -> Result<&'static str, GpuError> { + let kernel_name = self.kernels.kernel_name(kernel_type); + if !self.modules.contains_key(cache_key) { + let ptx = self.kernels.generate_ptx(kernel_type); + let module = self.compile_ptx(&ptx)?; + self.modules.insert(cache_key.to_string(), module); + } + Ok(kernel_name) + } + + /// Launch a Gated `DeltaNet` kernel whose arguments are all device pointers, + /// and record it for manual graph construction. + /// + /// Every GDN kernel folds its dimensions into the PTX as immediates, so the + /// argument list is pointers only — which is also what makes `arg_data` a + /// faithful replay record. + fn gdn_launch( + &mut self, + cache_key: &str, + kernel_name: &'static str, + config: LaunchConfig, + ptrs: &[u64], + ) -> Result<(), GpuError> { + for (i, &p) in ptrs.iter().enumerate() { + validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + } + let mut args: Vec = ptrs.to_vec(); + let mut raw: Vec<*mut std::ffi::c_void> = args + .iter_mut() + .map(|a| std::ptr::from_mut(a).cast::()) + .collect(); + + let module = self + .modules + .get_mut(cache_key) + .expect("module just inserted"); + // SAFETY: every pointer was checked non-null above, each buffer is + // allocated with the length the kernel's immediates encode, and the + // argument order matches the kernel's `.param` declarations. + unsafe { + self.stream + .launch_kernel(module, kernel_name, &config, &mut raw)?; + } + + // trueno#243 / #3413: the manual decode graph is rebuilt ONLY from + // `graph_recorded_kernels` — a kernel that skips this push runs during + // capture and never again during replay. + if self.graph_recording { + let module = self.modules.get_mut(cache_key).expect("module exists"); + let func = module.get_function(kernel_name)?; + self.graph_recorded_kernels.push(RecordedKernel { + func: SendCUfunction(func), + config, + arg_data: args, + }); + } + Ok(()) + } + + /// Launch a Gated `DeltaNet` kernel that takes device pointers FOLLOWED BY + /// scalar parameters, and record it for manual graph construction. + /// + /// The driver reads each argument slot as the parameter's declared width out + /// of the low half of a `u64`: a `.param(PtxType::U32, …)` is + /// `u64::from(x)` and a `.param(PtxType::F32, …)` is + /// `u64::from(x.to_bits())`. Only `ptrs` are validated as device pointers — + /// a scalar is not one, and `validate_device_ptr` would reject every small + /// integer. + fn gdn_launch_mixed( + &mut self, + cache_key: &str, + kernel_name: &'static str, + config: LaunchConfig, + ptrs: &[u64], + scalars: &[u64], + ) -> Result<(), GpuError> { + for (i, &p) in ptrs.iter().enumerate() { + validate_device_ptr(p, &format!("{kernel_name} arg {i}"))?; + } + let mut args: Vec = ptrs.to_vec(); + args.extend_from_slice(scalars); + let mut raw: Vec<*mut std::ffi::c_void> = args + .iter_mut() + .map(|a| std::ptr::from_mut(a).cast::()) + .collect(); + + let module = self + .modules + .get_mut(cache_key) + .expect("module just inserted"); + // SAFETY: every pointer was checked non-null above, each buffer is + // allocated with the length the kernel's immediates encode, the scalars + // are the widths the kernel declares, and the argument order matches the + // kernel's `.param` declarations. + unsafe { + self.stream + .launch_kernel(module, kernel_name, &config, &mut raw)?; + } + + // trueno#243 / #3413: the manual decode graph is rebuilt ONLY from + // `graph_recorded_kernels` — a kernel that skips this push runs during + // capture and never again during replay. + if self.graph_recording { + let module = self.modules.get_mut(cache_key).expect("module exists"); + let func = module.get_function(kernel_name)?; + self.graph_recorded_kernels.push(RecordedKernel { + func: SendCUfunction(func), + config, + arg_data: args, + }); + } + Ok(()) + } + + /// De-interleave the joint `[q | gate]` attention projection into two + /// contiguous buffers, one `head_dim` slice of each per head. + /// + /// `src` is `[num_heads * 2 * head_dim]`; `q` and `gate` are + /// `[num_heads * head_dim]`. Pure data movement — the result is + /// bit-identical to the CPU's two `copy_from_slice`s. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + pub fn gdn_split_interleaved_into( + &mut self, + src: &GpuBuffer, + q: &GpuBuffer, + gate: &GpuBuffer, + num_heads: u32, + head_dim: u32, + ) -> Result<(), GpuError> { + let kernel = trueno_gpu::kernels::gdn::SplitInterleavedKernel::new(num_heads, head_dim); + let kernel_type = KernelType::GdnSplitInterleaved { + num_heads, + head_dim, + }; + let cache_key = format!("gdn_split_interleaved_{num_heads}_{head_dim}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[src.as_ptr(), q.as_ptr(), gate.as_ptr()], + ) + } + + /// Partial NEOX RoPE over the first `n_rot` dimensions of each head of `x`, + /// in place (`apply_partial_neox_rope`). + /// + /// `theta_scale` MUST be [`trueno_gpu::kernels::gdn::PartialNeoxRopeKernel::theta_scale`] + /// (`freq_base.powf(-2.0 / n_rot)`), computed on the HOST exactly as the CPU + /// reference computes it: the kernel rebuilds the CPU's iterative `theta` by + /// multiplying it in, and the error is amplified by `j * theta_j`, so one + /// ulp of a device-computed scale is already more than the parity budget. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + pub fn gdn_partial_neox_rope_into( + &mut self, + x: &GpuBuffer, + num_heads: u32, + head_dim: u32, + n_rot: u32, + position: u32, + theta_scale: f32, + ) -> Result<(), GpuError> { + let kernel = + trueno_gpu::kernels::gdn::PartialNeoxRopeKernel::new(num_heads, head_dim, n_rot); + let kernel_type = KernelType::GdnPartialNeoxRope { + num_heads, + head_dim, + n_rot, + }; + let cache_key = format!("gdn_partial_neox_rope_{num_heads}_{head_dim}_{n_rot}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch_mixed( + &cache_key, + kernel_name, + config, + &[x.as_ptr()], + &[u64::from(position), u64::from(theta_scale.to_bits())], + ) + } + + /// Single-query decode attention over a resident KV cache, `head_dim <= 256` + /// (the scores / softmax / value accumulation block of `forward_attention`). + /// + /// `k_cache` and `v_cache` are `[max_len][num_kv_heads * head_dim]`; + /// `seq_len` is the number of valid positions, i.e. `position + 1`, and is a + /// runtime parameter so one compiled module serves the whole decode. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + #[allow(clippy::too_many_arguments)] + pub fn gdn_decode_attention_into( + &mut self, + q: &GpuBuffer, + k_cache: &GpuBuffer, + v_cache: &GpuBuffer, + output: &GpuBuffer, + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + seq_len: u32, + ) -> Result<(), GpuError> { + let kernel = trueno_gpu::kernels::gdn::DecodeAttention256Kernel::new( + num_heads, + num_kv_heads, + head_dim, + ); + let kernel_type = KernelType::GdnDecodeAttention { + num_heads, + num_kv_heads, + head_dim, + }; + let cache_key = format!("gdn_decode_attention_{num_heads}_{num_kv_heads}_{head_dim}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch_mixed( + &cache_key, + kernel_name, + config, + &[ + q.as_ptr(), + k_cache.as_ptr(), + v_cache.as_ptr(), + output.as_ptr(), + ], + &[u64::from(seq_len)], + ) + } + + /// Fused causal depthwise conv1d + SiLU for one decode step (`causal_conv1d` + /// plus the SiLU loop that follows it in `forward_deltanet`). + /// + /// `state` is `[channels * (kernel_size - 1)]` and is updated IN PLACE (the + /// window shifts left and the new sample is appended), exactly as the CPU + /// reference does. `weight` is `[channels * kernel_size]`, channel-outer. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + pub fn gdn_causal_conv1d_silu_into( + &mut self, + input: &GpuBuffer, + state: &GpuBuffer, + weight: &GpuBuffer, + output: &GpuBuffer, + channels: u32, + kernel_size: u32, + ) -> Result<(), GpuError> { + let kernel = trueno_gpu::kernels::gdn::CausalConv1dSiluKernel::new(channels, kernel_size); + let kernel_type = KernelType::GdnCausalConv1dSilu { + channels, + kernel_size, + }; + let cache_key = format!("gdn_causal_conv1d_silu_{channels}_{kernel_size}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[ + input.as_ptr(), + state.as_ptr(), + weight.as_ptr(), + output.as_ptr(), + ], + ) + } + + /// Per-head L2 normalisation of `x`, in place (`l2_norm_per_head`). + /// + /// Epsilon is added to the SUM of squares, not to a mean — this is + /// llama.cpp's `build_gdn_l2_norm`, not RMSNorm. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + pub fn gdn_per_head_l2_norm_into( + &mut self, + x: &GpuBuffer, + head_dim: u32, + num_heads: u32, + epsilon: f32, + ) -> Result<(), GpuError> { + let kernel = + trueno_gpu::kernels::gdn::PerHeadL2NormKernel::new(head_dim, num_heads, epsilon); + let kernel_type = KernelType::GdnPerHeadL2Norm { + head_dim, + num_heads, + epsilon, + }; + // epsilon is an immediate in the PTX, so it belongs in the cache key. + let cache_key = format!("gdn_per_head_l2_norm_{head_dim}_{num_heads}_{epsilon:e}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch(&cache_key, kernel_name, config, &[x.as_ptr()]) + } + + /// The per-head gates: `dt[h] = softplus(alpha[h] + dt_bias[h]) * a[h]` and + /// `beta[h] = sigmoid(beta_raw[h])`. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + #[allow(clippy::too_many_arguments)] + pub fn gdn_gates_into( + &mut self, + alpha: &GpuBuffer, + dt_bias: &GpuBuffer, + a: &GpuBuffer, + beta_raw: &GpuBuffer, + dt_out: &GpuBuffer, + beta_out: &GpuBuffer, + num_heads: u32, + ) -> Result<(), GpuError> { + let kernel = trueno_gpu::kernels::gdn::GdnGatesKernel::new(num_heads); + let kernel_type = KernelType::GdnGates { num_heads }; + let cache_key = format!("gdn_gates_{num_heads}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[ + alpha.as_ptr(), + dt_bias.as_ptr(), + a.as_ptr(), + beta_raw.as_ptr(), + dt_out.as_ptr(), + beta_out.as_ptr(), + ], + ) + } + + /// The gated delta-rule recurrence for one token + /// (`delta_rule_recurrence`). `state` is `[num_v_heads * D * D]` and is + /// updated in place; `output` is `[num_v_heads * D]`. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + #[allow(clippy::too_many_arguments)] + pub fn gdn_delta_rule_into( + &mut self, + q: &GpuBuffer, + k: &GpuBuffer, + v: &GpuBuffer, + beta: &GpuBuffer, + gate: &GpuBuffer, + state: &GpuBuffer, + output: &GpuBuffer, + num_v_heads: u32, + head_v_dim: u32, + ) -> Result<(), GpuError> { + let kernel = + trueno_gpu::kernels::gdn::DeltaRuleRecurrenceKernel::new(num_v_heads, head_v_dim); + let kernel_type = KernelType::GdnDeltaRule { + num_v_heads, + head_v_dim, + }; + let cache_key = format!("gdn_delta_rule_{num_v_heads}_{head_v_dim}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[ + q.as_ptr(), + k.as_ptr(), + v.as_ptr(), + beta.as_ptr(), + gate.as_ptr(), + state.as_ptr(), + output.as_ptr(), + ], + ) + } + + /// Gated RMSNorm (`gated_rmsnorm`): the INPUT is normalised per head, the + /// gate is not. `weight` is `[head_dim]`, shared across heads. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + #[allow(clippy::too_many_arguments)] + pub fn gdn_gated_rmsnorm_into( + &mut self, + input: &GpuBuffer, + gate: &GpuBuffer, + weight: &GpuBuffer, + output: &GpuBuffer, + head_dim: u32, + num_heads: u32, + epsilon: f32, + ) -> Result<(), GpuError> { + let kernel = + trueno_gpu::kernels::gdn::GatedRmsNormKernel::new(head_dim, num_heads, epsilon); + let kernel_type = KernelType::GdnGatedRmsNorm { + head_dim, + num_heads, + epsilon, + }; + let cache_key = format!("gdn_gated_rmsnorm_{head_dim}_{num_heads}_{epsilon:e}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[ + input.as_ptr(), + gate.as_ptr(), + weight.as_ptr(), + output.as_ptr(), + ], + ) + } + + /// `x[i] *= sigmoid(gate[i])`, in place (`apply_sigmoid_gate`) — the output + /// gate of Qwen3.5's full-attention layers. + /// + /// # Errors + /// PTX compilation or kernel launch failure, or a null device pointer. + pub fn gdn_sigmoid_gate_into( + &mut self, + x: &GpuBuffer, + gate: &GpuBuffer, + n: u32, + ) -> Result<(), GpuError> { + let kernel = trueno_gpu::kernels::gdn::SigmoidGateKernel::new(n); + let kernel_type = KernelType::GdnSigmoidGate { n }; + let cache_key = format!("gdn_sigmoid_gate_{n}"); + let kernel_name = self.gdn_prepare(&kernel_type, &cache_key)?; + let (gx, _, _) = kernel.grid(); + let (bx, _, _) = kernel.block(); + let config = LaunchConfig::grid_2d(gx, 1, bx, 1); + self.gdn_launch( + &cache_key, + kernel_name, + config, + &[x.as_ptr(), gate.as_ptr()], + ) + } +} diff --git a/crates/aprender-serve/src/cuda/executor/mod.rs b/crates/aprender-serve/src/cuda/executor/mod.rs index 82f6d3407f..ba746ce993 100644 --- a/crates/aprender-serve/src/cuda/executor/mod.rs +++ b/crates/aprender-serve/src/cuda/executor/mod.rs @@ -263,6 +263,8 @@ mod activations; mod attention; mod bound_dispatch; mod core; +/// PMAT-3477 (#3090): wrappers for the six Gated `DeltaNet` device kernels. +mod gdn_ops; mod gemm; /// PMAT-291: Transformer layer graph builder for Qwen2.5 architecture mod graph_builder; diff --git a/crates/aprender-serve/src/cuda/generate.rs b/crates/aprender-serve/src/cuda/generate.rs index 6bad067e2b..9aeaa792e4 100644 --- a/crates/aprender-serve/src/cuda/generate.rs +++ b/crates/aprender-serve/src/cuda/generate.rs @@ -250,6 +250,44 @@ impl CudaKernels { fn generate_norm_rope_ptx(kernel_type: &KernelType) -> Option { Self::generate_rmsnorm_ptx(kernel_type) .or_else(|| Self::generate_rope_residual_ptx(kernel_type)) + .or_else(|| Self::generate_gdn_ptx(kernel_type)) + } + + /// PMAT-3477 (#3090): PTX for the nine Gated `DeltaNet` / Qwen3.5 attention + /// kernels (sm_70 default). + fn generate_gdn_ptx(kernel_type: &KernelType) -> Option { + use trueno_gpu::kernels::gdn::{ + CausalConv1dSiluKernel, DecodeAttention256Kernel, DeltaRuleRecurrenceKernel, + GatedRmsNormKernel, GdnGatesKernel, PartialNeoxRopeKernel, PerHeadL2NormKernel, + SigmoidGateKernel, SplitInterleavedKernel, + }; + let ptx = match kernel_type { + KernelType::GdnCausalConv1dSilu { channels, kernel_size } => { + CausalConv1dSiluKernel::new(*channels, *kernel_size).emit_ptx() + }, + KernelType::GdnPerHeadL2Norm { head_dim, num_heads, epsilon } => { + PerHeadL2NormKernel::new(*head_dim, *num_heads, *epsilon).emit_ptx() + }, + KernelType::GdnGates { num_heads } => GdnGatesKernel::new(*num_heads).emit_ptx(), + KernelType::GdnDeltaRule { num_v_heads, head_v_dim } => { + DeltaRuleRecurrenceKernel::new(*num_v_heads, *head_v_dim).emit_ptx() + }, + KernelType::GdnGatedRmsNorm { head_dim, num_heads, epsilon } => { + GatedRmsNormKernel::new(*head_dim, *num_heads, *epsilon).emit_ptx() + }, + KernelType::GdnSigmoidGate { n } => SigmoidGateKernel::new(*n).emit_ptx(), + KernelType::GdnSplitInterleaved { num_heads, head_dim } => { + SplitInterleavedKernel::new(*num_heads, *head_dim).emit_ptx() + }, + KernelType::GdnPartialNeoxRope { num_heads, head_dim, n_rot } => { + PartialNeoxRopeKernel::new(*num_heads, *head_dim, *n_rot).emit_ptx() + }, + KernelType::GdnDecodeAttention { num_heads, num_kv_heads, head_dim } => { + DecodeAttention256Kernel::new(*num_heads, *num_kv_heads, *head_dim).emit_ptx() + }, + _ => return None, + }; + Some(ptx) } /// Generate PTX for activation, fusion, and miscellaneous kernels diff --git a/crates/aprender-serve/src/cuda/kernel.rs b/crates/aprender-serve/src/cuda/kernel.rs index 4ef3886da9..5654c9efdc 100644 --- a/crates/aprender-serve/src/cuda/kernel.rs +++ b/crates/aprender-serve/src/cuda/kernel.rs @@ -10,9 +10,30 @@ impl CudaKernels { .or_else(|| Self::attention_kernel_name(kernel_type)) .or_else(|| Self::norm_rope_kernel_name(kernel_type)) .or_else(|| Self::activation_misc_kernel_name(kernel_type)) + .or_else(|| Self::gdn_kernel_name(kernel_type)) .unwrap_or("unknown") } + /// PMAT-3477 (#3090): Gated `DeltaNet` kernel names. + /// + /// Kept as its own arm rather than folded into `activation_misc_kernel_name` + /// so neither function's complexity grows past the gate's ceiling. + fn gdn_kernel_name(kernel_type: &KernelType) -> Option<&'static str> { + let name = match kernel_type { + KernelType::GdnCausalConv1dSilu { .. } => "gdn_causal_conv1d_silu", + KernelType::GdnPerHeadL2Norm { .. } => "gdn_per_head_l2_norm", + KernelType::GdnGates { .. } => "gdn_gates", + KernelType::GdnDeltaRule { .. } => "gdn_delta_rule_recurrence", + KernelType::GdnGatedRmsNorm { .. } => "gdn_gated_rmsnorm", + KernelType::GdnSigmoidGate { .. } => "gdn_sigmoid_gate", + KernelType::GdnSplitInterleaved { .. } => "gdn_split_interleaved_q_gate", + KernelType::GdnPartialNeoxRope { .. } => "gdn_partial_neox_rope", + KernelType::GdnDecodeAttention { .. } => "gdn_decode_attention", + _ => return None, + }; + Some(name) + } + /// GEMM kernel names (matrix-matrix multiplication) fn gemm_kernel_name(kernel_type: &KernelType) -> Option<&'static str> { let name = match kernel_type { diff --git a/crates/aprender-serve/src/cuda/kernel_type.rs b/crates/aprender-serve/src/cuda/kernel_type.rs index 5b0687120b..5deecac9e6 100644 --- a/crates/aprender-serve/src/cuda/kernel_type.rs +++ b/crates/aprender-serve/src/cuda/kernel_type.rs @@ -573,4 +573,38 @@ pub enum KernelType { k: u32, n: u32, }, + /// PMAT-3477 (#3090): fused causal depthwise conv1d + SiLU, one decode step. + GdnCausalConv1dSilu { channels: u32, kernel_size: u32 }, + /// PMAT-3477 (#3090): per-head L2 normalisation of Gated `DeltaNet` q/k, in place. + GdnPerHeadL2Norm { + head_dim: u32, + num_heads: u32, + epsilon: f32, + }, + /// PMAT-3477 (#3090): the Gated `DeltaNet` per-head `dt`/`beta` gates. + GdnGates { num_heads: u32 }, + /// PMAT-3477 (#3090): the gated delta-rule recurrence, one token. + GdnDeltaRule { num_v_heads: u32, head_v_dim: u32 }, + /// PMAT-3477 (#3090): gated RMSNorm — the Gated `DeltaNet` output norm. + GdnGatedRmsNorm { + head_dim: u32, + num_heads: u32, + epsilon: f32, + }, + /// PMAT-3477 (#3090): `x *= sigmoid(gate)`, the full-attention output gate. + GdnSigmoidGate { n: u32 }, + /// PMAT-3477 (#3090): de-interleave the joint `[q | gate]` Q projection. + GdnSplitInterleaved { num_heads: u32, head_dim: u32 }, + /// PMAT-3477 (#3090): partial NEOX RoPE over the first `n_rot` dimensions. + GdnPartialNeoxRope { + num_heads: u32, + head_dim: u32, + n_rot: u32, + }, + /// PMAT-3477 (#3090): single-query decode attention, `head_dim <= 256`. + GdnDecodeAttention { + num_heads: u32, + num_kv_heads: u32, + head_dim: u32, + }, } diff --git a/crates/aprender-serve/src/cuda/kernels_generate_gemm_cuda.rs b/crates/aprender-serve/src/cuda/kernels_generate_gemm_cuda.rs index 6d88120120..1b74153ad3 100644 --- a/crates/aprender-serve/src/cuda/kernels_generate_gemm_cuda.rs +++ b/crates/aprender-serve/src/cuda/kernels_generate_gemm_cuda.rs @@ -288,6 +288,52 @@ impl CudaKernels { fn generate_norm_rope_ptx(kernel_type: &KernelType, target: &str) -> Option { Self::generate_rmsnorm_ptx(kernel_type, target) .or_else(|| Self::generate_rope_residual_ptx(kernel_type, target)) + .or_else(|| Self::generate_gdn_ptx(kernel_type, target)) + } + + /// PMAT-3477 (#3090): PTX for the six Gated `DeltaNet` kernels. + /// + /// Its own function rather than an arm of `generate_rmsnorm_ptx` so neither + /// match grows past the complexity ceiling. + fn generate_gdn_ptx(kernel_type: &KernelType, target: &str) -> Option { + use trueno_gpu::kernels::gdn::{ + CausalConv1dSiluKernel, DecodeAttention256Kernel, DeltaRuleRecurrenceKernel, + GatedRmsNormKernel, GdnGatesKernel, PartialNeoxRopeKernel, PerHeadL2NormKernel, + SigmoidGateKernel, SplitInterleavedKernel, + }; + let ptx = match kernel_type { + KernelType::GdnCausalConv1dSilu { channels, kernel_size } => { + CausalConv1dSiluKernel::new(*channels, *kernel_size).emit_ptx_for_target(target) + }, + KernelType::GdnPerHeadL2Norm { head_dim, num_heads, epsilon } => { + PerHeadL2NormKernel::new(*head_dim, *num_heads, *epsilon).emit_ptx_for_target(target) + }, + KernelType::GdnGates { num_heads } => { + GdnGatesKernel::new(*num_heads).emit_ptx_for_target(target) + }, + KernelType::GdnDeltaRule { num_v_heads, head_v_dim } => { + DeltaRuleRecurrenceKernel::new(*num_v_heads, *head_v_dim).emit_ptx_for_target(target) + }, + KernelType::GdnGatedRmsNorm { head_dim, num_heads, epsilon } => { + GatedRmsNormKernel::new(*head_dim, *num_heads, *epsilon).emit_ptx_for_target(target) + }, + KernelType::GdnSigmoidGate { n } => { + SigmoidGateKernel::new(*n).emit_ptx_for_target(target) + }, + KernelType::GdnSplitInterleaved { num_heads, head_dim } => { + SplitInterleavedKernel::new(*num_heads, *head_dim).emit_ptx_for_target(target) + }, + KernelType::GdnPartialNeoxRope { num_heads, head_dim, n_rot } => { + PartialNeoxRopeKernel::new(*num_heads, *head_dim, *n_rot) + .emit_ptx_for_target(target) + }, + KernelType::GdnDecodeAttention { num_heads, num_kv_heads, head_dim } => { + DecodeAttention256Kernel::new(*num_heads, *num_kv_heads, *head_dim) + .emit_ptx_for_target(target) + }, + _ => return None, + }; + Some(ptx) } /// Generate PTX for activation, fusion, and miscellaneous kernels diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs new file mode 100644 index 0000000000..b56ebc0e52 --- /dev/null +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs @@ -0,0 +1,1485 @@ +//! PMAT-3477 / aprender#3090: the Qwen3.5 Gated `DeltaNet` layer on the GPU. +//! +//! [`Qwen35CudaModel`] is built from the CPU [`Qwen35Model`] plus a +//! [`CudaExecutor`]: every `DeltaNet` layer's quantized projections go into the +//! executor's cached-weight store and its f32 vectors into device buffers, and +//! [`Qwen35CudaModel::forward_deltanet_layer`] reproduces +//! `Qwen35Model::forward_deltanet` operation for operation on a device-resident +//! hidden state. +//! +//! The CPU function is the specification. Op order, verbatim: +//! +//! ```text +//! rms_norm -> attn_qkv GEMV -> causal_conv1d(+SiLU) -> split q|k|v +//! -> per-head L2 on q,k -> ssm_alpha/ssm_beta GEMV -> dt/beta gates +//! -> attn_gate GEMV -> delta rule -> gated rmsnorm -> ssm_out GEMV +//! -> residual -> post_attention_norm -> SwiGLU FFN -> residual +//! ``` +//! +//! The interleaved full-attention layers run here too (PMAT-3477 phase 2), from +//! the same specification — `Qwen35Model::forward_attention`: +//! +//! ```text +//! rms_norm -> attn_q|attn_k|attn_v GEMV -> split q|gate -> per-head RMSNorm on +//! q and k -> partial NEOX rope on q and k -> KV append at `position` +//! -> GQA decode attention -> sigmoid gate -> attn_output GEMV -> residual +//! -> post_attention_norm -> SwiGLU FFN -> residual +//! ``` +//! +//! and [`Qwen35CudaModel::forward_single`] runs a whole token through both layer +//! kinds, the output norm and the `lm_head`. +//! +//! Every state buffer is sized from the config +//! (`num_v_heads * head_v_dim * head_v_dim`, `conv_dim * (conv_kernel - 1)`, +//! `max_seq_len * num_kv_heads * head_dim`) — never from a constant. + +use super::{OwnedQuantizedTensor, RealizarError, Result}; +use crate::cuda::types::WeightQuantType; +use crate::cuda::CudaExecutor; +use crate::gguf::forward_qwen35::{Qwen35Model, Qwen35OwnedLayer}; +use trueno_gpu::driver::GpuBuffer; + +/// Positions the device KV cache holds unless a caller asks for more. +/// +/// The CPU model takes `max_seq_len` at `new_state`; this model takes it at +/// build time because the caches are allocated with the weights. +pub const DEFAULT_MAX_SEQ_LEN: usize = 512; + +/// A quantized projection already resident in the executor's weight cache. +struct CudaQuantWeight { + /// Device pointer into the executor's quantized weight cache. + ptr: u64, + /// The GEMV kernel family this tensor's GGML type binds to. + qtype: WeightQuantType, + /// Output width (`out_dim`). + n: u32, + /// Input width (`in_dim`). + k: u32, +} + +/// One Gated `DeltaNet` layer's device-resident weights. +struct CudaDeltaNetLayer { + attn_norm: GpuBuffer, + conv1d_weight: GpuBuffer, + ssm_a: GpuBuffer, + ssm_dt_bias: GpuBuffer, + ssm_norm_weight: GpuBuffer, + post_attention_norm: GpuBuffer, + attn_qkv: CudaQuantWeight, + ssm_alpha: CudaQuantWeight, + ssm_beta: CudaQuantWeight, + attn_gate: CudaQuantWeight, + ssm_out: CudaQuantWeight, + ffn_gate: CudaQuantWeight, + ffn_up: CudaQuantWeight, + ffn_down: CudaQuantWeight, +} + +/// One full-attention layer's device-resident weights. +struct CudaAttentionLayer { + attn_norm: GpuBuffer, + attn_q_norm: GpuBuffer, + attn_k_norm: GpuBuffer, + post_attention_norm: GpuBuffer, + attn_q: CudaQuantWeight, + attn_k: CudaQuantWeight, + attn_v: CudaQuantWeight, + attn_output: CudaQuantWeight, + ffn_gate: CudaQuantWeight, + ffn_up: CudaQuantWeight, + ffn_down: CudaQuantWeight, +} + +/// One resident layer, of either kind — the GPU mirror of `Qwen35OwnedLayer`. +enum CudaLayer { + DeltaNet(Box), + Attention(Box), +} + +/// Per-sequence Qwen3.5 decode state on the device: one causal-conv window and +/// one recurrent state per Gated `DeltaNet` layer, one K/V cache per +/// full-attention layer, all sized from the config. +pub struct Qwen35CudaState { + conv: Vec>, + ssm: Vec>, + /// `Some((k, v))` for a full-attention layer, `None` for a `DeltaNet` one. + /// Each is `[max_seq_len][num_kv_heads * head_dim]`, the CPU KV cache's + /// layout exactly. + kv: Vec, GpuBuffer)>>, + /// `conv_dim * (conv_kernel - 1)`. + conv_len: usize, + /// `num_v_heads * head_v_dim * head_v_dim`. + ssm_len: usize, + /// `num_kv_heads * head_dim` — one KV cache row. + kv_row: usize, + /// Positions the KV cache can hold. + max_seq_len: usize, + /// KV rows written so far (the CPU cache's `seq_len` after `advance`). + kv_len: usize, +} + +impl Qwen35CudaState { + /// Elements in one layer's causal-conv window. + #[must_use] + pub const fn conv_len(&self) -> usize { + self.conv_len + } + + /// Elements in one layer's recurrent state. + #[must_use] + pub const fn ssm_len(&self) -> usize { + self.ssm_len + } + + /// Elements in one KV cache row (`num_kv_heads * head_dim`). + #[must_use] + pub const fn kv_row(&self) -> usize { + self.kv_row + } + + /// Positions the KV cache can hold. + #[must_use] + pub const fn max_seq_len(&self) -> usize { + self.max_seq_len + } + + /// KV rows written so far. + #[must_use] + pub const fn kv_len(&self) -> usize { + self.kv_len + } + + /// A placeholder holding no device memory. + /// + /// Only ever alive inside [`Qwen35CudaModel::with_own_state`], which swaps + /// the real state back before returning — the layer bodies take the state as + /// an argument, so the model cannot lend it to itself. + const fn detached() -> Self { + Self { + conv: Vec::new(), + ssm: Vec::new(), + kv: Vec::new(), + conv_len: 0, + ssm_len: 0, + kv_row: 0, + max_seq_len: 0, + kv_len: 0, + } + } +} + +/// Scratch device buffers for one `DeltaNet` layer step, allocated once. +struct Qwen35CudaScratch { + normed: GpuBuffer, + conv_in: GpuBuffer, + conv_out: GpuBuffer, + alpha_raw: GpuBuffer, + beta_raw: GpuBuffer, + dt: GpuBuffer, + beta: GpuBuffer, + gate: GpuBuffer, + out_h: GpuBuffer, + ssm_out_in: GpuBuffer, + ssm_out: GpuBuffer, + post_normed: GpuBuffer, + ffn_gate: GpuBuffer, + ffn_up: GpuBuffer, + ffn_act: GpuBuffer, + ffn_down: GpuBuffer, +} + +/// Scratch device buffers for one full-attention layer step, allocated once. +/// +/// `post_normed` and the FFN buffers are shared with [`Qwen35CudaScratch`] — +/// the two layer kinds have the same FFN. +struct Qwen35AttnScratch { + /// The joint `[q | gate]` projection, `[num_heads * 2 * head_dim]`. + q_full: GpuBuffer, + /// `q` out of the split, before the per-head norm. + q: GpuBuffer, + /// `q` after the per-head norm (and rotated in place). + q_normed: GpuBuffer, + /// `gate` out of the split. + gate: GpuBuffer, + /// `k` out of the GEMV, before the per-head norm (which writes the cache). + k_raw: GpuBuffer, + /// The attention output before the output projection. + attn_out_in: GpuBuffer, + /// The output projection's result. + attn_out: GpuBuffer, +} + +/// The shapes the layer forward reads, all derived from the model config. +#[derive(Debug, Clone, Copy)] +struct Qwen35CudaDims { + hidden_dim: u32, + intermediate_dim: u32, + conv_dim: u32, + k_dim: u32, + v_dim: u32, + head_k_dim: u32, + num_k_heads: u32, + head_v_dim: u32, + num_v_heads: u32, + conv_kernel: u32, + eps: f32, + /// `config.num_heads` — the full-attention query heads. + num_heads: u32, + /// `config.num_kv_heads`. + num_kv_heads: u32, + /// `attn_q_norm.len()` (256 for the 0.8B file), NOT `hidden_dim / num_heads`. + attn_head_dim: u32, + /// `2 * sum(rope.dimension_sections)`. + n_rot: u32, + /// `freq_base.powf(-2.0 / n_rot)`, computed on the HOST exactly as the CPU + /// reference does — see `gdn_partial_neox_rope_into`. + theta_scale: f32, + vocab_size: u32, +} + +/// Qwen3.5's Gated `DeltaNet` block, resident on one CUDA device (#3090). +/// +/// Built from the CPU [`Qwen35Model`], which stays the specification and the +/// parity reference. +pub struct Qwen35CudaModel<'a> { + model: &'a Qwen35Model<'a>, + executor: CudaExecutor, + layers: Vec, + state: Qwen35CudaState, + scratch: Qwen35CudaScratch, + attn_scratch: Qwen35AttnScratch, + /// The final `output_norm` gamma, device-resident — `base.output_norm_weight()`. + output_norm: GpuBuffer, + /// The `lm_head` (`output.weight`) in the executor's quantized weight cache. + lm_head: CudaQuantWeight, + /// `[hidden_dim]`, the output norm's result. Feeds the `lm_head` GEMV and is + /// never read by the host. + out_normed: GpuBuffer, + /// `[vocab_size]`, the logits — the ONE buffer a token's forward downloads. + logits_buf: GpuBuffer, + dims: Qwen35CudaDims, + /// Positions the device KV caches hold. + max_seq_len: usize, +} + +/// Map a GPU error into the crate error type with the operation that raised it. +fn gpu_err(operation: &str, e: &trueno_gpu::GpuError) -> RealizarError { + RealizarError::UnsupportedOperation { + operation: operation.to_string(), + reason: format!("{e}"), + } +} + +impl<'a> Qwen35CudaModel<'a> { + /// Upload one quantized projection into the executor's cache and bind its + /// GEMV kernel family. + fn upload_quant( + executor: &mut CudaExecutor, + name: &str, + tensor: &OwnedQuantizedTensor, + ) -> Result { + let qtype = WeightQuantType::from_ggml_type(tensor.qtype).ok_or_else(|| { + RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_upload".to_string(), + reason: format!( + "'{name}': GGML type {} has no verified GPU GEMV kernel", + tensor.qtype + ), + } + })?; + if tensor.data.is_empty() { + return Err(RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_upload".to_string(), + reason: format!("'{name}': tensor data is empty (it lay outside the file)"), + }); + } + executor + .load_quantized_weights_with_type(name, &tensor.data, tensor.qtype) + .map_err(|e| gpu_err("qwen35_cuda_upload", &e))?; + let ptr = executor + .get_quantized_weight_ptr(name) + .map_err(|e| gpu_err("qwen35_cuda_upload", &e))?; + Ok(CudaQuantWeight { + ptr, + qtype, + n: u32::try_from(tensor.out_dim).unwrap_or(0), + k: u32::try_from(tensor.in_dim).unwrap_or(0), + }) + } + + /// Upload an f32 vector as a device buffer. + fn upload_f32(executor: &CudaExecutor, name: &str, v: &[f32]) -> Result> { + if v.is_empty() { + return Err(RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_upload".to_string(), + reason: format!("'{name}': f32 vector is empty"), + }); + } + GpuBuffer::from_host(executor.context(), v).map_err(|e| gpu_err("qwen35_cuda_upload", &e)) + } + + /// Zero-filled device buffer of `len` f32. + fn zeros(executor: &CudaExecutor, len: usize) -> Result> { + GpuBuffer::from_host(executor.context(), &vec![0.0f32; len]) + .map_err(|e| gpu_err("qwen35_cuda_alloc", &e)) + } + + /// The shapes, read from the model — never hard-coded. + fn dims_of(model: &Qwen35Model<'_>) -> Qwen35CudaDims { + let k_dim = model.head_k_dim * model.num_k_heads; + let v_dim = model.head_v_dim * model.num_v_heads; + let n_rot = 2 * model.rope_sections.iter().sum::(); + // The CPU computes theta_scale once per call with f32::powf; the kernel + // must be given THAT value, not one derived on the device. + let theta_scale = trueno_gpu::kernels::gdn::PartialNeoxRopeKernel::new( + model.base.config.num_heads as u32, + model.head_dim as u32, + n_rot as u32, + ) + .theta_scale(model.base.config.rope_theta); + Qwen35CudaDims { + num_heads: model.base.config.num_heads as u32, + num_kv_heads: model.base.config.num_kv_heads as u32, + attn_head_dim: model.head_dim as u32, + n_rot: n_rot as u32, + theta_scale, + vocab_size: model.base.config.vocab_size as u32, + hidden_dim: model.base.config.hidden_dim as u32, + intermediate_dim: model.base.config.intermediate_dim as u32, + conv_dim: (k_dim * 2 + v_dim) as u32, + k_dim: k_dim as u32, + v_dim: v_dim as u32, + head_k_dim: model.head_k_dim as u32, + num_k_heads: model.num_k_heads as u32, + head_v_dim: model.head_v_dim as u32, + num_v_heads: model.num_v_heads as u32, + conv_kernel: model.conv_kernel as u32, + eps: model.base.config.eps, + } + } + + /// The delta rule reads q, k and v with one head stride, so a file whose key + /// heads differ from its value heads cannot run this kernel — refuse rather + /// than index past a head. + fn check_head_symmetry(d: Qwen35CudaDims) -> Result<()> { + if d.head_k_dim == d.head_v_dim && d.num_k_heads == d.num_v_heads { + return Ok(()); + } + Err(RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_deltanet".to_string(), + reason: format!( + "the delta-rule kernel indexes q/k/v with one head stride: \ + head_k_dim {} != head_v_dim {} or num_k_heads {} != num_v_heads {}", + d.head_k_dim, d.head_v_dim, d.num_k_heads, d.num_v_heads + ), + }) + } + + /// Upload one `DeltaNet` layer's tensors. + fn build_layer( + executor: &mut CudaExecutor, + il: usize, + d: &crate::gguf::forward_qwen35::Qwen35OwnedDeltaNetLayer, + ) -> Result { + let q = |e: &mut CudaExecutor, suffix: &str, t: &OwnedQuantizedTensor| { + Self::upload_quant(e, &format!("qwen35.blk.{il}.{suffix}"), t) + }; + Ok(CudaDeltaNetLayer { + attn_norm: Self::upload_f32(executor, "attn_norm", &d.attn_norm)?, + conv1d_weight: Self::upload_f32(executor, "ssm_conv1d", &d.ssm_conv1d_weight)?, + ssm_a: Self::upload_f32(executor, "ssm_a", &d.ssm_a)?, + ssm_dt_bias: Self::upload_f32(executor, "ssm_dt_bias", &d.ssm_dt_bias)?, + ssm_norm_weight: Self::upload_f32(executor, "ssm_norm", &d.ssm_norm_weight)?, + post_attention_norm: Self::upload_f32( + executor, + "post_attention_norm", + &d.post_attention_norm, + )?, + attn_qkv: q(executor, "attn_qkv.weight", &d.attn_qkv)?, + ssm_alpha: q(executor, "ssm_alpha.weight", &d.ssm_alpha)?, + ssm_beta: q(executor, "ssm_beta.weight", &d.ssm_beta)?, + attn_gate: q(executor, "attn_gate.weight", &d.attn_gate)?, + ssm_out: q(executor, "ssm_out.weight", &d.ssm_out)?, + ffn_gate: q(executor, "ffn_gate.weight", &d.ffn_gate)?, + ffn_up: q(executor, "ffn_up.weight", &d.ffn_up)?, + ffn_down: q(executor, "ffn_down.weight", &d.ffn_down)?, + }) + } + + /// Upload one full-attention layer's tensors. + fn build_attention_layer( + executor: &mut CudaExecutor, + il: usize, + a: &crate::gguf::forward_qwen35::Qwen35OwnedAttentionLayer, + ) -> Result { + let q = |e: &mut CudaExecutor, suffix: &str, t: &OwnedQuantizedTensor| { + Self::upload_quant(e, &format!("qwen35.blk.{il}.{suffix}"), t) + }; + Ok(CudaAttentionLayer { + attn_norm: Self::upload_f32(executor, "attn_norm", &a.attn_norm)?, + attn_q_norm: Self::upload_f32(executor, "attn_q_norm", &a.attn_q_norm)?, + attn_k_norm: Self::upload_f32(executor, "attn_k_norm", &a.attn_k_norm)?, + post_attention_norm: Self::upload_f32( + executor, + "post_attention_norm", + &a.post_attention_norm, + )?, + attn_q: q(executor, "attn_q.weight", &a.attn_q)?, + attn_k: q(executor, "attn_k.weight", &a.attn_k)?, + attn_v: q(executor, "attn_v.weight", &a.attn_v)?, + attn_output: q(executor, "attn_output.weight", &a.attn_output)?, + ffn_gate: q(executor, "ffn_gate.weight", &a.ffn_gate)?, + ffn_up: q(executor, "ffn_up.weight", &a.ffn_up)?, + ffn_down: q(executor, "ffn_down.weight", &a.ffn_down)?, + }) + } + + /// Allocate the scratch buffers one full-attention layer step needs. + fn build_attn_scratch(executor: &CudaExecutor, d: Qwen35CudaDims) -> Result { + let q_dim = (d.num_heads * d.attn_head_dim) as usize; + let kv_dim = (d.num_kv_heads * d.attn_head_dim) as usize; + Ok(Qwen35AttnScratch { + q_full: Self::zeros(executor, q_dim * 2)?, + q: Self::zeros(executor, q_dim)?, + q_normed: Self::zeros(executor, q_dim)?, + gate: Self::zeros(executor, q_dim)?, + k_raw: Self::zeros(executor, kv_dim)?, + attn_out_in: Self::zeros(executor, q_dim)?, + attn_out: Self::zeros(executor, d.hidden_dim as usize)?, + }) + } + + /// Allocate the scratch buffers one layer step needs. + fn build_scratch(executor: &CudaExecutor, d: Qwen35CudaDims) -> Result { + let hidden = d.hidden_dim as usize; + let inter = d.intermediate_dim as usize; + let conv = d.conv_dim as usize; + let v = d.v_dim as usize; + let nv = d.num_v_heads as usize; + Ok(Qwen35CudaScratch { + normed: Self::zeros(executor, hidden)?, + conv_in: Self::zeros(executor, conv)?, + conv_out: Self::zeros(executor, conv)?, + alpha_raw: Self::zeros(executor, nv)?, + beta_raw: Self::zeros(executor, nv)?, + dt: Self::zeros(executor, nv)?, + beta: Self::zeros(executor, nv)?, + gate: Self::zeros(executor, v)?, + out_h: Self::zeros(executor, v)?, + ssm_out_in: Self::zeros(executor, v)?, + ssm_out: Self::zeros(executor, hidden)?, + post_normed: Self::zeros(executor, hidden)?, + ffn_gate: Self::zeros(executor, inter)?, + ffn_up: Self::zeros(executor, inter)?, + ffn_act: Self::zeros(executor, inter)?, + ffn_down: Self::zeros(executor, hidden)?, + }) + } + + /// Build the GPU model from the CPU model and an executor. + /// + /// Uploads every `DeltaNet` layer's quantized projections into the + /// executor's cache and its f32 vectors into device buffers, and allocates + /// the per-layer state from the config. + /// + /// # Errors + /// A tensor whose GGML type has no GPU GEMV kernel, an empty tensor, a + /// key/value head mismatch the delta-rule kernel cannot index, or any CUDA + /// allocation failure. + pub fn new(model: &'a Qwen35Model<'a>, executor: CudaExecutor) -> Result { + Self::with_max_seq_len(model, executor, DEFAULT_MAX_SEQ_LEN) + } + + /// Build the GPU model with room for `max_seq_len` KV cache positions. + /// + /// # Errors + /// As [`Self::new`], plus a `max_seq_len` of zero. + pub fn with_max_seq_len( + model: &'a Qwen35Model<'a>, + mut executor: CudaExecutor, + max_seq_len: usize, + ) -> Result { + let dims = Self::dims_of(model); + Self::check_head_symmetry(dims)?; + if max_seq_len == 0 { + return Err(RealizarError::InvalidShape { + reason: "qwen35_cuda: max_seq_len must be at least 1".to_string(), + }); + } + + // PRODUCTION DEFAULT, not a test affordance (PMAT-3477 / #3090): this + // architecture runs the FLOAT Q4_K/Q6_K GEMV kernels, never the DP4A + // ones `GpuProfile::detect` picks for a dense decode. The DP4A kernels + // quantize the ACTIVATION to int8, and Qwen3.5 feeds its projections + // straight into a recurrence, which compounds that error instead of + // absorbing it. Measured on the real 0.8B file: with the float variants + // pinned, a whole DeltaNet layer's output is 0.000 relative from a + // second float run and inside the layer budget against the CPU; with + // `HwDp4a` the DeltaNet-only path lands **1.656 relative** away, and the + // end-to-end argmax is garbage — a wrong token at position 0, not a + // rounding difference. The falsifier lives in the tests file + // (`qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence`). + // + // Recovering the DP4A throughput for this architecture (a higher- + // precision activation quantization, or DP4A only on the layers that do + // not feed the recurrence) is the DP4A-through-recurrence ticket, + // 0.69.0. Until it lands, correctness is not optional here. + Self::pin_float_gemv(&mut executor.gpu_profile); + + let mut layers = Vec::with_capacity(model.layers.len()); + for (il, layer) in model.layers.iter().enumerate() { + layers.push(match layer { + Qwen35OwnedLayer::DeltaNet(d) => { + CudaLayer::DeltaNet(Box::new(Self::build_layer(&mut executor, il, d)?)) + }, + Qwen35OwnedLayer::Attention(a) => CudaLayer::Attention(Box::new( + Self::build_attention_layer(&mut executor, il, a)?, + )), + }); + } + + // The tail (output norm + lm_head) runs HERE, on the device buffer the + // layers left the hidden state in — not through `hidden_to_logits`, + // which would take the hidden state down to the host and put it back. + // The gamma is ours; the lm_head goes in the executor's weight cache + // under the name its own path also expects. + let output_norm = + Self::upload_f32(&executor, "output_norm", model.base.output_norm_weight())?; + executor + .preload_output_norm(model.base.output_norm_weight()) + .map_err(|e| gpu_err("qwen35_cuda_upload", &e))?; + let lm_head = + Self::upload_quant(&mut executor, "output.weight", model.base.lm_head_weight())?; + if lm_head.n != dims.vocab_size || lm_head.k != dims.hidden_dim { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda: the lm_head is {}x{}, the config says vocab {} x hidden {}", + lm_head.n, lm_head.k, dims.vocab_size, dims.hidden_dim + ), + }); + } + + // The DP4A GEMV kernels quantize the activation into + // `workspace.q8_activation_buf`, which `init_workspace` sizes from + // `max(hidden_dim, intermediate_dim, q_dim)`. Our widest GEMV input is + // `v_dim` (the ssm_out projection) or `num_heads * attn_head_dim` (the + // attn_output projection), neither of which is one of those — pass the + // widest as the "hidden" width so the buffer covers every input this + // model feeds a kernel. Without this the Q6_K path panics on an + // uninitialized buffer. + let widest_gemv_input = dims + .hidden_dim + .max(dims.v_dim) + .max(dims.num_heads * dims.attn_head_dim) as usize; + executor + .init_workspace(widest_gemv_input, dims.intermediate_dim as usize) + .map_err(|e| gpu_err("qwen35_cuda_workspace", &e))?; + + let scratch = Self::build_scratch(&executor, dims)?; + let attn_scratch = Self::build_attn_scratch(&executor, dims)?; + let out_normed = Self::zeros(&executor, dims.hidden_dim as usize)?; + let logits_buf = Self::zeros(&executor, dims.vocab_size as usize)?; + let state = Self::build_state(&executor, &layers, dims, max_seq_len)?; + Ok(Self { + model, + executor, + layers, + state, + scratch, + attn_scratch, + output_norm, + lm_head, + out_normed, + logits_buf, + dims, + max_seq_len, + }) + } + + /// Allocate a fresh decode state: every layer's conv window and recurrent + /// state, and a K/V cache for each full-attention layer. + fn build_state( + executor: &CudaExecutor, + layers: &[CudaLayer], + dims: Qwen35CudaDims, + max_seq_len: usize, + ) -> Result { + let conv_len = (dims.conv_dim * (dims.conv_kernel - 1)) as usize; + let ssm_len = (dims.num_v_heads * dims.head_v_dim * dims.head_v_dim) as usize; + let kv_row = (dims.num_kv_heads * dims.attn_head_dim) as usize; + let mut conv = Vec::with_capacity(layers.len()); + let mut ssm = Vec::with_capacity(layers.len()); + let mut kv = Vec::with_capacity(layers.len()); + for layer in layers { + conv.push(Self::zeros(executor, conv_len)?); + ssm.push(Self::zeros(executor, ssm_len)?); + kv.push(match layer { + CudaLayer::DeltaNet(_) => None, + CudaLayer::Attention(_) => Some(( + Self::zeros(executor, kv_row * max_seq_len)?, + Self::zeros(executor, kv_row * max_seq_len)?, + )), + }); + } + Ok(Qwen35CudaState { + conv, + ssm, + kv, + conv_len, + ssm_len, + kv_row, + max_seq_len, + kv_len: 0, + }) + } + + /// A fresh decode state, independent of the model's own. + /// + /// The GPU analogue of `Qwen35Model::new_state`; `max_seq_len` is the one + /// this model was built with. + /// + /// # Errors + /// Any CUDA allocation failure. + pub fn new_state(&self) -> Result { + Self::build_state(&self.executor, &self.layers, self.dims, self.max_seq_len) + } + + /// Run `f` with the model's own state detached. + /// + /// The layer bodies take `&mut Qwen35CudaState` so a caller can drive an + /// external state ([`Self::forward_single`]); a method that uses the + /// model's own state cannot pass `&mut self.state` and `&mut self` at once, + /// so it swaps the state out for the call and back after. + fn with_own_state(&mut self, f: impl FnOnce(&mut Self, &mut Qwen35CudaState) -> R) -> R { + let mut state = std::mem::replace(&mut self.state, Qwen35CudaState::detached()); + let out = f(self, &mut state); + self.state = state; + out + } + + /// The device state buffers (their configured lengths). + #[must_use] + pub const fn state(&self) -> &Qwen35CudaState { + &self.state + } + + /// The executor, for a caller that needs to synchronise or inspect it. + pub fn executor_mut(&mut self) -> &mut CudaExecutor { + &mut self.executor + } + + /// Pin the FLOAT (non-DP4A) Q4_K / Q6_K GEMV kernels for this model. + /// + /// The CPU reference dequantizes to f32 and accumulates in f32; the DP4A + /// kernels quantize the activation to int8 first, which is an intentional + /// and already-measured approximation (the load-time parity gate budgets + /// cosine 0.9887 for it versus 0.9999 for the float path). Measured here on + /// the real 0.8B file, the DP4A path puts the `DeltaNet` layer output + /// 1.1e-2 relative from the CPU while the float path is inside 1e-3 — so a + /// test that means to measure the Gated `DeltaNet` kernels, and not the GEMV + /// quantization choice, pins this first. + /// + /// This is ALSO what [`Self::with_max_seq_len`] does at build time for + /// every model of this architecture (see the comment there and + /// [`Self::gemv_variants`]); the method stays because a test that wants to + /// say "the float GEMV, explicitly" should be able to, and because a caller + /// that has re-armed DP4A on the executor can get back to the pinned state. + pub fn pin_reference_gemv(&mut self) { + Self::pin_float_gemv(&mut self.executor.gpu_profile); + } + + /// Set the float (non-DP4A) Q4_K / Q6_K variants on a profile. + fn pin_float_gemv(profile: &mut crate::cuda::gpu_profile::GpuProfile) { + profile.q4k = crate::cuda::gpu_profile::Q4kVariant::Mwv; + profile.q6k = crate::cuda::gpu_profile::Q6kVariant::Mwv; + } + + /// The Q4_K and Q6_K GEMV variants this model will actually dispatch. + /// + /// A freshly built model reports the float pair — see the pinning comment + /// in [`Self::with_max_seq_len`]. A caller that overrides the executor's + /// profile afterwards sees its own choice here, which is what the DP4A + /// falsifier test reads. + #[must_use] + pub const fn gemv_variants( + &self, + ) -> ( + crate::cuda::gpu_profile::Q4kVariant, + crate::cuda::gpu_profile::Q6kVariant, + ) { + (self.executor.gpu_profile.q4k, self.executor.gpu_profile.q6k) + } + + /// Read layer `il`'s state back to the host — the parity tests' observation + /// point. What comes back depends on the layer kind: + /// + /// | layer | pair | + /// |-------|------| + /// | Gated `DeltaNet` | `(causal-conv window, recurrent state)` | + /// | full attention | `(K rows written so far, V rows written so far)`, each `kv_len * kv_row` long | + /// + /// An attention layer at position 0 of a fresh state therefore returns two + /// EMPTY vectors, not two zero-filled ones — nothing has been written. + /// + /// # Errors + /// A device-to-host copy failure, or a layer index out of range. + pub fn download_layer(&mut self, il: usize) -> Result<(Vec, Vec)> { + match self.layers.get(il) { + Some(CudaLayer::DeltaNet(_)) => {}, + Some(CudaLayer::Attention(_)) => return self.download_attention_layer(il), + None => { + return Err(RealizarError::InvalidShape { + reason: format!("qwen35_cuda: layer {il} is out of range"), + }) + }, + } + self.executor + .sync_stream() + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + let mut conv = vec![0.0f32; self.state.conv_len]; + let mut ssm = vec![0.0f32; self.state.ssm_len]; + self.state.conv[il] + .copy_to_host(&mut conv) + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + self.state.ssm[il] + .copy_to_host(&mut ssm) + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + Ok((conv, ssm)) + } + + /// The K and V rows written so far for full-attention layer `il`. + fn download_attention_layer(&mut self, il: usize) -> Result<(Vec, Vec)> { + self.executor + .sync_stream() + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + let n = self.state.kv_len * self.state.kv_row; + let (kc, vc) = self.state.kv[il] + .as_ref() + .expect("an attention layer owns a KV cache"); + let mut k = vec![0.0f32; n]; + let mut v = vec![0.0f32; n]; + if n > 0 { + kc.copy_to_host_at(&mut k, 0) + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + vc.copy_to_host_at(&mut v, 0) + .map_err(|e| gpu_err("qwen35_cuda_download", &e))?; + } + Ok((k, v)) + } + + /// Write the K and V rows of full-attention layer `il` — teacher forcing + /// for the parity tests. `k` and `v` are whole rows (`kv_row` each) and set + /// the state's `kv_len`. + /// + /// # Errors + /// A length that is not a whole number of rows or exceeds `max_seq_len`, a + /// host-to-device copy failure, or a layer that is not a full-attention one. + pub fn upload_attention_kv(&mut self, il: usize, k: &[f32], v: &[f32]) -> Result<()> { + self.require_attention(il)?; + let row = self.state.kv_row; + if k.len() != v.len() || k.len() % row != 0 || k.len() > row * self.state.max_seq_len { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda upload_attention_kv({il}): expected equal K/V lengths that are a \ + multiple of {row} and at most {}, got {} / {}", + row * self.state.max_seq_len, + k.len(), + v.len() + ), + }); + } + let rows = k.len() / row; + { + let (kc, vc) = self.state.kv[il] + .as_mut() + .expect("require_attention checked this layer"); + if !k.is_empty() { + kc.copy_from_host_at(k, 0) + .map_err(|e| gpu_err("qwen35_cuda_upload_state", &e))?; + vc.copy_from_host_at(v, 0) + .map_err(|e| gpu_err("qwen35_cuda_upload_state", &e))?; + } + } + self.state.kv_len = rows; + Ok(()) + } + + /// Write layer `il`'s causal-conv window and recurrent state — teacher + /// forcing for the parity tests. + /// + /// # Errors + /// A length that does not match the configured state size, a host-to-device + /// copy failure, or a layer index that is not a `DeltaNet` layer. + pub fn upload_layer(&mut self, il: usize, conv: &[f32], ssm: &[f32]) -> Result<()> { + self.require_deltanet(il)?; + if conv.len() != self.state.conv_len || ssm.len() != self.state.ssm_len { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda upload_layer({il}): expected conv {} / ssm {}, got {} / {}", + self.state.conv_len, + self.state.ssm_len, + conv.len(), + ssm.len() + ), + }); + } + self.state.conv[il] + .copy_from_host(conv) + .map_err(|e| gpu_err("qwen35_cuda_upload_state", &e))?; + self.state.ssm[il] + .copy_from_host(ssm) + .map_err(|e| gpu_err("qwen35_cuda_upload_state", &e))?; + Ok(()) + } + + /// Refuse a layer index that is not a resident `DeltaNet` layer. + fn require_deltanet(&self, il: usize) -> Result<()> { + match self.layers.get(il) { + Some(CudaLayer::DeltaNet(_)) => Ok(()), + Some(CudaLayer::Attention(_)) => Err(RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_attention".to_string(), + reason: format!("layer {il} is a full-attention layer, not a Gated DeltaNet one"), + }), + None => Err(RealizarError::InvalidShape { + reason: format!("qwen35_cuda: layer {il} is out of range"), + }), + } + } + + /// Refuse a layer index that is not a resident full-attention layer. + fn require_attention(&self, il: usize) -> Result<()> { + match self.layers.get(il) { + Some(CudaLayer::Attention(_)) => Ok(()), + Some(CudaLayer::DeltaNet(_)) => Err(RealizarError::UnsupportedOperation { + operation: "qwen35_cuda_deltanet".to_string(), + reason: format!("layer {il} is a Gated DeltaNet layer, not a full-attention one"), + }), + None => Err(RealizarError::InvalidShape { + reason: format!("qwen35_cuda: layer {il} is out of range"), + }), + } + } + + /// Run one full-attention layer in place on a device-resident hidden state, + /// appending this token's K and V at `position`. + /// + /// Mirrors `Qwen35Model::forward_attention` operation for operation, on the + /// model's own state. + /// + /// # Errors + /// A kernel launch or GEMV dispatch failure, a layer index that is not a + /// full-attention layer, or a position past the state's `max_seq_len`. + pub fn forward_attention_layer( + &mut self, + il: usize, + hidden: &GpuBuffer, + position: usize, + ) -> Result<()> { + self.require_attention(il)?; + self.with_own_state(|m, s| m.attention_layer(s, il, hidden, position)) + } + + /// The attention body against an explicit state. + fn attention_layer( + &mut self, + state: &mut Qwen35CudaState, + il: usize, + hidden: &GpuBuffer, + position: usize, + ) -> Result<()> { + self.require_attention(il)?; + if position >= state.max_seq_len { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda: position {position} is past the KV cache ({} rows)", + state.max_seq_len + ), + }); + } + self.attention_layer_inner(state, il, hidden, position) + .map_err(|e| gpu_err("qwen35_cuda_attention", &e)) + } + + /// The op-for-op attention body; every error here is a GPU error. + #[allow(clippy::too_many_lines)] + fn attention_layer_inner( + &mut self, + state: &mut Qwen35CudaState, + il: usize, + hidden: &GpuBuffer, + position: usize, + ) -> std::result::Result<(), trueno_gpu::GpuError> { + let d = self.dims; + let CudaLayer::Attention(w) = &self.layers[il] else { + unreachable!("require_attention checked this layer") + }; + let s = &self.scratch; + let a = &self.attn_scratch; + let ex = &mut self.executor; + let q_dim = d.num_heads * d.attn_head_dim; + let kv_dim = d.num_kv_heads * d.attn_head_dim; + let (k_cache, v_cache) = state.kv[il] + .as_ref() + .expect("an attention layer owns a KV cache"); + + // This token's row in the KV cache: the GEMVs and the norm write + // straight into it, which IS the CPU's `kv_cache.append`. + let k_row = Self::view( + k_cache, + u32::try_from(position).unwrap_or(0) * kv_dim, + kv_dim, + ); + let v_row = Self::view( + v_cache, + u32::try_from(position).unwrap_or(0) * kv_dim, + kv_dim, + ); + + // rms_norm(hidden, attn_norm) + ex.rmsnorm_into(hidden, &w.attn_norm, &s.normed, d.hidden_dim, d.eps)?; + + // attn_q . normed -> [q | gate] per head ; attn_k, attn_v -> the cache row + ex.gemv_dispatch( + w.attn_q.qtype, + w.attn_q.ptr, + &s.normed, + &a.q_full, + w.attn_q.n, + w.attn_q.k, + )?; + ex.gemv_dispatch( + w.attn_k.qtype, + w.attn_k.ptr, + &s.normed, + &a.k_raw, + w.attn_k.n, + w.attn_k.k, + )?; + ex.gemv_dispatch( + w.attn_v.qtype, + w.attn_v.ptr, + &s.normed, + &v_row, + w.attn_v.n, + w.attn_v.k, + )?; + + // split [q | gate] per head + ex.gdn_split_interleaved_into(&a.q_full, &a.q, &a.gate, d.num_heads, d.attn_head_dim)?; + + // per-head RMSNorm on q (num_heads) and k (num_kv_heads); k's result is + // written into its cache row. + ex.per_head_rmsnorm_into( + &a.q, + &w.attn_q_norm, + &a.q_normed, + d.attn_head_dim, + d.num_heads, + d.eps, + )?; + ex.per_head_rmsnorm_into( + &a.k_raw, + &w.attn_k_norm, + &k_row, + d.attn_head_dim, + d.num_kv_heads, + d.eps, + )?; + + // partial NEOX rope on q and k, in place, at this position + let pos32 = u32::try_from(position).unwrap_or(u32::MAX); + ex.gdn_partial_neox_rope_into( + &a.q_normed, + d.num_heads, + d.attn_head_dim, + d.n_rot, + pos32, + d.theta_scale, + )?; + ex.gdn_partial_neox_rope_into( + &k_row, + d.num_kv_heads, + d.attn_head_dim, + d.n_rot, + pos32, + d.theta_scale, + )?; + + // GQA decode attention over positions 0..=position + ex.gdn_decode_attention_into( + &a.q_normed, + k_cache, + v_cache, + &a.attn_out_in, + d.num_heads, + d.num_kv_heads, + d.attn_head_dim, + pos32 + 1, + )?; + std::mem::forget(k_row); + std::mem::forget(v_row); + + // the output gate, then the output projection and the first residual + ex.gdn_sigmoid_gate_into(&a.attn_out_in, &a.gate, q_dim)?; + ex.gemv_dispatch( + w.attn_output.qtype, + w.attn_output.ptr, + &a.attn_out_in, + &a.attn_out, + w.attn_output.n, + w.attn_output.k, + )?; + ex.residual_add_into(hidden, &a.attn_out, hidden, d.hidden_dim)?; + + // post_attention_norm -> SwiGLU FFN -> the second residual + ex.rmsnorm_into( + hidden, + &w.post_attention_norm, + &s.post_normed, + d.hidden_dim, + d.eps, + )?; + ex.gemv_dispatch( + w.ffn_gate.qtype, + w.ffn_gate.ptr, + &s.post_normed, + &s.ffn_gate, + w.ffn_gate.n, + w.ffn_gate.k, + )?; + ex.gemv_dispatch( + w.ffn_up.qtype, + w.ffn_up.ptr, + &s.post_normed, + &s.ffn_up, + w.ffn_up.n, + w.ffn_up.k, + )?; + ex.fused_swiglu_into(&s.ffn_gate, &s.ffn_up, &s.ffn_act, d.intermediate_dim)?; + ex.gemv_dispatch( + w.ffn_down.qtype, + w.ffn_down.ptr, + &s.ffn_act, + &s.ffn_down, + w.ffn_down.n, + w.ffn_down.k, + )?; + ex.residual_add_into(hidden, &s.ffn_down, hidden, d.hidden_dim)?; + + // NO sync here (#3090 review). Every op above is enqueued on the one + // stream this model uses, so the next layer's first kernel is already + // ordered after this layer's last one; a sync inside the per-token loop + // only stalls the host. The bookkeeping below is a host-side counter of + // rows the stream has been ASKED to write, which no device read + // observes — the syncs that matter are the ones in front of a host read + // (`download_layer`, `dump_stage`, the logits download). + // + // The row is written: the cache now holds `position + 1` rows. + state.kv_len = state.kv_len.max(position + 1); + Ok(()) + } + + /// Read one named scratch buffer back to the host. + /// + /// The op-by-op observation point: a per-layer L∞ that fails says only THAT + /// the layer diverged, never WHERE, and the twelve intermediates between + /// `attn_qkv` and the second residual are what tell the two apart. + /// + /// # Panics + /// On an unknown stage name, or a failed device-to-host copy. + #[cfg(test)] + pub(crate) fn dump_stage(&mut self, which: &str) -> Vec { + self.executor.sync_stream().expect("sync"); + let s = &self.scratch; + let a = &self.attn_scratch; + let buf = match which { + "q_full" => &a.q_full, + "q" => &a.q, + "q_normed" => &a.q_normed, + "attn_gate" => &a.gate, + "k_raw" => &a.k_raw, + "attn_out_in" => &a.attn_out_in, + "attn_out" => &a.attn_out, + "normed" => &s.normed, + "conv_in" => &s.conv_in, + "conv_out" => &s.conv_out, + "alpha_raw" => &s.alpha_raw, + "beta_raw" => &s.beta_raw, + "dt" => &s.dt, + "beta" => &s.beta, + "gate" => &s.gate, + "out_h" => &s.out_h, + "ssm_out_in" => &s.ssm_out_in, + "ssm_out" => &s.ssm_out, + "post_normed" => &s.post_normed, + "ffn_gate" => &s.ffn_gate, + "ffn_up" => &s.ffn_up, + "ffn_act" => &s.ffn_act, + "ffn_down" => &s.ffn_down, + other => panic!("qwen35_cuda: no scratch stage named '{other}'"), + }; + let mut host = vec![0.0f32; buf.len()]; + buf.copy_to_host(&mut host).expect("download stage"); + host + } + + /// Run attention layer `il`'s `attn_q` projection on a host input vector and + /// read the result back — the observation point for the parity-floor test. + /// + /// The whole-layer and end-to-end comparisons are bounded from below by the + /// projection GEMVs, and "is the GPU wrong or is the reference?" is not + /// answerable from a GPU-vs-CPU number alone. This exposes one GEMV so a + /// test can put BOTH sides against an exact dequantized f64 reference. + /// + /// # Errors + /// Any allocation, dispatch or transfer failure. + #[cfg(test)] + pub(crate) fn attn_q_gemv_of_host_input(&mut self, il: usize, x: &[f32]) -> Result> { + let CudaLayer::Attention(w) = &self.layers[il] else { + unreachable!("an attention layer") + }; + let (qtype, ptr, n, k) = (w.attn_q.qtype, w.attn_q.ptr, w.attn_q.n, w.attn_q.k); + let dev = + GpuBuffer::from_host(self.executor.context(), x).map_err(|e| gpu_err("diag", &e))?; + let out = Self::zeros(&self.executor, n as usize)?; + self.executor + .gemv_dispatch(qtype, ptr, &dev, &out, n, k) + .map_err(|e| gpu_err("diag", &e))?; + self.executor + .sync_stream() + .map_err(|e| gpu_err("diag", &e))?; + let mut host = vec![0.0f32; n as usize]; + out.copy_to_host(&mut host) + .map_err(|e| gpu_err("diag", &e))?; + Ok(host) + } + + /// A non-owning view of `elems` f32 starting `offset` elements into `buf`. + /// + /// The delta rule wants q, k and v as three separate pointers; the conv + /// output holds them contiguously, exactly as the CPU reference slices it. + /// The returned buffer must be `std::mem::forget`-ed — it does not own the + /// allocation. + fn view(buf: &GpuBuffer, offset: u32, elems: u32) -> GpuBuffer { + let ptr = buf.as_ptr() + u64::from(offset) * 4; + // SAFETY: `offset + elems <= buf.len()` at every call site below + // (0/k_dim/2*k_dim into a conv_dim-long buffer), and the view is + // forgotten before it can free memory it does not own. + unsafe { GpuBuffer::::from_raw_parts(ptr, elems as usize) } + } + + /// Run one Gated `DeltaNet` layer in place on a device-resident hidden state. + /// + /// Mirrors `Qwen35Model::forward_deltanet` operation for operation. The + /// layer's conv window and recurrent state are read and written in place, so + /// a caller that wants a specific starting state sets it with + /// [`Self::upload_layer`] first. + /// + /// # Errors + /// A kernel launch or GEMV dispatch failure, or a layer index that is not a + /// `DeltaNet` layer. + pub fn forward_deltanet_layer(&mut self, il: usize, hidden: &GpuBuffer) -> Result<()> { + self.require_deltanet(il)?; + self.with_own_state(|m, s| m.deltanet_layer(s, il, hidden)) + } + + /// The `DeltaNet` body against an explicit state. + fn deltanet_layer( + &mut self, + state: &mut Qwen35CudaState, + il: usize, + hidden: &GpuBuffer, + ) -> Result<()> { + self.require_deltanet(il)?; + self.deltanet_layer_inner(state, il, hidden) + .map_err(|e| gpu_err("qwen35_cuda_deltanet", &e)) + } + + /// The op-for-op body; every error here is a GPU error. + #[allow(clippy::too_many_lines)] + fn deltanet_layer_inner( + &mut self, + state: &Qwen35CudaState, + il: usize, + hidden: &GpuBuffer, + ) -> std::result::Result<(), trueno_gpu::GpuError> { + let d = self.dims; + let CudaLayer::DeltaNet(w) = &self.layers[il] else { + unreachable!("require_deltanet checked this layer") + }; + let s = &self.scratch; + let ex = &mut self.executor; + + // rms_norm(hidden, attn_norm) + ex.rmsnorm_into(hidden, &w.attn_norm, &s.normed, d.hidden_dim, d.eps)?; + + // attn_qkv . normed -> conv_in + ex.gemv_dispatch( + w.attn_qkv.qtype, + w.attn_qkv.ptr, + &s.normed, + &s.conv_in, + w.attn_qkv.n, + w.attn_qkv.k, + )?; + + // causal_conv1d + SiLU (the window is updated in place) + ex.gdn_causal_conv1d_silu_into( + &s.conv_in, + &state.conv[il], + &w.conv1d_weight, + &s.conv_out, + d.conv_dim, + d.conv_kernel, + )?; + + // split q | k | v out of conv_out, then per-head L2 on q and k only + let q_view = Self::view(&s.conv_out, 0, d.k_dim); + let k_view = Self::view(&s.conv_out, d.k_dim, d.k_dim); + let v_view = Self::view(&s.conv_out, d.k_dim * 2, d.v_dim); + ex.gdn_per_head_l2_norm_into(&q_view, d.head_k_dim, d.num_k_heads, d.eps)?; + ex.gdn_per_head_l2_norm_into(&k_view, d.head_k_dim, d.num_k_heads, d.eps)?; + + // Gated DeltaNet applies NO RoPE: position comes from the causal conv. + + // dt = softplus(ssm_alpha . x + dt_bias) * a ; beta = sigmoid(ssm_beta . x) + ex.gemv_dispatch( + w.ssm_alpha.qtype, + w.ssm_alpha.ptr, + &s.normed, + &s.alpha_raw, + w.ssm_alpha.n, + w.ssm_alpha.k, + )?; + ex.gemv_dispatch( + w.ssm_beta.qtype, + w.ssm_beta.ptr, + &s.normed, + &s.beta_raw, + w.ssm_beta.n, + w.ssm_beta.k, + )?; + ex.gdn_gates_into( + &s.alpha_raw, + &w.ssm_dt_bias, + &w.ssm_a, + &s.beta_raw, + &s.dt, + &s.beta, + d.num_v_heads, + )?; + + // attn_gate . x + ex.gemv_dispatch( + w.attn_gate.qtype, + w.attn_gate.ptr, + &s.normed, + &s.gate, + w.attn_gate.n, + w.attn_gate.k, + )?; + + // the delta rule (the recurrent state is updated in place) + ex.gdn_delta_rule_into( + &q_view, + &k_view, + &v_view, + &s.beta, + &s.dt, + &state.ssm[il], + &s.out_h, + d.num_v_heads, + d.head_v_dim, + )?; + std::mem::forget(q_view); + std::mem::forget(k_view); + std::mem::forget(v_view); + + // gated rmsnorm, then ssm_out, then the first residual + ex.gdn_gated_rmsnorm_into( + &s.out_h, + &s.gate, + &w.ssm_norm_weight, + &s.ssm_out_in, + d.head_v_dim, + d.num_v_heads, + d.eps, + )?; + ex.gemv_dispatch( + w.ssm_out.qtype, + w.ssm_out.ptr, + &s.ssm_out_in, + &s.ssm_out, + w.ssm_out.n, + w.ssm_out.k, + )?; + ex.residual_add_into(hidden, &s.ssm_out, hidden, d.hidden_dim)?; + + // post_attention_norm -> SwiGLU FFN -> the second residual + ex.rmsnorm_into( + hidden, + &w.post_attention_norm, + &s.post_normed, + d.hidden_dim, + d.eps, + )?; + ex.gemv_dispatch( + w.ffn_gate.qtype, + w.ffn_gate.ptr, + &s.post_normed, + &s.ffn_gate, + w.ffn_gate.n, + w.ffn_gate.k, + )?; + ex.gemv_dispatch( + w.ffn_up.qtype, + w.ffn_up.ptr, + &s.post_normed, + &s.ffn_up, + w.ffn_up.n, + w.ffn_up.k, + )?; + ex.fused_swiglu_into(&s.ffn_gate, &s.ffn_up, &s.ffn_act, d.intermediate_dim)?; + ex.gemv_dispatch( + w.ffn_down.qtype, + w.ffn_down.ptr, + &s.ffn_act, + &s.ffn_down, + w.ffn_down.n, + w.ffn_down.k, + )?; + ex.residual_add_into(hidden, &s.ffn_down, hidden, d.hidden_dim)?; + + // NO sync here — see `attention_layer_inner`. Stream order IS the + // dependency; the host only has to wait where it reads. + Ok(()) + } + + /// Run one token at `position` through every layer of both kinds, the + /// output norm and the `lm_head`, and return the logits — the GPU twin of + /// `Qwen35Model::forward_single_qwen35`. + /// + /// `state` is updated in place (conv windows, recurrent states and the K/V + /// caches) and its KV length advanced to `position + 1`, exactly as the CPU + /// advances `cache.kv_cache` at the end of the token. + /// + /// # Errors + /// A token id outside the vocabulary, a position past the state's + /// `max_seq_len`, or any kernel / GEMV / transfer failure. + pub fn forward_single( + &mut self, + token: u32, + state: &mut Qwen35CudaState, + position: usize, + ) -> Result> { + let hidden_dim = self.dims.hidden_dim as usize; + let embedding = self.model.base.token_embedding(); + let start = (token as usize) * hidden_dim; + if start + hidden_dim > embedding.len() { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda: token {token} is outside the {}-row embedding table", + embedding.len() / hidden_dim + ), + }); + } + if position >= state.max_seq_len { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda: position {position} is past the KV cache ({} rows)", + state.max_seq_len + ), + }); + } + + let dev = GpuBuffer::from_host( + self.executor.context(), + &embedding[start..start + hidden_dim], + ) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + for il in 0..self.layers.len() { + match self.layers[il] { + CudaLayer::DeltaNet(_) => self.deltanet_layer(state, il, &dev)?, + CudaLayer::Attention(_) => self.attention_layer(state, il, &dev, position)?, + } + } + // The tail stays on the device (#3090 review). `hidden_to_logits` would + // sync, copy `dev` to the host and upload it again; the hidden state is + // already where the output norm wants it, so run the norm into + // `out_normed` and the lm_head GEMV into `logits_buf` on the same + // stream. The CPU reference applies no lm_head bias (`lm_head_bias` is + // None for this architecture — `forward_single_qwen35` goes straight + // from `rms_norm_into` to `fused_matmul_into`), so neither does this. + let d = self.dims; + self.executor + .rmsnorm_into( + &dev, + &self.output_norm, + &self.out_normed, + d.hidden_dim, + d.eps, + ) + .map_err(|e| gpu_err("qwen35_cuda_lm_head", &e))?; + self.executor + .gemv_dispatch( + self.lm_head.qtype, + self.lm_head.ptr, + &self.out_normed, + &self.logits_buf, + self.lm_head.n, + self.lm_head.k, + ) + .map_err(|e| gpu_err("qwen35_cuda_lm_head", &e))?; + + // The ONE sync of the whole token, in front of the ONE download. + self.executor + .sync_stream() + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + let mut logits = vec![0.0f32; d.vocab_size as usize]; + self.logits_buf + .copy_to_host(&mut logits) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + + state.kv_len = state.kv_len.max(position + 1); + Ok(logits) + } + + /// Run every Gated `DeltaNet` layer over a host hidden state and read the + /// result back — the tests' end-to-end handle on the GPU block. + /// + /// Full-attention layers are skipped, so this is NOT a decoder forward + /// pass; it is exactly "the `DeltaNet` layers, in order". + /// + /// # Errors + /// A hidden state of the wrong width, or any per-layer failure. + pub fn forward_hidden_deltanet_only(&mut self, hidden_host: &[f32]) -> Result> { + let hidden_dim = self.dims.hidden_dim as usize; + if hidden_host.len() != hidden_dim { + return Err(RealizarError::InvalidShape { + reason: format!( + "qwen35_cuda: hidden state is {} wide, the model's is {hidden_dim}", + hidden_host.len() + ), + }); + } + let dev = GpuBuffer::from_host(self.executor.context(), hidden_host) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + for il in 0..self.model.layers.len() { + if matches!(self.model.layers[il], Qwen35OwnedLayer::DeltaNet(_)) { + self.forward_deltanet_layer(il, &dev)?; + } + } + self.executor + .sync_stream() + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + let mut out = vec![0.0f32; hidden_dim]; + dev.copy_to_host(&mut out) + .map_err(|e| gpu_err("qwen35_cuda_forward", &e))?; + Ok(out) + } +} + +/// Per-layer CPU parity on the real Qwen3.5-0.8B file. +#[cfg(test)] +#[path = "forward_qwen35_cuda_tests.rs"] +mod qwen35_cuda_tests; diff --git a/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs new file mode 100644 index 0000000000..116359c2d5 --- /dev/null +++ b/crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs @@ -0,0 +1,1302 @@ +//! PMAT-3477 / aprender#3090: CPU parity for [`Qwen35CudaModel`] on the real +//! Qwen3.5-0.8B file, at three scopes that must not be confused. +//! +//! 1. [`qwen35_cuda_gdn_ops_match_cpu_given_the_same_inputs`] — the six Gated +//! `DeltaNet` kernels, each fed the GPU's OWN input and compared against the +//! CPU reference function on that same input, at **1e-3 relative**. This is +//! the kernel contract, and it is what the beta-sign mutant falsifies. +//! 2. [`qwen35_cuda_deltanet_layers_match_cpu_on_the_real_file`] — the whole +//! wired layer, every `DeltaNet` layer, three tokens, teacher-forced. This +//! one also runs the **projection GEMVs**, and those are a separate, +//! pre-existing CPU/GPU approximation: measured op by op on layer 0 (the +//! diagnostic that produced these numbers is the op-by-op test above), +//! +//! | stage | qtype | relative L∞ | +//! |-------|-------|-------------| +//! | `rms_norm` -> `normed` | f32 | 9.1e-8 | +//! | `attn_qkv` GEMV | Q5_K | 6.6e-7 | +//! | conv+SiLU+per-head L2 | — | 6.0e-7 | +//! | `ssm_alpha` GEMV | Q8_0 | **5.3e-3** | +//! | `ssm_beta` GEMV | Q8_0 | **2.4e-3** | +//! | `attn_gate` GEMV | Q4_K | **4.8e-3** | +//! +//! so the layer output cannot be inside 1e-3 and the GEMV kernels be what +//! they are today. [`LAYER_TOL`] is the measured budget, not a wish; the +//! kernels this ticket adds are held to [`TOL`]. +//! +//! 3. the two WHOLE-FORWARD comparisons against the CPU's production forward +//! ([`qwen35_cuda_attention_layers_match_cpu_on_the_real_file`] and +//! [`qwen35_cuda_forward_single_matches_cpu_logits_end_to_end`]) — these +//! cannot be held to a tight L∞ at all, because the reference is the noisier +//! side (it quantizes its activation to Q8_K). They assert identical argmax +//! + cosine >= [`COSINE_FLOOR`] + an L∞ BUDGET. Read [`COSINE_FLOOR`] before +//! touching any number in this file: it carries the measurements, and the +//! difference between a bar and a budget is the whole point. +//! +//! Every comparison is **scale-relative** L∞ — `max|gpu - cpu| <= tol * +//! max|cpu|` — and asserts `max|cpu| > 0` first. An absolute 1e-3 on these +//! tensors is vacuous: the delta-rule outputs are ~5e-3 (q and k are unit-norm +//! per head and the recurrence scales by `D^-0.5`), so the whole signal sits +//! inside the tolerance and even a 1 % mutation stays GREEN. + +use super::Qwen35CudaModel; +use crate::gguf::forward_qwen35::{ + causal_conv1d, delta_rule_recurrence, gated_rmsnorm, l2_norm_per_head, silu, softplus, + Qwen35Model, Qwen35OwnedDeltaNetLayer, Qwen35OwnedLayer, +}; +use trueno_gpu::driver::GpuBuffer; + +/// The real hybrid file this phase is specified against. +const MODEL_PATH: &str = "/home/noah/models/Qwen3.5-0.8B-Q4_K_M.gguf"; + +/// Tolerance for the Gated `DeltaNet` kernels themselves. They route exp/ln +/// through `ex2.approx`/`lg2.approx` (~2 ulp), so 1e-4 is not available; 1e-3 +/// relative is the contract the kernel module states. +const TOL: f32 = 1e-3; + +/// Tolerance for the whole wired layer, which also runs the Q4_K/Q8_0 +/// projection GEMVs. See the module docs: those contribute ~5e-3 relative on +/// their own, before the layer's own arithmetic. Measured max over the three +/// tokens and every `DeltaNet` layer: 9.8e-3 with the float GEMV pinned. +const LAYER_TOL: f32 = 2e-2; + +/// Tolerance for the K and V rows an attention layer appends, against the +/// EXACT-arithmetic reference ([`exact_gemv`] + the CPU's own norm and rope, +/// which are unquantized ops). One GEMV plus a per-head RMSNorm plus the +/// partial rope away from the layer input, with no quantized activation on +/// either side, so this is the kernel contract [`TOL`], not a budget. +const KV_TOL: f32 = TOL; + +/// THE PARITY CONTRACT FOR A WHOLE-FORWARD COMPARISON (PMAT-3477 / #3090). +/// +/// The per-op tests above are held to [`TOL`] against an EXACT reference +/// ([`exact_gemv`]) wherever the CPU production op quantizes its activation. +/// The two comparisons that run the CPU's own production forward — +/// [`qwen35_cuda_attention_layers_match_cpu_on_the_real_file`] and +/// [`qwen35_cuda_forward_single_matches_cpu_logits_end_to_end`] — cannot be, and +/// `qwen35_cuda_the_parity_floor_is_the_cpu_references_own_activation_quantization` +/// says why with numbers: on the same projection the GPU float GEMV is 9.96e-8 +/// from exact arithmetic and the CPU `fused_matmul_into` is 2.95e-3, because it +/// quantizes the ACTIVATION to Q8_K. The CPU is the noisier side, so a tight +/// L∞ against it is not a correctness bar — it is a demand that the GPU +/// reproduce the reference's own rounding. +/// +/// So those two tests assert three things instead, all of which can fail: +/// +/// 1. **identical argmax at every position** — exact, and the only property a +/// greedy decode actually consumes; +/// 2. **cosine >= [`COSINE_FLOOR`]** — direction, which an error that is pure +/// GEMV rounding barely moves and a wrong kernel destroys; +/// 3. **relative L∞ <= [`LOGITS_BUDGET`] / [`LAYER_BUDGET`]** — a budget, held +/// as a regression tripwire. It is a reading, never a claim of exactness. +/// +/// MEASURED, on the real 0.8B file with the float GEMV pinned — the tests print +/// every one of these lines, so the constants below are a reading: +/// +/// | e2e position | 0 | 1 | 2 | 3 | 4 | 5 | +/// |---|---|---|---|---|---|---| +/// | cosine | **0.998348** | 0.999487 | 0.998936 | 0.999510 | 0.999309 | 0.999636 | +/// | relative L∞ | **6.721e-2** | 2.123e-2 | 3.490e-2 | 2.338e-2 | 3.065e-2 | 1.972e-2 | +/// | argmax (both sides) | 13 | 198 | 11 | 198 | 0 | 908 | +/// +/// and over the whole-attention-layer comparison (18 comparisons: every +/// attention layer × three positions) the worst is layer 15 at position 0 — +/// cosine **0.997419**, relative L∞ **4.867e-2** — with every other comparison +/// at cosine ≥ 0.999407. +/// +/// `COSINE_FLOOR` is the minimum over BOTH tests (0.997419) rounded DOWN to +/// three decimals — 0.997 — minus 0.001: **0.996**. That leaves ~1.4e-3 of +/// cosine margin under the worst comparison, and is still nowhere near what a +/// broken kernel produces: the DP4A falsifier below reads 0.466 down to +/// **-0.246**, so the assertion discriminates rather than decorates. +const COSINE_FLOOR: f32 = 0.996; + +/// The measured end-to-end logit L∞ — 6.721e-2 relative, at position 0 — +/// rounded UP to one significant figure. A budget on the accumulated +/// CPU-reference activation quantization through 24 layers and the `lm_head`, +/// not a tolerance anyone should read as accuracy: see [`COSINE_FLOOR`]. +const LOGITS_BUDGET: f32 = 7e-2; + +/// The same, for one whole attention layer's output hidden state: measured +/// 4.867e-2 relative (layer 15, position 0), rounded up to one significant +/// figure. +const LAYER_BUDGET: f32 = 5e-2; + +/// A fixed three-token prompt. Any ids work for parity — what matters is that +/// both sides see the same ones. +const PROMPT: [u32; 3] = [9707, 11, 1879]; + +/// A fixed six-token prompt for the end-to-end comparison. +const LONG_PROMPT: [u32; 6] = [9707, 11, 1879, 0, 2610, 525]; + +/// `max|want|`, the scale the tolerance is relative to. +fn max_abs(v: &[f32]) -> f32 { + v.iter().fold(0.0f32, |m, x| m.max(x.abs())) +} + +/// `max|got - want| / max|want|` — the scale-relative L∞ every assertion here +/// is written in. +fn rel_linf(got: &[f32], want: &[f32]) -> f32 { + assert_eq!(got.len(), want.len(), "rel_linf: length mismatch"); + let scale = max_abs(want); + assert!(scale > 0.0, "rel_linf: the reference is all zeros"); + got.iter() + .zip(want) + .fold(0.0f32, |m, (g, w)| m.max((g - w).abs())) + / scale +} + +/// Assert `max|got - want| <= tol * max|want|`, refusing a vacuous comparison. +fn assert_rel_linf(got: &[f32], want: &[f32], tol: f32, what: &str) { + assert_eq!(got.len(), want.len(), "{what}: length mismatch"); + let scale = max_abs(want); + assert!( + scale > 0.0, + "{what}: the reference is all zeros — the comparison would pass on anything" + ); + let (mut worst, mut at) = (0.0f32, 0usize); + for (i, (g, w)) in got.iter().zip(want).enumerate() { + let d = (g - w).abs(); + if d > worst { + worst = d; + at = i; + } + } + assert!( + worst <= tol * scale, + "{what}: relative L-inf {:.3e} (abs {:.3e} at [{at}], scale {:.3e}) exceeds {tol:.0e}; \ + cpu={:.6} gpu={:.6}", + worst / scale, + worst, + scale, + want[at], + got[at], + ); +} + +/// Cosine similarity, accumulated in f64 so the metric itself is not the thing +/// being measured. +fn cosine(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len(), "cosine: length mismatch"); + let (mut dot, mut na, mut nb) = (0.0f64, 0.0f64, 0.0f64); + for (x, y) in a.iter().zip(b) { + dot += f64::from(*x) * f64::from(*y); + na += f64::from(*x) * f64::from(*x); + nb += f64::from(*y) * f64::from(*y); + } + assert!( + na > 0.0 && nb > 0.0, + "cosine: a zero vector — the comparison would be undefined" + ); + (dot / (na.sqrt() * nb.sqrt())) as f32 +} + +/// The EXACT reference for one quantized projection: the weight dequantized to +/// f32, the row dots accumulated in **f64**. +/// +/// This is the third reference the parity contract rests on. The CPU +/// production op (`fused_matmul_into`) is NOT it: it quantizes the activation +/// to Q8_K and takes int8 dots, which costs ~3e-3 relative all by itself +/// (measured in the parity-floor test). Any comparison whose reference side +/// runs a quantized GEMV must come here instead, or it measures the reference. +/// +/// Panics on a qtype it has no dequantizer for, rather than silently comparing +/// against something else. +fn exact_gemv(tensor: &crate::gguf::quantized::OwnedQuantizedTensor, x: &[f32]) -> Vec { + use crate::gguf::types::{GGUF_TYPE_Q4_K, GGUF_TYPE_Q5_K, GGUF_TYPE_Q6_K, GGUF_TYPE_Q8_0}; + let w = match tensor.qtype { + GGUF_TYPE_Q4_K => crate::quantize::dequantize_q4_k(&tensor.data), + GGUF_TYPE_Q5_K => crate::quantize::dequantize_q5_k(&tensor.data), + GGUF_TYPE_Q6_K => crate::quantize::dequantize_q6_k(&tensor.data), + GGUF_TYPE_Q8_0 => crate::quantize::dequantize_q8_0(&tensor.data), + other => panic!("exact_gemv: no dequantizer for GGML type {other}"), + } + .expect("dequantize the projection"); + assert_eq!( + x.len(), + tensor.in_dim, + "exact_gemv: the input is not in_dim wide" + ); + assert!( + w.len() >= tensor.out_dim * tensor.in_dim, + "exact_gemv: the dequantized weight is short of out_dim x in_dim" + ); + (0..tensor.out_dim) + .map(|r| { + let row = &w[r * tensor.in_dim..(r + 1) * tensor.in_dim]; + row.iter() + .zip(x) + .fold(0.0f64, |s, (wv, xv)| s + f64::from(*wv) * f64::from(*xv)) as f32 + }) + .collect() +} + +/// The whole-forward parity contract of [`COSINE_FLOOR`], applied at one +/// position: exact argmax equality, cosine above the floor, L∞ inside the +/// budget. Returns `(cosine, relative L∞)` so the caller can print the reading +/// the constants were set from. +fn assert_forward_parity(got: &[f32], want: &[f32], budget: f32, what: &str) -> (f32, f32) { + let cos = cosine(got, want); + let linf = rel_linf(got, want); + assert_eq!( + crate::gguf::ops::argmax(got), + crate::gguf::ops::argmax(want), + "{what}: the GPU and the CPU must pick the same index (gpu {} vs cpu {}, cosine \ + {cos:.6}, relative L-inf {linf:.3e})", + crate::gguf::ops::argmax(got), + crate::gguf::ops::argmax(want), + ); + assert!( + cos >= COSINE_FLOOR, + "{what}: cosine {cos:.6} is below the measured floor {COSINE_FLOOR} — the two sides \ + point in different directions, which GEMV rounding does not do" + ); + assert!( + linf <= budget, + "{what}: relative L-inf {linf:.3e} exceeds the measured budget {budget:.0e}" + ); + (cos, linf) +} + +/// Skip unless both the device and the model file are here. +macro_rules! qwen35_cuda_fixture_or_skip { + () => {{ + if !std::path::Path::new(MODEL_PATH).exists() { + eprintln!("SKIP: {MODEL_PATH} is absent"); + return; + } + crate::cuda_executor_or_skip!(0) + }}; +} + +/// Map the file and build the CPU model. +fn load_cpu_model(mapped: &crate::gguf::MappedGGUFModel) -> crate::gguf::OwnedQuantizedModel { + Qwen35Model::create_base_model(&mapped.model, mapped.data()).expect("base model") +} + +/// Per-layer parity of the whole wired Gated `DeltaNet` GPU forward against the +/// CPU reference, over the first three tokens of a fixed prompt, on EVERY +/// `DeltaNet` layer, with teacher forcing — on the layer output hidden state AND +/// on the causal-conv window and the recurrent state the token leaves behind. +#[test] +#[serial_test::serial] +fn qwen35_cuda_deltanet_layers_match_cpu_on_the_real_file() { + let executor = qwen35_cuda_fixture_or_skip!(); + + // Load exactly as run_qwen35_generate does. + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let deltanet_layers = qwen + .layers + .iter() + .filter(|l| matches!(l, Qwen35OwnedLayer::DeltaNet(_))) + .count(); + assert!( + deltanet_layers > 0, + "the fixture must carry Gated DeltaNet layers, else this test proves nothing" + ); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + // The float (non-DP4A) GEMV is the one that reproduces the CPU's float + // dequant; DP4A quantizes the activation to int8 first and costs another + // ~1e-3 on top (1.1e-2 vs 9.8e-3 measured end to end). + gpu.pin_reference_gemv(); + + let hidden_dim = qwen.base.config.hidden_dim; + let mut cpu_state = qwen.new_state(PROMPT.len() + 1); + let mut normed = vec![0.0f32; hidden_dim]; + let mut post_normed = vec![0.0f32; hidden_dim]; + let mut compared = 0usize; + + for (pos, &token) in PROMPT.iter().enumerate() { + let mut hidden = qwen.base.token_embedding() + [(token as usize) * hidden_dim..(token as usize + 1) * hidden_dim] + .to_vec(); + + for (il, layer) in qwen.layers.iter().enumerate() { + match layer { + Qwen35OwnedLayer::DeltaNet(d) => { + // Teacher forcing: the GPU starts this layer from the CPU's + // conv window and recurrent state, and from the same hidden. + gpu.upload_layer(il, &cpu_state.conv_states[il], &cpu_state.ssm_states[il]) + .expect("seed the device state"); + let dev = GpuBuffer::from_host(gpu.executor_mut().context(), &hidden) + .expect("upload hidden"); + + qwen.forward_deltanet( + d, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu deltanet"); + + gpu.forward_deltanet_layer(il, &dev).expect("gpu deltanet"); + gpu.executor_mut().sync_stream().expect("sync"); + let mut got = vec![0.0f32; hidden_dim]; + dev.copy_to_host(&mut got).expect("download hidden"); + + assert_rel_linf( + &got, + &hidden, + LAYER_TOL, + &format!("pos {pos} layer {il} hidden"), + ); + + let (conv, ssm) = gpu.download_layer(il).expect("download state"); + // The conv window is downstream of the Q5_K attn_qkv GEMV + // only, which is exact to 1e-7 — hold it to the kernel + // tolerance, not the layer budget. + assert_rel_linf( + &conv, + &cpu_state.conv_states[il], + TOL, + &format!("pos {pos} layer {il} conv window"), + ); + assert_rel_linf( + &ssm, + &cpu_state.ssm_states[il], + LAYER_TOL, + &format!("pos {pos} layer {il} ssm state"), + ); + compared += 1; + }, + Qwen35OwnedLayer::Attention(a) => { + // Not on the GPU in this phase (#3090 phase 2) — the CPU + // advances the hidden state and the KV cache so the next + // DeltaNet layer sees the real input. + qwen.forward_attention( + a, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu attention"); + }, + } + } + cpu_state.kv_cache.advance(); + } + + assert_eq!( + compared, + deltanet_layers * PROMPT.len(), + "every DeltaNet layer must be compared at every position" + ); +} + +/// The six Gated `DeltaNet` kernels, each against the CPU reference function on +/// the GPU's OWN input, at 1e-3 relative. +/// +/// Feeding the CPU the GPU's inputs is what makes this a kernel test: the +/// projection GEMVs are upstream of every one of these and carry ~5e-3 of their +/// own (module docs), so comparing "the CPU's out_h" against "the GPU's out_h" +/// would measure the GEMVs and call it the delta rule. +#[test] +#[serial_test::serial] +#[allow(clippy::too_many_lines)] +fn qwen35_cuda_gdn_ops_match_cpu_given_the_same_inputs() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let il = qwen + .layers + .iter() + .position(|l| matches!(l, Qwen35OwnedLayer::DeltaNet(_))) + .expect("a DeltaNet layer"); + let Qwen35OwnedLayer::DeltaNet(d) = &qwen.layers[il] else { + unreachable!("just matched") + }; + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + gpu.pin_reference_gemv(); + + let hidden_dim = qwen.base.config.hidden_dim; + let eps = qwen.base.config.eps; + let k_dim = qwen.head_k_dim * qwen.num_k_heads; + let v_dim = qwen.head_v_dim * qwen.num_v_heads; + let conv_dim = k_dim * 2 + v_dim; + let mut cpu_state = qwen.new_state(PROMPT.len() + 1); + let mut normed = vec![0.0f32; hidden_dim]; + let mut post_normed = vec![0.0f32; hidden_dim]; + + for (pos, &token) in PROMPT.iter().enumerate() { + // Positions 1 and 2 run against a NON-ZERO conv window and recurrent + // state — a delta rule that ignored its state would pass at pos 0. + let conv0 = cpu_state.conv_states[il].clone(); + let ssm0 = cpu_state.ssm_states[il].clone(); + assert_eq!( + pos > 0, + max_abs(&ssm0) > 0.0, + "pos {pos}: the recurrent state must be non-zero after the first token" + ); + + let mut hidden = qwen.base.token_embedding() + [(token as usize) * hidden_dim..(token as usize + 1) * hidden_dim] + .to_vec(); + gpu.upload_layer(il, &conv0, &ssm0).expect("seed"); + let dev = + GpuBuffer::from_host(gpu.executor_mut().context(), &hidden).expect("upload hidden"); + gpu.forward_deltanet_layer(il, &dev).expect("gpu deltanet"); + + let conv_in_g = gpu.dump_stage("conv_in"); + let conv_out_g = gpu.dump_stage("conv_out"); + let alpha_g = gpu.dump_stage("alpha_raw"); + let beta_raw_g = gpu.dump_stage("beta_raw"); + let dt_g = gpu.dump_stage("dt"); + let beta_g = gpu.dump_stage("beta"); + let gate_g = gpu.dump_stage("gate"); + let out_h_g = gpu.dump_stage("out_h"); + let ssm_out_in_g = gpu.dump_stage("ssm_out_in"); + let (conv_state_g, ssm_state_g) = gpu.download_layer(il).expect("download state"); + + // --- causal_conv1d + SiLU + per-head L2, from the GPU's own conv input. + let mut conv_state_c = conv0.clone(); + let mut conv_out_c = vec![0.0f32; conv_dim]; + causal_conv1d( + &conv_in_g, + &mut conv_state_c[..], + &d.ssm_conv1d_weight, + qwen.conv_kernel, + conv_dim, + &mut conv_out_c, + ); + for x in &mut conv_out_c { + *x = silu(*x); + } + l2_norm_per_head(&mut conv_out_c[0..k_dim], qwen.head_k_dim, eps); + l2_norm_per_head(&mut conv_out_c[k_dim..k_dim * 2], qwen.head_k_dim, eps); + assert_rel_linf( + &conv_out_g, + &conv_out_c, + TOL, + &format!("pos {pos}: conv1d+SiLU+per-head L2"), + ); + assert_rel_linf( + &conv_state_g, + &conv_state_c, + TOL, + &format!("pos {pos}: conv window after the shift"), + ); + + // --- the dt / beta gates, from the GPU's own alpha and beta_raw. + let dt_c: Vec = alpha_g + .iter() + .enumerate() + .map(|(i, a)| softplus(a + d.ssm_dt_bias[i]) * d.ssm_a[i]) + .collect(); + let beta_c: Vec = beta_raw_g + .iter() + .map(|b| 1.0 / (1.0 + (-b).exp())) + .collect(); + assert_rel_linf(&dt_g, &dt_c, TOL, &format!("pos {pos}: dt gate")); + assert_rel_linf(&beta_g, &beta_c, TOL, &format!("pos {pos}: beta gate")); + + // --- the delta rule, from the GPU's own q/k/v, beta, dt and state. + let mut ssm_state_c = ssm0.clone(); + let mut out_h_c = vec![0.0f32; v_dim]; + delta_rule_recurrence( + &conv_out_g[0..k_dim], + &conv_out_g[k_dim..k_dim * 2], + &conv_out_g[k_dim * 2..conv_dim], + &beta_g, + &dt_g, + &mut ssm_state_c[..], + &mut out_h_c, + qwen.num_v_heads, + qwen.head_v_dim, + ); + assert_rel_linf( + &out_h_g, + &out_h_c, + TOL, + &format!("pos {pos}: delta-rule output"), + ); + assert_rel_linf( + &ssm_state_g, + &ssm_state_c, + TOL, + &format!("pos {pos}: recurrent state after the delta rule"), + ); + + // --- gated RMSNorm, from the GPU's own out_h and gate. + let mut ssm_out_in_c = vec![0.0f32; v_dim]; + gated_rmsnorm( + &out_h_g, + &gate_g, + &d.ssm_norm_weight, + eps, + qwen.head_v_dim, + &mut ssm_out_in_c, + ); + assert_rel_linf( + &ssm_out_in_g, + &ssm_out_in_c, + TOL, + &format!("pos {pos}: gated RMSNorm"), + ); + + // Advance the CPU reference so the next position starts from a real state. + qwen.forward_deltanet( + d, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu deltanet"); + cpu_state.kv_cache.advance(); + } +} + +/// The sigmoid-gate kernel (the full-attention output gate, phase 2's) against +/// `apply_sigmoid_gate` — wired here so it is not dark until phase 2 lands. +#[test] +#[serial_test::serial] +fn qwen35_cuda_sigmoid_gate_matches_cpu() { + let mut executor = qwen35_cuda_fixture_or_skip!(); + let n = 2048usize; + let x: Vec = (0..n).map(|i| (i as f32).mul_add(0.001, -1.0)).collect(); + let g: Vec = (0..n).map(|i| 2.0 - (i as f32) * 0.002).collect(); + let mut want = x.clone(); + crate::gguf::forward_qwen35::apply_sigmoid_gate(&mut want, &g); + + let xd = GpuBuffer::from_host(executor.context(), &x).expect("upload x"); + let gd = GpuBuffer::from_host(executor.context(), &g).expect("upload gate"); + executor + .gdn_sigmoid_gate_into(&xd, &gd, u32::try_from(n).expect("n fits")) + .expect("launch"); + executor.sync_stream().expect("sync"); + let mut got = vec![0.0f32; n]; + xd.copy_to_host(&mut got).expect("download"); + assert_rel_linf(&got, &want, TOL, "sigmoid gate"); +} + +/// Each layer path refuses the other kind, loudly and by name, instead of +/// silently running a `DeltaNet` layer's code on an attention layer or the +/// reverse. +#[test] +#[serial_test::serial] +fn qwen35_cuda_layer_kind_mismatch_refuses_by_name() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let attention_il = qwen + .layers + .iter() + .position(|l| matches!(l, Qwen35OwnedLayer::Attention(_))) + .expect("the hybrid file carries full-attention layers"); + let deltanet_il = qwen + .layers + .iter() + .position(|l| matches!(l, Qwen35OwnedLayer::DeltaNet(_))) + .expect("the hybrid file carries Gated DeltaNet layers"); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let dev = GpuBuffer::::from_host( + gpu.executor_mut().context(), + &vec![0.0f32; qwen.base.config.hidden_dim], + ) + .expect("upload hidden"); + + let err = gpu + .forward_deltanet_layer(attention_il, &dev) + .expect_err("an attention layer must not run the DeltaNet path"); + assert!( + format!("{err}").contains("qwen35_cuda_attention"), + "the refusal must name the seam: {err}" + ); + let err = gpu + .forward_attention_layer(deltanet_il, &dev, 0) + .expect_err("a DeltaNet layer must not run the attention path"); + assert!(format!("{err}").contains("qwen35_cuda_deltanet"), "{err}"); +} + +/// Per-layer parity of the wired full-attention GPU forward against the CPU +/// reference, over the first three tokens of a fixed prompt, on EVERY attention +/// layer, with teacher forcing — on the layer output hidden state AND on the K +/// and V rows the token appends at its position. +#[test] +#[serial_test::serial] +#[allow(clippy::too_many_lines)] +fn qwen35_cuda_attention_layers_match_cpu_on_the_real_file() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let attention_layers = qwen + .layers + .iter() + .filter(|l| matches!(l, Qwen35OwnedLayer::Attention(_))) + .count(); + assert!( + attention_layers > 0, + "the fixture must carry full-attention layers, else this test proves nothing" + ); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + gpu.pin_reference_gemv(); + + let hidden_dim = qwen.base.config.hidden_dim; + let head_dim = qwen.head_dim; + let num_kv_heads = qwen.base.config.num_kv_heads; + let kv_row = num_kv_heads * head_dim; + let eps = qwen.base.config.eps; + let n_rot = 2 * qwen.rope_sections.iter().sum::(); + let freq_base = qwen.base.config.rope_theta; + let mut cpu_state = qwen.new_state(PROMPT.len() + 1); + let mut normed = vec![0.0f32; hidden_dim]; + let mut post_normed = vec![0.0f32; hidden_dim]; + let mut compared = 0usize; + let mut worst_cos = 1.0f32; + let mut worst_linf = 0.0f32; + + for (pos, &token) in PROMPT.iter().enumerate() { + let mut hidden = qwen.base.token_embedding() + [(token as usize) * hidden_dim..(token as usize + 1) * hidden_dim] + .to_vec(); + + for (il, layer) in qwen.layers.iter().enumerate() { + match layer { + Qwen35OwnedLayer::DeltaNet(d) => { + // Already proven above; here it only advances the CPU + // reference so the next attention layer sees a real input. + qwen.forward_deltanet( + d, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu deltanet"); + }, + Qwen35OwnedLayer::Attention(a) => { + // Teacher forcing: the GPU starts from the CPU's KV rows for + // positions 0..pos, and from the same hidden state. + let k_before = cpu_state.kv_cache.get_k(il).to_vec(); + let v_before = cpu_state.kv_cache.get_v(il).to_vec(); + let hidden_in = hidden.clone(); + assert_eq!( + k_before.len(), + pos * kv_row, + "pos {pos} layer {il}: the CPU cache must hold exactly {pos} rows" + ); + gpu.upload_attention_kv(il, &k_before, &v_before) + .expect("seed the device KV cache"); + let dev = GpuBuffer::from_host(gpu.executor_mut().context(), &hidden) + .expect("upload hidden"); + + qwen.forward_attention( + a, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu attention"); + + gpu.forward_attention_layer(il, &dev, pos) + .expect("gpu attention"); + gpu.executor_mut().sync_stream().expect("sync"); + let mut got = vec![0.0f32; hidden_dim]; + dev.copy_to_host(&mut got).expect("download hidden"); + + // The CPU side here IS the production forward, which + // quantizes every projection's activation to Q8_K — so the + // contract is argmax + cosine + a budget, not a tight L∞. + // See COSINE_FLOOR. + let (cos, linf) = assert_forward_parity( + &got, + &hidden, + LAYER_BUDGET, + &format!("pos {pos} layer {il} hidden"), + ); + eprintln!( + "[attn layer] pos {pos} layer {il}: cosine {cos:.6} relative L-inf \ + {linf:.3e}" + ); + worst_cos = worst_cos.min(cos); + worst_linf = worst_linf.max(linf); + + // The appended row is the only one the GPU computed — the + // earlier rows were seeded from the CPU and would compare + // equal to themselves. + // + // The reference for it is EXACT arithmetic, not the CPU's + // own cache row: K and V are one quantized GEMV off the + // layer input, and the CPU's GEMV quantizes its activation + // to Q8_K (parity-floor test). Comparing against the CPU row + // measured that quantization and nothing else — it read + // 7.6e-3 relative on a K row whose every non-GEMV op is + // inside 1e-3. Built from the same layer input with + // dequantized weights and f64 dots, the same row is a + // kernel-contract comparison and holds at TOL. + let (k_after, v_after) = gpu.download_layer(il).expect("download KV"); + assert_eq!(k_after.len(), (pos + 1) * kv_row, "K rows written"); + let mut normed_ref = vec![0.0f32; hidden_dim]; + crate::gguf::ops::rms_norm_into(&hidden_in, &a.attn_norm, eps, &mut normed_ref); + let mut k_ref = exact_gemv(&a.attn_k, &normed_ref); + crate::gguf::ops::apply_per_head_rms_norm( + &mut k_ref, + &a.attn_k_norm, + num_kv_heads, + eps, + ); + crate::gguf::forward_qwen35::apply_partial_neox_rope( + &mut k_ref, + num_kv_heads, + head_dim, + n_rot, + pos, + freq_base, + ); + let v_ref = exact_gemv(&a.attn_v, &normed_ref); + eprintln!( + "[attn layer] pos {pos} layer {il}: K row vs exact {:.3e} V row vs \ + exact {:.3e}", + rel_linf(&k_after[pos * kv_row..], &k_ref), + rel_linf(&v_after[pos * kv_row..], &v_ref), + ); + assert_rel_linf( + &k_after[pos * kv_row..], + &k_ref, + KV_TOL, + &format!("pos {pos} layer {il} K row"), + ); + assert_rel_linf( + &v_after[pos * kv_row..], + &v_ref, + KV_TOL, + &format!("pos {pos} layer {il} V row"), + ); + compared += 1; + }, + } + } + cpu_state.kv_cache.advance(); + } + + assert_eq!( + compared, + attention_layers * PROMPT.len(), + "every attention layer must be compared at every position" + ); + eprintln!( + "[attn layer] worst over {compared} comparisons: cosine {worst_cos:.6} relative L-inf \ + {worst_linf:.3e}" + ); +} + +/// The three attention-side kernels (split, partial rope, decode attention) and +/// the per-head RMSNorm, each against the CPU reference function on the GPU's +/// OWN input, at 1e-3 relative. +/// +/// Feeding the CPU the GPU's inputs is what makes this a kernel test: the +/// projection GEMVs are upstream of every one of them and carry ~5e-3 of their +/// own, so comparing "the CPU's attention output" against "the GPU's" would +/// measure the GEMVs and call it the attention block. +#[test] +#[serial_test::serial] +#[allow(clippy::too_many_lines)] +fn qwen35_cuda_attention_ops_match_cpu_given_the_same_inputs() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let il = qwen + .layers + .iter() + .position(|l| matches!(l, Qwen35OwnedLayer::Attention(_))) + .expect("an attention layer"); + let Qwen35OwnedLayer::Attention(a) = &qwen.layers[il] else { + unreachable!("just matched") + }; + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + gpu.pin_reference_gemv(); + + let hidden_dim = qwen.base.config.hidden_dim; + let eps = qwen.base.config.eps; + let num_heads = qwen.base.config.num_heads; + let num_kv_heads = qwen.base.config.num_kv_heads; + let head_dim = a.attn_q_norm.len(); + let kv_row = num_kv_heads * head_dim; + let n_rot = 2 * qwen.rope_sections.iter().sum::(); + let freq_base = qwen.base.config.rope_theta; + let mut cpu_state = qwen.new_state(PROMPT.len() + 1); + let mut normed = vec![0.0f32; hidden_dim]; + let mut post_normed = vec![0.0f32; hidden_dim]; + + for (pos, &token) in PROMPT.iter().enumerate() { + let k_before = cpu_state.kv_cache.get_k(il).to_vec(); + let v_before = cpu_state.kv_cache.get_v(il).to_vec(); + let mut hidden = qwen.base.token_embedding() + [(token as usize) * hidden_dim..(token as usize + 1) * hidden_dim] + .to_vec(); + gpu.upload_attention_kv(il, &k_before, &v_before) + .expect("seed"); + let dev = + GpuBuffer::from_host(gpu.executor_mut().context(), &hidden).expect("upload hidden"); + gpu.forward_attention_layer(il, &dev, pos) + .expect("gpu attention"); + + let normed_g = gpu.dump_stage("normed"); + let q_full_g = gpu.dump_stage("q_full"); + let q_g = gpu.dump_stage("q"); + let q_rot_g = gpu.dump_stage("q_normed"); + let gate_g = gpu.dump_stage("attn_gate"); + let k_raw_g = gpu.dump_stage("k_raw"); + let attn_out_in_g = gpu.dump_stage("attn_out_in"); + let (k_all_g, v_all_g) = gpu.download_layer(il).expect("download KV"); + + // --- the q|gate split, from the GPU's own joint projection. Pure data + // movement: this one must be EXACT. + let mut q_c = vec![0.0f32; num_heads * head_dim]; + let mut gate_c = vec![0.0f32; num_heads * head_dim]; + for h in 0..num_heads { + let src = h * head_dim * 2; + let dst = h * head_dim; + q_c[dst..dst + head_dim].copy_from_slice(&q_full_g[src..src + head_dim]); + gate_c[dst..dst + head_dim] + .copy_from_slice(&q_full_g[src + head_dim..src + head_dim * 2]); + } + assert_eq!(q_g, q_c, "pos {pos}: the q half of the split is not exact"); + assert_eq!( + gate_g, gate_c, + "pos {pos}: the gate half of the split is not exact" + ); + + // --- per-head RMSNorm + partial rope on q, from the GPU's own q. + let mut q_rot_c = q_g.clone(); + crate::gguf::ops::apply_per_head_rms_norm(&mut q_rot_c, &a.attn_q_norm, num_heads, eps); + crate::gguf::forward_qwen35::apply_partial_neox_rope( + &mut q_rot_c, + num_heads, + head_dim, + n_rot, + pos, + freq_base, + ); + assert_rel_linf(&q_rot_g, &q_rot_c, TOL, &format!("pos {pos}: q norm+rope")); + + // --- the same on k, whose result IS the appended cache row. + let mut k_rot_c = k_raw_g.clone(); + crate::gguf::ops::apply_per_head_rms_norm(&mut k_rot_c, &a.attn_k_norm, num_kv_heads, eps); + crate::gguf::forward_qwen35::apply_partial_neox_rope( + &mut k_rot_c, + num_kv_heads, + head_dim, + n_rot, + pos, + freq_base, + ); + assert_rel_linf( + &k_all_g[pos * kv_row..], + &k_rot_c, + TOL, + &format!("pos {pos}: k norm+rope (the appended row)"), + ); + + // --- the attention block + output gate, from the GPU's own q, KV cache + // and gate. + let mut attn_c = vec![0.0f32; num_heads * head_dim]; + let group_size = num_heads / num_kv_heads; + for h in 0..num_heads { + let kv_h = h / group_size; + let q_h = &q_rot_g[h * head_dim..(h + 1) * head_dim]; + let mut scores = vec![0.0f32; pos + 1]; + for (p, score) in scores.iter_mut().enumerate() { + let base = p * kv_row + kv_h * head_dim; + let mut dot = 0.0; + for i in 0..head_dim { + dot += q_h[i] * k_all_g[base + i]; + } + *score = dot / (head_dim as f32).sqrt(); + } + crate::gguf::ops::softmax(&mut scores); + let out_h = &mut attn_c[h * head_dim..(h + 1) * head_dim]; + for (p, &w) in scores.iter().enumerate() { + let base = p * kv_row + kv_h * head_dim; + for i in 0..head_dim { + out_h[i] += w * v_all_g[base + i]; + } + } + } + crate::gguf::forward_qwen35::apply_sigmoid_gate(&mut attn_c, &gate_g); + assert_rel_linf( + &attn_out_in_g, + &attn_c, + TOL, + &format!("pos {pos}: decode attention + output gate"), + ); + + // --- the projection GEMVs, from the GPU's own normed input, against + // EXACT arithmetic (dequantized weight, f64 dots) and NOT against the + // CPU production op: `fused_matmul_into` quantizes its activation to + // Q8_K and is itself ~3e-3 from exact, so it cannot be the reference a + // 1e-3 bar is written against. See COSINE_FLOOR and the parity-floor + // test. These are not this ticket's kernels, but the float GEMV is what + // the constructor pins, so this is where a regression in that choice + // shows up first. + let q_full_e = exact_gemv(&a.attn_q, &normed_g); + let k_e = exact_gemv(&a.attn_k, &normed_g); + let out_e = exact_gemv(&a.attn_output, &attn_out_in_g); + let attn_out_g = gpu.dump_stage("attn_out"); + eprintln!( + "[GEMV vs exact pos {pos}] attn_q(t{}) {:.3e} attn_k(t{}) {:.3e} attn_output(t{}) \ + {:.3e}", + a.attn_q.qtype, + rel_linf(&q_full_g, &q_full_e), + a.attn_k.qtype, + rel_linf(&k_raw_g, &k_e), + a.attn_output.qtype, + rel_linf(&attn_out_g, &out_e), + ); + assert_rel_linf( + &q_full_g, + &q_full_e, + TOL, + &format!("pos {pos}: attn_q GEMV"), + ); + assert_rel_linf(&k_raw_g, &k_e, TOL, &format!("pos {pos}: attn_k GEMV")); + assert_rel_linf( + &attn_out_g, + &out_e, + TOL, + &format!("pos {pos}: attn_output GEMV"), + ); + + // --- rms_norm itself, on the same hidden state. + let mut normed_c = vec![0.0f32; hidden_dim]; + crate::gguf::ops::rms_norm_into(&hidden, &a.attn_norm, eps, &mut normed_c); + assert_rel_linf(&normed_g, &normed_c, TOL, &format!("pos {pos}: rms_norm")); + + qwen.forward_attention( + a, + &mut hidden, + &mut cpu_state, + il, + pos, + &mut normed, + &mut post_normed, + ) + .expect("cpu attention"); + cpu_state.kv_cache.advance(); + } +} + +/// END TO END: the whole per-token forward — both layer kinds, the output norm +/// and the `lm_head` — against `forward_single_qwen35`, from fresh states, token +/// by token over a fixed six-token prompt. +/// +/// No teacher forcing: after the first token the two sides diverge or they do +/// not, and the argmax equality is what a decode actually depends on. +/// +/// The contract is [`COSINE_FLOOR`]'s three assertions, and the restatement did +/// NOT cost the test its teeth: skipping `gdn_sigmoid_gate_into` in +/// `deltanet_layer_inner` (one line, reverted) turns this RED at position 0 on +/// all three at once — argmax **220 vs 13**, cosine **0.765610** (floor 0.996), +/// relative L∞ **1.427e0** (budget 7e-2). A gate the forward does not apply is +/// a different token, not a rounding difference. +#[test] +#[serial_test::serial] +fn qwen35_cuda_forward_single_matches_cpu_logits_end_to_end() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + gpu.pin_reference_gemv(); + let mut gpu_state = gpu.new_state().expect("device state"); + let mut cpu_state = qwen.new_state(LONG_PROMPT.len() + 1); + let mut worst_cos = 1.0f32; + let mut worst_linf = 0.0f32; + + for (pos, &token) in LONG_PROMPT.iter().enumerate() { + let want = qwen + .forward_single_qwen35(token, &mut cpu_state, pos) + .expect("cpu forward"); + // Wall time of ONE device token, printed and never asserted on: a + // timing assertion in a correctness test is a flake, but the reading is + // what says whether a sync change cost or bought anything (#3090). + let t0 = std::time::Instant::now(); + let got = gpu + .forward_single(token, &mut gpu_state, pos) + .expect("gpu forward"); + let gpu_ms = t0.elapsed().as_secs_f64() * 1e3; + let (cos, linf) = + assert_forward_parity(&got, &want, LOGITS_BUDGET, &format!("pos {pos} logits")); + eprintln!( + "[e2e] pos {pos}: argmax {} cosine {cos:.6} relative L-inf {linf:.3e} \ + forward_single {gpu_ms:.3} ms", + crate::gguf::ops::argmax(&want), + ); + worst_cos = worst_cos.min(cos); + worst_linf = worst_linf.max(linf); + assert_eq!( + gpu_state.kv_len(), + pos + 1, + "pos {pos}: the device KV cache must have advanced" + ); + } + eprintln!( + "[e2e] worst over {} positions: cosine {worst_cos:.6} relative L-inf {worst_linf:.3e}", + LONG_PROMPT.len(), + ); +} + +/// The device state is sized from the config, never from a constant. +#[test] +#[serial_test::serial] +fn qwen35_cuda_state_is_sized_from_the_config() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let conv_dim = qwen.head_k_dim * qwen.num_k_heads * 2 + qwen.head_v_dim * qwen.num_v_heads; + + let gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + assert_eq!( + gpu.state().conv_len(), + conv_dim * (qwen.conv_kernel - 1), + "conv window is conv_dim * (conv_kernel - 1)" + ); + assert_eq!( + gpu.state().ssm_len(), + qwen.num_v_heads * qwen.head_v_dim * qwen.head_v_dim, + "recurrent state is num_v_heads * head_v_dim^2" + ); +} + +/// `forward_hidden_deltanet_only` refuses a hidden state of the wrong width +/// instead of reading past the buffer. +#[test] +#[serial_test::serial] +fn qwen35_cuda_forward_hidden_refuses_a_wrong_width() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let err = gpu + .forward_hidden_deltanet_only(&[0.0f32; 7]) + .expect_err("7 is not the hidden width"); + assert!(format!("{err}").contains("hidden state is 7 wide"), "{err}"); +} + +/// WHERE THE PARITY FLOOR COMES FROM (PMAT-3477 / #3090). +/// +/// The wired-layer and end-to-end comparisons above cannot be driven below the +/// projection GEMVs, and a GPU-vs-CPU number on its own never says WHICH side +/// moved. This puts both sides against a third reference that is neither: the +/// Q4_K weight dequantized to f32 and the row dots accumulated in **f64**. +/// +/// Measured on `blk..attn_q` of the real 0.8B file: +/// +/// | side | relative L∞ vs the exact f64 dot | +/// |------|----------------------------------| +/// | GPU, float (`Mwv`) GEMV | **9.96e-8** | +/// | CPU `fused_matmul_into` | **2.95e-3** | +/// +/// The CPU is the noisier side by four and a half orders of magnitude, because +/// `fused_q4k_parallel_matvec_into` quantizes the **activation** to Q8_K and +/// takes int8 dots; the GPU float path dequantizes the weight and accumulates +/// in f32. So ~3e-3 per projection is a property of the *reference*, not a GPU +/// defect, and it is the floor under [`LAYER_TOL`], [`LAYER_BUDGET`] and +/// [`LOGITS_BUDGET`], and the reason the two whole-forward tests assert argmax +/// and [`COSINE_FLOOR`] instead of a tight L∞: no +/// correct GPU implementation can be closer to this CPU than the CPU is to +/// arithmetic. +/// +/// Both bounds bite. If the GPU float GEMV ever regresses it fails on the first +/// assertion; if the CPU path ever stops quantizing its activation the second +/// assertion fails and says so — at which point the end-to-end budget can come +/// down and this test is the thing that tells you. +#[test] +#[serial_test::serial] +fn qwen35_cuda_the_parity_floor_is_the_cpu_references_own_activation_quantization() { + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + let il = qwen + .layers + .iter() + .position(|l| matches!(l, Qwen35OwnedLayer::Attention(_))) + .expect("the hybrid file carries full-attention layers"); + let Qwen35OwnedLayer::Attention(a) = &qwen.layers[il] else { + unreachable!("just matched") + }; + assert_eq!( + a.attn_q.qtype, + crate::gguf::types::GGUF_TYPE_Q4_K, + "this test reads the Q4_K dequantizer; the projection is no longer Q4_K" + ); + + let hidden_dim = qwen.base.config.hidden_dim; + let x: Vec = (0..hidden_dim) + .map(|i| ((i as f32) * 0.37).sin() * 1.3) + .collect(); + + // the exact reference: dequantized weight, f64 accumulation — the same + // helper every GEMV-fed assertion in this file compares against. + let exact = exact_gemv(&a.attn_q, &x); + + let mut cpu = vec![0.0f32; a.attn_q.out_dim]; + qwen.base + .fused_matmul_into(&x, &a.attn_q, &mut cpu) + .expect("cpu attn_q"); + + // NOT `pin_reference_gemv()`: the constructor pins the float variants for + // every model of this architecture, and a test that re-pins measures its + // own call instead of production behaviour (#3090 review). The pin itself + // is asserted by `qwen35_cuda_a_fresh_model_pins_the_float_gemv_variants`; + // if it ever stops holding, the first assertion below turns red here too. + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + let got = gpu + .attn_q_gemv_of_host_input(il, &x) + .expect("gpu attn_q GEMV"); + + let gpu_err = rel_linf(&got, &exact); + let cpu_err = rel_linf(&cpu, &exact); + eprintln!("[parity floor] gpu-vs-exact {gpu_err:.3e} cpu-vs-exact {cpu_err:.3e}"); + assert!( + gpu_err <= 1e-5, + "the GPU float GEMV must reproduce the exact dot: {gpu_err:.3e} > 1e-5" + ); + assert!( + cpu_err > 1e-4, + "the CPU reference no longer quantizes its activation (cpu-vs-exact {cpu_err:.3e} <= \ + 1e-4) — the end-to-end parity budget can now come down; re-measure LOGITS_BUDGET and \ + consider a tight L-inf bar again" + ); + assert!( + cpu_err > gpu_err * 100.0, + "the CPU, not the GPU, must be the side that is far from arithmetic: cpu {cpu_err:.3e} \ + vs gpu {gpu_err:.3e}" + ); +} + +/// A model built the ordinary way already runs the FLOAT GEMV kernels — the pin +/// is production behaviour, not something a test remembers to do. +/// +/// `GpuProfile::detect` picks the DP4A variants for this device; the +/// constructor overrides them for this architecture because DP4A is +/// catastrophic through the recurrence (the falsifier below). If that override +/// is ever dropped, decode silently returns garbage tokens and only this +/// assertion says so before the parity tests do. +#[test] +#[serial_test::serial] +fn qwen35_cuda_a_fresh_model_pins_the_float_gemv_variants() { + use crate::cuda::gpu_profile::{Q4kVariant, Q6kVariant}; + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + assert_eq!( + gpu.gemv_variants(), + (Q4kVariant::Mwv, Q6kVariant::Mwv), + "Qwen35CudaModel::new must pin the float GEMV variants for this architecture" + ); +} + +/// WHY THE PIN EXISTS (PMAT-3477 / #3090): with the DP4A GEMV kernels armed, +/// the same forward that matches the CPU token for token produces a DIFFERENT +/// token, or a direction nowhere near the CPU's. +/// +/// This is a falsifier, not a bug reproduction: it asserts the failure, so if a +/// future DP4A activation-quantization change makes the path correct, this test +/// goes RED and the pin (and the DP4A-through-recurrence ticket, 0.69.0) can be +/// reconsidered on evidence. +/// +/// Measured on the real 0.8B file at position 0: see the printed reading. The +/// DeltaNet-only path is 1.656 relative away from the CPU under `HwDp4a` versus +/// 0.000 float-vs-float; end to end the argmax is simply wrong. +#[test] +#[serial_test::serial] +fn qwen35_cuda_dp4a_gemv_is_catastrophic_through_the_recurrence() { + use crate::cuda::gpu_profile::{Q4kVariant, Q6kVariant}; + let executor = qwen35_cuda_fixture_or_skip!(); + let mapped = crate::gguf::MappedGGUFModel::from_path(MODEL_PATH).expect("map the GGUF"); + let base = load_cpu_model(&mapped); + let qwen = + Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data()).expect("qwen35"); + + let mut gpu = Qwen35CudaModel::new(&qwen, executor).expect("build the CUDA model"); + // The variant IS selectable from a test: the executor's profile is what + // every `gemv_dispatch` reads, and `gemv_variants()` reports it back. + gpu.executor_mut().gpu_profile.q4k = Q4kVariant::HwDp4a; + gpu.executor_mut().gpu_profile.q6k = Q6kVariant::HwDp4a; + assert_eq!( + gpu.gemv_variants(), + (Q4kVariant::HwDp4a, Q6kVariant::HwDp4a), + "the DP4A variants must actually be armed, else this test proves nothing" + ); + + let mut gpu_state = gpu.new_state().expect("device state"); + let mut cpu_state = qwen.new_state(LONG_PROMPT.len() + 1); + let mut broken = 0usize; + + for (pos, &token) in LONG_PROMPT.iter().enumerate() { + let want = qwen + .forward_single_qwen35(token, &mut cpu_state, pos) + .expect("cpu forward"); + let got = gpu + .forward_single(token, &mut gpu_state, pos) + .expect("gpu forward"); + let cos = cosine(&got, &want); + let (gpu_arg, cpu_arg) = ( + crate::gguf::ops::argmax(&got), + crate::gguf::ops::argmax(&want), + ); + eprintln!( + "[dp4a] pos {pos}: gpu argmax {gpu_arg} cpu argmax {cpu_arg} cosine {cos:.6} \ + relative L-inf {:.3e}", + rel_linf(&got, &want), + ); + if gpu_arg != cpu_arg || cos < COSINE_FLOOR { + broken += 1; + } + } + + assert!( + broken > 0, + "the DP4A GEMV path passed the parity contract at every position — it is no longer \ + catastrophic through the recurrence, so re-measure and revisit the pin in \ + Qwen35CudaModel::with_max_seq_len (DP4A-through-recurrence ticket, 0.69.0)" + ); +} + +/// Keep the layer type in the compiled surface: the parity tests bind it, and a +/// rename of the CPU struct must break here, not silently at phase 2. +const _: Option<&Qwen35OwnedDeltaNetLayer> = None; diff --git a/crates/aprender-serve/src/gguf/cuda/mod.rs b/crates/aprender-serve/src/gguf/cuda/mod.rs index fdddd95fc9..0a03c501f0 100644 --- a/crates/aprender-serve/src/gguf/cuda/mod.rs +++ b/crates/aprender-serve/src/gguf/cuda/mod.rs @@ -35,12 +35,16 @@ mod backend; mod forward; +/// PMAT-3477 (#3090): Qwen3.5's Gated `DeltaNet` block on the GPU. +mod forward_qwen35_cuda; mod generation; mod speculative; mod weights; // Re-export types for public API pub use backend::CudaBackend; +// PMAT-3477 (#3090): the Gated DeltaNet GPU model and its device state. +pub use forward_qwen35_cuda::{Qwen35CudaModel, Qwen35CudaState}; // PMAT-072: Step-wise batched decode state for lock-releasing scheduler pub use generation::BatchedDecodeState; diff --git a/crates/aprender-serve/src/gguf/dtype.rs b/crates/aprender-serve/src/gguf/dtype.rs index 09dae94f48..d8788190d4 100644 --- a/crates/aprender-serve/src/gguf/dtype.rs +++ b/crates/aprender-serve/src/gguf/dtype.rs @@ -1,4 +1,3 @@ - /// GH-321: Convert GGML qtype to APR dtype string using unified enum. /// /// FAILS on an unrecognized qtype. This used to be @@ -48,6 +47,150 @@ pub(crate) fn gpu_unsupported_quant_qtype(qtype: u32) -> bool { !matches!(qtype, 0 | 2 | 3 | 6 | 8 | 12 | 13 | 14) } +/// #3477 / PMAT-781/783/785: the quantized projections the Qwen3.5 hybrid +/// (Gated `DeltaNet` + gated attention) forward uploads, by tensor-name stem. +/// +/// The first five are the Gated `DeltaNet` weights `Qwen35CudaModel::new` +/// uploads (`attn_qkv`, `ssm_alpha`, `ssm_beta`, `attn_gate`, `ssm_out`); the +/// rest are the attention-layer and FFN projections the same forward reaches. +/// `ssm_conv1d` / `ssm_a` / `ssm_dt_bias` / the norms are NOT here: they are +/// loaded as f32 vectors, never through a GEMV kernel. +pub(crate) const HYBRID_GPU_PROJECTIONS: [&str; 12] = [ + "attn_qkv", + "ssm_alpha", + "ssm_beta", + "attn_gate", + "ssm_out", + "attn_q", + "attn_k", + "attn_v", + "attn_output", + "ffn_gate", + "ffn_up", + "ffn_down", +]; + +/// Is this tensor one of the hybrid forward's quantized projections? +/// +/// Matched on the dot-delimited stem (`blk.0.ssm_alpha.weight` → `ssm_alpha`) +/// so `attn_q` cannot swallow `attn_q_norm` and `ssm_out` cannot swallow +/// `ssm_out_something`: a substring test over these names is wrong in both +/// directions (`attn_q` ⊂ `attn_qkv`). +fn is_hybrid_gpu_projection(name: &str) -> bool { + name.split('.') + .any(|part| HYBRID_GPU_PROJECTIONS.contains(&part)) +} + +/// The first hybrid projection whose GGML type has no verified GPU GEMV kernel. +/// +/// The dense gate [`OwnedQuantizedModel::has_gpu_unsupported_quant`] CANNOT see +/// these tensors: for `qwen35` the `OwnedQuantizedModel` it inspects is the base +/// built by `Qwen35Model::create_base_model`, whose `layers` is deliberately +/// EMPTY (the hybrid weights live in `Qwen35Model::layers`), so the dense gate +/// judges the lm_head and nothing else. Answering from `(name, GGML type)` +/// pairs makes this gate runnable straight off the GGUF header — before a byte +/// is uploaded — and testable without a model. +/// +/// Same policy as the dense gate: anything outside +/// [`gpu_unsupported_quant_qtype`]'s whitelist would hit the GPU upload's +/// `resolve_qtype().unwrap_or(Q4K)` and be silently decoded as Q4_K +/// (PMAT-781/783). `transformer::hybrid_gpu_quant_refusal` is the public +/// wording of this verdict. +pub(crate) fn hybrid_gpu_unsupported_quant_tensor<'n>( + tensors: impl IntoIterator, +) -> Option<(String, u32)> { + tensors + .into_iter() + .find(|&(name, qtype)| is_hybrid_gpu_projection(name) && gpu_unsupported_quant_qtype(qtype)) + .map(|(name, qtype)| (name.to_string(), qtype)) +} + +#[cfg(test)] +mod hybrid_gpu_unsupported_quant_tests { + use super::{hybrid_gpu_unsupported_quant_tensor, is_hybrid_gpu_projection}; + + /// Every Gated `DeltaNet` projection is judged, not just the dense ones. + /// Before #3477 the only quant gate was `has_gpu_unsupported_quant`, which + /// for qwen35 inspects an EMPTY layer list — so a Q3_K `ssm_alpha` reached + /// the GPU upload and was decoded as Q4_K (PMAT-781/783). + #[test] + fn every_deltanet_projection_is_gpu_unsupported_quant_checked() { + for stem in ["attn_qkv", "ssm_alpha", "ssm_beta", "attn_gate", "ssm_out"] { + let name = format!("blk.7.{stem}.weight"); + assert_eq!( + hybrid_gpu_unsupported_quant_tensor([(name.as_str(), 10u32)]), + Some((name.clone(), 10)), + "{stem}: Q2_K has no GPU GEMV kernel" + ); + assert_eq!( + hybrid_gpu_unsupported_quant_tensor([(name.as_str(), 12u32)]), + None, + "{stem}: Q4_K does have one" + ); + } + } + + /// A tensor the hybrid forward never sends through a GEMV kernel must not + /// veto the GPU: `ssm_conv1d` / `ssm_a` / the norms are f32 vectors, and an + /// f32 tensor is whitelisted anyway — but a name-keyed gate that matched + /// them would refuse on any future non-f32 storage for no reason. + #[test] + fn non_projection_tensors_are_not_gpu_unsupported_quant_vetoes() { + for name in [ + "blk.0.ssm_conv1d.weight", + "blk.0.ssm_a", + "blk.0.ssm_dt_bias", + "blk.0.ssm_norm.weight", + "blk.0.attn_norm.weight", + "token_embd.weight", + ] { + assert!( + !is_hybrid_gpu_projection(name), + "{name} is not a GEMV projection" + ); + assert_eq!(hybrid_gpu_unsupported_quant_tensor([(name, 11u32)]), None); + } + } + + /// The stem match is exact per dot-delimited part: `attn_q` is a PREFIX of + /// `attn_qkv` and `attn_q_norm`, so a `contains` test would both mis-name + /// the offending tensor and drag a per-head norm into the projection set. + #[test] + fn gpu_unsupported_quant_stems_match_whole_name_parts_only() { + assert!(is_hybrid_gpu_projection("blk.0.attn_qkv.weight")); + assert!(is_hybrid_gpu_projection("blk.0.attn_q.weight")); + assert!(!is_hybrid_gpu_projection("blk.0.attn_q_norm.weight")); + assert!(!is_hybrid_gpu_projection("blk.0.ssm_outer.weight")); + } + + /// The FIRST offending tensor is reported, and a clean model returns None. + #[test] + fn gpu_unsupported_quant_reports_the_first_offender() { + assert_eq!( + hybrid_gpu_unsupported_quant_tensor([ + ("blk.0.attn_qkv.weight", 8u32), + ("blk.0.ssm_beta.weight", 15), + ("blk.1.ssm_out.weight", 7), + ]), + Some(("blk.0.ssm_beta.weight".to_string(), 15)), + "Q8_K (15) is the first tensor without a kernel" + ); + assert_eq!( + hybrid_gpu_unsupported_quant_tensor([ + ("blk.0.attn_qkv.weight", 8u32), + ("blk.0.ssm_beta.weight", 13), + ("blk.0.ffn_down.weight", 14), + ("output.weight", 12), + ]), + None + ); + assert_eq!( + hybrid_gpu_unsupported_quant_tensor(Vec::<(&str, u32)>::new()), + None + ); + } +} + /// GH-321: Convert APR dtype string to byte using unified enum. /// GH-191 FIX: Use GGML dtype values directly so they match TensorEntry::from_binary reader. fn apr_dtype_to_byte(dtype: &str) -> u8 { @@ -109,6 +252,14 @@ impl OwnedQuantizedModel { /// (`OwnedQuantizedModelCuda::with_max_seq_len` → `check_quant_gpu_capability`) /// is protected by this single check, so an unsupported-quant model routes /// to CPU (loud) or errors rather than shipping GPU garbage. + /// + /// #3477 — what this gate CANNOT see: the Qwen3.5 hybrid. Its + /// `OwnedQuantizedModel` is the base built by + /// `Qwen35Model::create_base_model`, whose `layers` is deliberately empty + /// (the Gated DeltaNet weights live in `Qwen35Model::layers`), so the loop + /// below inspects the lm_head alone. The hybrid tensors are judged by the + /// header-level [`hybrid_gpu_unsupported_quant_tensor`] against the same + /// whitelist; do not read a `false` here as "this hybrid model is GPU-safe". #[must_use] pub(crate) fn has_gpu_unsupported_quant(&self) -> bool { if gpu_unsupported_quant_qtype(self.lm_head_weight.qtype) { @@ -186,9 +337,7 @@ impl OwnedQuantizedModel { let offset = tensor_data_bytes.len() as u64; let size = data.len() as u64; - tensor_index_bytes.extend(write_apr_tensor_entry( - name, dtype, shape, offset, size, - )); + tensor_index_bytes.extend(write_apr_tensor_entry(name, dtype, shape, offset, size)); tensor_data_bytes.extend_from_slice(data); } diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs index b49aeb35bf..c4f9ad77da 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs @@ -851,7 +851,11 @@ impl<'a> Qwen35Model<'a> { Ok(logits) } - fn forward_deltanet( + /// One Gated `DeltaNet` layer, in place on `hidden`. + /// + /// `pub(crate)` (PMAT-3477, #3090) so the GPU model's per-layer parity test can + /// drive the CPU reference layer by layer with teacher forcing. No logic change. + pub(crate) fn forward_deltanet( &self, d: &Qwen35OwnedDeltaNetLayer, hidden: &mut [f32], @@ -982,7 +986,12 @@ impl<'a> Qwen35Model<'a> { Ok(()) } - fn forward_attention( + /// One full-attention layer, in place on `hidden`. + /// + /// `pub(crate)` (PMAT-3477, #3090) so the GPU parity test can advance the CPU + /// reference across the interleaved attention layers while it compares the + /// `DeltaNet` ones. No logic change. + pub(crate) fn forward_attention( &self, a: &Qwen35OwnedAttentionLayer, hidden: &mut [f32], @@ -1169,6 +1178,351 @@ pub fn run_qwen35_generate( Ok(tokens) } +/// Why a Qwen3.5 run served its tokens from the CPU forward. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Qwen35CpuReason { + /// The caller asked for the CPU (`--no-gpu` / `force_cpu`). + Requested, + /// This binary was built without the `cuda` feature, so there is no GPU + /// backend to route to — the hybrid forward itself exists on both (#3090). + NoCudaBackend, +} + +/// Which backend serves a Qwen3.5 (Gated `DeltaNet`) GGUF. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Qwen35Route { + /// `Qwen35CudaModel::forward_single` (#3090). + Gpu, + /// `Qwen35Model::forward_single_qwen35` (#3091). + Cpu(Qwen35CpuReason), +} + +/// Pure: which backend a Qwen3.5 GGUF is routed to. +/// +/// Split out from the dispatcher so the decision is falsifiable on a host with +/// no GPU and in a build without `cuda`: `cuda_backend` is the caller's +/// `cfg!(feature = "cuda")`, not a device probe (a device that is present but +/// unusable is a *fallback*, which is printed — see +/// [`run_qwen35_generate_dispatch`] — never a silent re-route). +#[must_use] +pub fn qwen35_route(no_gpu: bool, cuda_backend: bool) -> Qwen35Route { + if no_gpu { + Qwen35Route::Cpu(Qwen35CpuReason::Requested) + } else if cuda_backend { + Qwen35Route::Gpu + } else { + Qwen35Route::Cpu(Qwen35CpuReason::NoCudaBackend) + } +} + +/// The one-line notice a route owes the user, or `None` when it owes none. +/// +/// #3477: the GPU case used to print `[qwen35: Gated DeltaNet runs on the CPU; +/// the GPU backend does not implement it yet (#3090)]` — which is now false: the +/// hybrid runs on the GPU. What a user still needs told is the case where they +/// asked for the GPU and this *binary* cannot give them one. Asking for the CPU +/// explicitly is not news. +#[must_use] +pub fn qwen35_route_notice(route: Qwen35Route) -> Option<&'static str> { + match route { + Qwen35Route::Cpu(Qwen35CpuReason::NoCudaBackend) => Some( + "[qwen35: this binary has no CUDA backend (built without --features cuda); \ + the Gated DeltaNet forward runs on the CPU (#3091)]", + ), + Qwen35Route::Gpu | Qwen35Route::Cpu(Qwen35CpuReason::Requested) => None, + } +} + +/// The prefix of the loud, never-silent CPU fallback for the hybrid GPU path. +pub const QWEN35_GPU_FALLBACK_PREFIX: &str = "warning: GPU (CUDA) qwen35 path rejected"; + +/// Generate with the Qwen3.5 hybrid on the backend the caller asked for, +/// returning `(tokens, used_gpu)` (#3090/#3091). +/// +/// The GPU is attempted whenever it was requested and this build has a CUDA +/// backend; a failure to build the model, a failure inside the forward, or a +/// rejection by the F2 CPU-parity guard falls back to the CPU forward **with the +/// reason printed** — an unannounced backend downgrade is the defect class +/// `QWEN35_GPU_FALLBACK_PREFIX` exists to make impossible. +/// +/// # Errors +/// Only a CPU-forward failure: the GPU path never propagates its error, it falls +/// back. +pub fn run_qwen35_generate_dispatch( + mapped: &crate::gguf::MappedGGUFModel, + base: &OwnedQuantizedModel, + input_tokens: &[u32], + gen_config: &crate::gguf::QuantizedGenerateConfig, + no_gpu: bool, +) -> Result<(Vec, bool)> { + let route = qwen35_route(no_gpu, cfg!(feature = "cuda")); + if let Some(notice) = qwen35_route_notice(route) { + eprintln!("{notice}"); + } + #[cfg(feature = "cuda")] + if route == Qwen35Route::Gpu { + match run_qwen35_generate_gpu(mapped, base, input_tokens, gen_config) { + Ok(tokens) => return Ok((tokens, true)), + Err(reason) => { + eprintln!("{QWEN35_GPU_FALLBACK_PREFIX}, falling back to CPU: {reason}"); + }, + } + } + let tokens = run_qwen35_generate(mapped, base, input_tokens, gen_config)?; + Ok((tokens, false)) +} + +/// Positions the F2 hybrid guard forwards on both backends before it will let +/// the GPU serve a token. Same cap as the dense guard's `gpu_probe`. +#[cfg(feature = "cuda")] +const QWEN35_F2_PROBE_MAX: usize = 64; + +/// The GPU twin of [`run_qwen35_generate`]: build the hybrid on CUDA, prove it +/// against its own CPU forward, then decode. +/// +/// `Err` is a fallback reason, never a user-visible failure — the caller prints +/// it and runs the CPU forward. +#[cfg(feature = "cuda")] +fn run_qwen35_generate_gpu( + mapped: &crate::gguf::MappedGGUFModel, + base: &OwnedQuantizedModel, + input_tokens: &[u32], + gen_config: &crate::gguf::QuantizedGenerateConfig, +) -> std::result::Result, String> { + if input_tokens.is_empty() { + return Err("the prompt is empty".to_string()); + } + let qwen = Qwen35Model::from_model_and_layers(base, &mapped.model, mapped.data()) + .map_err(|e| format!("the hybrid layers would not load: {e}"))?; + + let mut executor = crate::cuda::CudaExecutor::new(0) + .map_err(|e| format!("CUDA initialization failed: {e}"))?; + let device_name = executor + .device_name() + .unwrap_or_else(|_| "Unknown GPU".to_string()); + let vram_mb = executor.memory_info().unwrap_or((0, 0)).1 / (1024 * 1024); + + let max_seq_len = input_tokens.len() + gen_config.max_tokens + 1; + let mut gpu = + crate::gguf::cuda::Qwen35CudaModel::with_max_seq_len(&qwen, executor, max_seq_len) + .map_err(|e| format!("the CUDA model would not build: {e}"))?; + + // Unconditional, like every other backend-selection line on this path: the + // user must be able to tell a GPU run from a CPU one without --verbose. + eprintln!( + "Backend: GPU (CUDA, {device_name}, {vram_mb} MB VRAM) [qwen35 hybrid forward, #3090]" + ); + + if !f2_validate_qwen35(&mut gpu, &qwen, input_tokens) { + return Err("the F2 CPU-parity guard rejected the GPU path".to_string()); + } + qwen35_gpu_decode(&mut gpu, input_tokens, gen_config) +} + +/// Prefill + decode on the GPU, with the token choice +/// [`run_qwen35_generate`] makes, from a state that has never seen the guard's +/// probe. +#[cfg(feature = "cuda")] +fn qwen35_gpu_decode( + gpu: &mut crate::gguf::cuda::Qwen35CudaModel<'_>, + input_tokens: &[u32], + gen_config: &crate::gguf::QuantizedGenerateConfig, +) -> std::result::Result, String> { + use rand::SeedableRng; + let max_seq_len = input_tokens.len() + gen_config.max_tokens + 1; + let mut state = gpu + .new_state() + .map_err(|e| format!("the decode state would not allocate: {e}"))?; + let mut rng = rand::rngs::StdRng::seed_from_u64(gen_config.seed); + + let mut logits = Vec::new(); + for (pos, &token) in input_tokens.iter().enumerate() { + logits = gpu + .forward_single(token, &mut state, pos) + .map_err(|e| format!("the GPU forward failed at prompt position {pos}: {e}"))?; + } + let mut tokens = input_tokens.to_vec(); + for _ in 0..gen_config.max_tokens { + let next = if gen_config.temperature == 0.0 || gen_config.top_k == 1 { + crate::gguf::ops::argmax(&logits) + } else { + OwnedQuantizedModel::sample_topk_seeded( + &logits, + gen_config.temperature, + gen_config.top_k, + gen_config.top_p, + &mut rng, + ) + }; + tokens.push(next); + if gen_config.stop_tokens.contains(&next) || tokens.len() >= max_seq_len { + break; + } + let pos = tokens.len() - 1; + logits = gpu + .forward_single(next, &mut state, pos) + .map_err(|e| format!("the GPU forward failed at decode position {pos}: {e}"))?; + } + Ok(tokens) +} + +/// The F2 runtime guard for the hybrid: forward the real prompt through BOTH +/// backends and accept the GPU only if every real position agrees. +/// +/// The dense `validate_gpu_first_token` cannot serve here — its CPU reference is +/// `forward_single_with_cache` and its GPU probe `forward_gpu_resident`, neither +/// of which exists for this architecture. The decision itself is shared: +/// `f2_multi_position_report` with its floors (0.95 / 0.98 / 0.90), so the +/// hybrid is judged by the same rule as every other GPU path. The probe path is +/// [`F2ProbePath::Serial`] because there is no batched prefill for the hybrid — +/// `qwen35_gpu_decode` prefills token by token, so the serial probe IS the path +/// the run takes. +/// +/// Both states are throwaway: the guard allocates its own, and the generation +/// that follows allocates another. +#[cfg(feature = "cuda")] +fn f2_validate_qwen35( + gpu: &mut crate::gguf::cuda::Qwen35CudaModel<'_>, + cpu: &Qwen35Model<'_>, + probe_context: &[u32], +) -> bool { + // Same escape hatch as the dense gate, and the same one `apr parity` uses. + if std::env::var("SKIP_PARITY_GATE").is_ok_and(|v| v == "1") { + return true; + } + let probe = &probe_context[probe_context.len().saturating_sub(QWEN35_F2_PROBE_MAX)..]; + // A one-token probe has no REAL position (≥1) to judge; position 0 is the + // context-less near-tie the dense gate excludes for the same reason. + if probe.len() < 2 { + return true; + } + let Some(cpu_per_pos) = f2_qwen35_cpu_reference(cpu, probe) else { + return true; // the CPU forward itself failed: nothing to judge against. + }; + let decode_token = cpu_per_pos + .get(probe.len().saturating_sub(1)) + .map_or(0, |l| crate::infer::argmax_u32(l)); + let gpu_per_pos = match f2_qwen35_gpu_logits(gpu, probe, decode_token) { + Ok(v) => v, + Err(msg) => { + eprintln!("{msg}"); + return false; // fail closed. + }, + }; + let report = crate::infer::f2_multi_position_report(&cpu_per_pos, &gpu_per_pos); + if report.accepted { + true + } else { + eprintln!( + "{}", + crate::infer::f2_divergence_msg(&report, crate::infer::F2ProbePath::Serial) + ); + false + } +} + +/// CPU logits at every probe position plus one greedy decode step — the +/// reference half of [`f2_validate_qwen35`]. +#[cfg(feature = "cuda")] +fn f2_qwen35_cpu_reference(cpu: &Qwen35Model<'_>, probe: &[u32]) -> Option>> { + let mut state = cpu.new_state(probe.len() + 2); + let mut per_pos: Vec> = Vec::with_capacity(probe.len() + 1); + for (pos, &tok) in probe.iter().enumerate() { + per_pos.push(cpu.forward_single_qwen35(tok, &mut state, pos).ok()?); + } + let last = per_pos.last()?; + let next = crate::infer::argmax_u32(last); + per_pos.push( + cpu.forward_single_qwen35(next, &mut state, probe.len()) + .ok()?, + ); + Some(per_pos) +} + +/// GPU logits for the same positions, from a fresh device state. +#[cfg(feature = "cuda")] +fn f2_qwen35_gpu_logits( + gpu: &mut crate::gguf::cuda::Qwen35CudaModel<'_>, + probe: &[u32], + decode_token: u32, +) -> std::result::Result>, String> { + let steps = probe.len() + 1; + let mut state = gpu + .new_state() + .map_err(|e| format!("F2 qwen35 probe: the device state would not allocate: {e}"))?; + let mut per_pos: Vec> = Vec::with_capacity(steps); + for (pos, tok) in probe + .iter() + .copied() + .chain(std::iter::once(decode_token)) + .enumerate() + { + match gpu.forward_single(tok, &mut state, pos) { + Ok(logits) => per_pos.push(logits), + Err(e) => return Err(crate::infer::gpu_forward_failure_msg(pos, steps, &e)), + } + } + Ok(per_pos) +} + +#[cfg(test)] +mod qwen35_route_tests { + use super::{qwen35_route, qwen35_route_notice, Qwen35CpuReason, Qwen35Route}; + + // #3090/#3477: with a CUDA build and no --no-gpu, the hybrid goes to the GPU. + // This is the whole point of the ticket; if it ever reads Cpu again, `apr run + // --gpu` is silently serving CPU tokens. + #[test] + fn a_cuda_build_that_was_not_told_otherwise_routes_to_the_gpu() { + assert_eq!(qwen35_route(false, true), Qwen35Route::Gpu); + assert_eq!(qwen35_route_notice(Qwen35Route::Gpu), None); + } + + #[test] + fn no_gpu_wins_over_a_present_cuda_backend() { + assert_eq!( + qwen35_route(true, true), + Qwen35Route::Cpu(Qwen35CpuReason::Requested) + ); + // The user asked for this: it is not news. + assert_eq!( + qwen35_route_notice(Qwen35Route::Cpu(Qwen35CpuReason::Requested)), + None + ); + } + + // A binary without the cuda feature still runs the model — but the user who + // asked for a GPU must be told why they did not get one. + #[test] + fn a_build_without_cuda_says_so() { + let route = qwen35_route(false, false); + assert_eq!(route, Qwen35Route::Cpu(Qwen35CpuReason::NoCudaBackend)); + let notice = qwen35_route_notice(route).expect("the no-backend case owes a notice"); + assert!( + notice.contains("no CUDA backend"), + "the notice must name the missing backend, not the architecture: {notice}" + ); + assert!( + !notice.contains("#3090"), + "#3090 is the GPU forward, which now exists — citing it here is the \ + withdrawn 'the GPU does not implement it' notice: {notice}" + ); + } + + // Both CPU reasons are reachable from the routing function, so neither arm + // of the notice is dead code. + #[test] + fn every_cpu_reason_is_produced_by_the_router() { + for (no_gpu, cuda, want) in [ + (true, true, Qwen35CpuReason::Requested), + (true, false, Qwen35CpuReason::Requested), + (false, false, Qwen35CpuReason::NoCudaBackend), + ] { + assert_eq!(qwen35_route(no_gpu, cuda), Qwen35Route::Cpu(want)); + } + } +} + #[cfg(test)] mod qwen35_ssm_meta_tests { use super::{qwen35_ssm_meta, Qwen35SsmMeta}; diff --git a/crates/aprender-serve/src/gguf/transformer.rs b/crates/aprender-serve/src/gguf/transformer.rs index 19613ae48e..90d54c3ce0 100644 --- a/crates/aprender-serve/src/gguf/transformer.rs +++ b/crates/aprender-serve/src/gguf/transformer.rs @@ -79,6 +79,14 @@ pub struct QuantizedGGUFTransformerLayer { /// /// Takes tensor names rather than a `GGUFModel` so the predicate is directly /// testable and callable from any crate that has already parsed the header. +/// +/// #3477 (operator ruling 2026-09-19, GPU support for Qwen3.5 is required): +/// the hybrid forward now exists on BOTH backends for the architecture +/// [`hybrid_forward_handles`] names — `forward_qwen35` on the CPU (#3091) and +/// `Qwen35CudaModel` on the GPU (#3090) — so this predicate no longer refuses +/// it. Every other SSM architecture keeps its refusal, reworded: the old +/// sentence claimed the CPU cannot run Gated DeltaNet, which has not been true +/// since #3091. pub fn unsupported_architecture_reason<'n>( architecture: &str, tensor_names: impl IntoIterator, @@ -86,15 +94,86 @@ pub fn unsupported_architecture_reason<'n>( let ssm_tensor = tensor_names .into_iter() .find(|name| name.contains("ssm_") || name.contains("ssm.")); - if let Some(tensor_name) = ssm_tensor { - return Some(format!( - "Architecture '{architecture}' uses SSM/Gated Delta Net layers (detected tensor '{tensor_name}'). \ - NEITHER the CPU nor the GPU backend implements Gated DeltaNet/SSM layers yet. \ - Tracking issues: #3090 (GPU) and #3091 (CPU). \ - Use a standard transformer model (e.g., Qwen2.5, LLaMA, Mistral) or wait for SSM support in a future release." - )); + let tensor_name = ssm_tensor?; + if hybrid_forward_handles(architecture) { + return None; } - None + Some(format!( + "Architecture '{architecture}' uses SSM/Gated Delta Net layers (detected tensor '{tensor_name}'). \ + There is no CPU or GPU forward for architecture '{architecture}' with SSM tensors. \ + Tracking issues: #3090 (GPU) and #3091 (CPU) — both implement the Qwen3.5 hybrid ('qwen35'), not this architecture. \ + Use a standard transformer model (e.g., Qwen2.5, LLaMA, Mistral) or wait for SSM support in a future release." + )) +} + +/// Is this the architecture string the hybrid (Gated `DeltaNet` + gated +/// attention) forward handles? +/// +/// This is the ONE spelling set the runtime dispatches on: `inference_result.rs` +/// compares the GGUF architecture to the literal `"qwen35"` and, on a match, +/// builds `forward_qwen35::Qwen35Model` (CPU, #3091), which +/// `cuda::forward_qwen35_cuda::Qwen35CudaModel` wraps on the GPU (#3090). +/// `apr qa`'s capability gate, `apr parity`'s refusal and `apr ptx-map` all ask +/// THIS function rather than keeping private copies, so a tool can no longer +/// admit a spelling the runtime refuses (or refuse one it serves). +/// +/// Deliberately exact, not case-insensitive and not a spelling family: +/// `qwen3_5` / `qwen3.5` reach no forward, so calling them handled would have +/// the tooling promise a load `apr run` cannot perform. +#[must_use] +pub fn hybrid_forward_handles(architecture: &str) -> bool { + architecture == "qwen35" +} + +/// Why the DENSE loader ([`QuantizedGGUFTransformer::from_gguf`]) cannot load +/// this file, even though the hybrid forward can run it. +/// +/// #3477: lifting the SSM refusal from [`unsupported_architecture_reason`] would +/// otherwise let `apr check`-style callers walk into the dense tensor lookups +/// with a Qwen3.5 GGUF and fail on a missing `blk.0.attn_q.weight` — the cryptic +/// surface the GH-704 refusal was introduced to replace. The dense loader is not +/// the hybrid path, so it refuses with a message that names the path that IS. +pub(crate) fn dense_loader_refusal<'n>( + architecture: &str, + tensor_names: impl IntoIterator, +) -> Option { + if !hybrid_forward_handles(architecture) { + return None; + } + let tensor_name = tensor_names + .into_iter() + .find(|name| name.contains("ssm_") || name.contains("ssm."))?; + Some(format!( + "Architecture '{architecture}' is the Qwen3.5 hybrid (Gated Delta Net, detected tensor '{tensor_name}'): \ + it runs through `Qwen35Model` (realizar::gguf::forward_qwen35, CPU #3091) or `Qwen35CudaModel` (GPU #3090), \ + NOT through the dense QuantizedGGUFTransformer loader, whose per-layer tensor names this file does not carry. \ + Load it with `apr run` / `apr chat`, which dispatch this architecture to the hybrid forward." + )) +} + +/// The GPU-quant refusal for a hybrid (Gated `DeltaNet`) GGUF, or `None`. +/// +/// The tensor set and the whitelist both live in `dtype.rs` beside +/// `gpu_unsupported_quant_qtype`, the single source of truth this delegates to +/// (PMAT-785); this is only the public, header-level entry point the tools call. +/// +/// Answers from the header alone — `(tensor name, GGML type)` pairs — so a +/// caller refuses BEFORE uploading a single weight. `None` for every +/// architecture the hybrid forward does not handle: the dense quant gate +/// already speaks for those. +pub fn hybrid_gpu_quant_refusal<'n>( + architecture: &str, + tensors: impl IntoIterator, +) -> Option { + if !hybrid_forward_handles(architecture) { + return None; + } + let (name, qtype) = super::loader::hybrid_gpu_unsupported_quant_tensor(tensors)?; + Some(format!( + "Architecture '{architecture}': tensor '{name}' has GGML type {qtype}, which has no verified GPU GEMV kernel. \ + The GPU weight upload would decode it as Q4_K and produce garbage logits (PMAT-781/783/785). \ + Run this model on the CPU (`apr run --no-gpu`), or requantize to Q4_0/Q4_1/Q5_0/Q8_0/Q4_K/Q5_K/Q6_K." + )) } /// Quantized GGUF Transformer for fused inference @@ -174,6 +253,18 @@ impl<'a> QuantizedGGUFTransformer<'a> { return Err(crate::RealizarError::FormatError { reason }); } + // #3477: the hybrid architectures are no longer refused by the shared + // predicate — `apr run` serves them through `Qwen35Model` / + // `Qwen35CudaModel`. THIS loader is still the dense one, so it refuses + // them here with a message that names the path that does load them, + // rather than falling through to a missing `blk.0.attn_q.weight`. + if let Some(reason) = dense_loader_refusal( + &config.architecture, + model.tensors.iter().map(|t| t.name.as_str()), + ) { + return Err(crate::RealizarError::FormatError { reason }); + } + // M32b: refuse Mixture-of-Experts architectures with a structured, // contract-named error before reaching the dense-FFN tensor lookup. // Replaces the pre-M32 cryptic "Tensor 'blk.0.ffn_up.weight' not @@ -813,44 +904,155 @@ mod tensor_byte_size_tests { #[cfg(test)] mod unsupported_architecture_tests { - use super::unsupported_architecture_reason; + use super::{ + dense_loader_refusal, hybrid_forward_handles, hybrid_gpu_quant_refusal, + unsupported_architecture_reason, + }; + + // #3477 (operator ruling 2026-09-19): the hybrid forward now exists on both + // backends for `qwen35` — `forward_qwen35` (CPU, #3091) and + // `Qwen35CudaModel` (GPU, #3090). The shared predicate must stop refusing + // it, or every tool that asks it (apr qa, apr parity, apr ptx-map) keeps + // contradicting the runtime. + #[test] + fn qwen35_gated_delta_net_tensors_are_admitted() { + assert_eq!( + unsupported_architecture_reason( + "qwen35", + [ + "token_embd.weight", + "blk.0.ssm_conv1d.weight", + "blk.0.attn_q.weight", + ], + ), + None, + "the hybrid forward runs qwen35 on both backends (#3090/#3091)" + ); + assert_eq!( + unsupported_architecture_reason("qwen35", ["blk.0.ssm.a"]), + None, + "the dotted `ssm.` spelling is the same architecture" + ); + } - // dogfood-0.63.0 #2399 finding 2: the SSM refusal must be answerable from - // tensor names alone, so read-only tools get the same verdict `from_gguf` - // gives. Qwen3.5 names its Gated Delta Net weights `blk.N.ssm_*`. + // dogfood-0.63.0 #2399 finding 2: the SSM refusal must still be answerable + // from tensor names alone for the architectures NO forward handles, so + // read-only tools get the same verdict `from_gguf` gives. #[test] - fn qwen35_gated_delta_net_tensors_are_refused() { + fn an_ssm_architecture_with_no_forward_is_still_refused() { let reason = unsupported_architecture_reason( - "qwen35", + "mamba", [ "token_embd.weight", "blk.0.ssm_conv1d.weight", "blk.0.attn_q.weight", ], ) - .expect("a GGUF carrying ssm_ tensors must be refused"); + .expect("an SSM architecture with no forward must be refused"); assert!( - reason.contains("qwen35") && reason.contains("SSM/Gated Delta Net"), + reason.contains("mamba") && reason.contains("SSM/Gated Delta Net"), "refusal must name the architecture and the reason, got: {reason}" ); - assert!( - reason.contains("NEITHER the CPU nor the GPU backend implements"), - "must mention both backends" - ); assert!(reason.contains("#3090"), "must mention GPU #3090"); assert!(reason.contains("#3091"), "must mention CPU #3091"); assert!( reason.contains("blk.0.ssm_conv1d.weight"), "must mention tensor name" ); + // #3091 gave the CPU a Gated DeltaNet forward and #3090 the GPU one; + // the old sentence claimed neither backend implements it at all. + assert!( + !reason.contains("NEITHER the CPU nor the GPU"), + "the 'neither backend implements Gated DeltaNet' claim is false since #3091: {reason}" + ); + // The dotted spelling (`blk.0.ssm.a`) is the other form seen in the wild. + assert!( + unsupported_architecture_reason("mamba", ["blk.0.ssm.a"]).is_some(), + "`ssm.` spelling must be refused too" + ); + } + + // The admission mirrors the runtime dispatch, which compares the GGUF + // architecture to the literal "qwen35" (infer/inference_result.rs). A + // spelling the runtime does not dispatch must keep the refusal, or the + // tooling would promise a load `apr run` cannot perform. + #[test] + fn unsupported_architecture_handled_set_is_the_runtime_dispatch_literal() { + assert!(hybrid_forward_handles("qwen35")); + for other in ["qwen3_5", "qwen3.5", "QWEN35", "mamba", "qwen2", "llama"] { + assert!( + !hybrid_forward_handles(other), + "'{other}' reaches no hybrid forward" + ); + assert!( + unsupported_architecture_reason(other, ["blk.0.ssm_a"]).is_some(), + "'{other}' carries SSM tensors and has no forward — it must be refused" + ); + } } - // The dotted spelling (`blk.0.ssm.a`) is the other form seen in the wild. + // The DENSE loader is not the hybrid path: it must refuse qwen35 with a + // message naming the path that does load it, instead of falling through to + // a missing `blk.0.attn_q.weight`. #[test] - fn dotted_ssm_tensor_names_are_refused() { + fn unsupported_architecture_dense_loader_still_refuses_the_hybrid() { + let reason = dense_loader_refusal("qwen35", ["token_embd.weight", "blk.0.ssm_a"]) + .expect("the dense loader cannot load a hybrid GGUF"); assert!( - unsupported_architecture_reason("qwen35", ["blk.0.ssm.a"]).is_some(), - "`ssm.` spelling must be refused too" + reason.contains("Qwen35Model") && reason.contains("forward_qwen35"), + "the refusal must name the loader that DOES work: {reason}" + ); + assert!( + !reason.contains("NEITHER the CPU nor the GPU"), + "the dense loader's limits are not the runtime's: {reason}" + ); + assert_eq!( + dense_loader_refusal("qwen2", ["blk.0.attn_q.weight"]), + None, + "a dense model must still load through the dense loader" + ); + assert_eq!( + dense_loader_refusal("mamba", ["blk.0.ssm_a"]), + None, + "an architecture no forward handles is refused by the shared predicate, not here" + ); + } + + // PMAT-781/783/785: the hybrid GPU upload decodes an unrecognized GGML type + // as Q4_K. The dense quant gate cannot see the Gated DeltaNet tensors (the + // qwen35 base model carries an empty layer list), so the header-level gate + // has to judge them by name. + #[test] + fn unsupported_architecture_hybrid_quant_gate_judges_the_deltanet_tensors() { + let reason = hybrid_gpu_quant_refusal( + "qwen35", + [ + ("token_embd.weight", 12u32), + ("blk.0.ssm_alpha.weight", 11), // Q3_K: no GPU GEMV kernel + ("blk.0.ffn_down.weight", 12), + ], + ) + .expect("a Q3_K DeltaNet projection has no GPU kernel — it must be refused"); + assert!( + reason.contains("blk.0.ssm_alpha.weight") && reason.contains("11"), + "the refusal must name the tensor and its GGML type: {reason}" + ); + assert_eq!( + hybrid_gpu_quant_refusal( + "qwen35", + [ + ("blk.0.attn_qkv.weight", 8u32), + ("blk.0.ssm_alpha.weight", 12), + ("blk.0.ssm_out.weight", 14), + ], + ), + None, + "Q8_0/Q4_K/Q6_K all have verified GPU GEMV kernels" + ); + assert_eq!( + hybrid_gpu_quant_refusal("qwen2", [("blk.0.attn_q.weight", 11u32)]), + None, + "a dense model is judged by the dense quant gate, not this one" ); } diff --git a/crates/aprender-serve/src/infer/inference_result.rs b/crates/aprender-serve/src/infer/inference_result.rs index ba92932fa9..40a279f606 100644 --- a/crates/aprender-serve/src/infer/inference_result.rs +++ b/crates/aprender-serve/src/infer/inference_result.rs @@ -242,16 +242,17 @@ fn run_gguf_inference( )?; (tokens, false) // CPU-only path; GPU MoE wiring is M32d follow-up } else if is_qwen35 { - if !config.no_gpu && cfg!(feature = "cuda") { - eprintln!("[qwen35: Gated DeltaNet runs on the CPU; the GPU backend does not implement it yet (#3090)]"); - } - let tokens = crate::gguf::forward_qwen35::run_qwen35_generate( + // #3477: the hybrid now has a GPU forward (#3090), so `apr run --gpu` + // routes to it and reports CUDA; the CPU forward (#3091) serves + // `--no-gpu`, a build without cuda, and any GPU failure — the last of + // which is printed, never silent. + crate::gguf::forward_qwen35::run_qwen35_generate_dispatch( &mapped, &model, &input_tokens, &gen_config, - )?; - (tokens, false) + config.no_gpu, + )? } else { run_gguf_generate(model, &input_tokens, &gen_config, config)? }; diff --git a/docs/audits/quorum-PMAT-3477-pr3517.json b/docs/audits/quorum-PMAT-3477-pr3517.json new file mode 100644 index 0000000000..bc3acfa5c5 --- /dev/null +++ b/docs/audits/quorum-PMAT-3477-pr3517.json @@ -0,0 +1 @@ +{"width":3,"executor":"agy","lanes":[{"lane":1,"status":"SUCCESS","verdict":"FAIL","summary":"The read-only review of PMAT-3477-gdn-gpu is complete. The GPU PTX implementations, F2 probe guarding, and GGUF refusal logic correctly reproduce the Qwen3.5 architecture and pass all checks (including the 256-wide KV attention loop quirk reproduction). However, the PR fails checklist item (3): the `forward_single` per-token loop contains an explicit `ex.sync_stream()?` at the end of every attention layer, and after the layer loop, it synchronizes again, downloads the intermediate `hidden` state to the host, and passes it to `hidden_to_logits` (which immediately re-uploads it to the GPU). This is a severe violation of the 'no host<->device sync inside the per-token loop beyond logits download' constraint.","findings":[{"claim":"State reset between F2 probe and generation is correct.","file":"crates/aprender-serve/src/infer/inference_result.rs","grounding":"asserted"},{"claim":"GPU `attention_layer_inner` introduces an unconditional host<->device sync inside the layer, stalling the GPU pipeline every 6th layer.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()?` from `attention_layer_inner`; `state.kv_len = state.kv_len.max(position + 1)` on the host does not require a device sync.","grounding":"cited","line":1041},{"claim":"GPU `forward_single` per-token loop syncs and downloads the hidden state to the CPU before the LM head, violating the 'no sync beyond logits download' rule.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Keep `dev` resident on the GPU and execute the final RMSNorm and LM head on the device buffer directly, instead of downloading it and passing it to `hidden_to_logits` (which just re-uploads it).","grounding":"cited","line":1371},{"claim":"Refusal sites and quant gates correctly identify the Gated DeltaNet architecture without masking dense models.","file":"crates/aprender-serve/src/gguf/load_verdict.rs","grounding":"asserted"},{"claim":"Float GEMV is properly pinned in Qwen35CudaModel::new to prevent DP4A int8 activation quantization errors from compounding through the recurrence.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","grounding":"cited","line":500}],"raw_bytes":5895,"err_bytes":424,"envelope_status":"SUCCESS","verdict_source":"structured_output","grounding_check":"parity","model":"gemini-3.1-pro-high","model_measured":"gemini-3.1-pro-high","model_source":"measured","family":"gemini"},{"lane":2,"status":"SUCCESS","verdict":"FAIL","summary":"Pre-merge review of branch PMAT-3477-gdn-gpu (PR: Qwen3.5 Gated-DeltaNet hybrid on CUDA, GitHub #3090, aprender 0.68.2 EPIC #3477) against origin/main. Verdict: PASS-WITH-CHANGES. The PTX kernels, delta-rule recurrence, partial NEOX RoPE, GQA decode attention, executor wrappers, F2 runtime guard, and refusal sites are mathematically correct and adhere to CPU reference specifications. However, per-layer stream synchronizations and an extraneous host DtoH/HtoD roundtrip in forward_single violate the zero-sync requirement and must be resolved before merge.","findings":[{"claim":"ex.sync_stream()?; in attention_layer_inner causes redundant host-device stream synchronization on every attention layer in the per-token loop.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()?;` from line 1041. Layers in the per-token loop should execute asynchronously on the CUDA stream without synchronizing until logits are downloaded.","grounding":"cited","line":1041},{"claim":"ex.sync_stream() in deltanet_layer_inner causes redundant host-device stream synchronization on every DeltaNet layer in the per-token loop.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()` from line 1318 and return `Ok(())`. Streams must remain unsynchronized across layer boundaries.","grounding":"cited","line":1318},{"claim":"forward_single synchronizes the stream, copies the hidden buffer to host, and passes it to hidden_to_logits which immediately copies it back to device via pad_and_upload_input.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"In forward_single (lines 1370-1388), avoid copying `hidden` to host and back to device. Apply output RMSNorm and LM head GEMV directly on the device buffer `dev` on GPU and synchronize only at final logits download.","grounding":"cited","line":1370},{"claim":"qwen35_cuda_the_parity_floor_is_the_cpu_references_own_activation_quantization explicitly calls gpu.pin_reference_gemv(), which masks whether Qwen35CudaModel::with_max_seq_len properly defaults to pinned float GEMV.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs","fix":"Remove line 1172 (`gpu.pin_reference_gemv();`) so the test exercises the constructor's default float GEMV configuration rather than manually re-pinning it.","grounding":"cited","line":1172}],"raw_bytes":6238,"err_bytes":404,"envelope_status":"SUCCESS","verdict_source":"structured_output","grounding_check":"parity","model":"gemini-3.8-flash-high","model_measured":"gemini-3.8-flash-high","model_source":"measured","family":"gemini"},{"lane":3,"status":"SUCCESS","verdict":"PASS","summary":"Comprehensive pre-merge review of branch PMAT-3477-gdn-gpu (PR: Qwen3.5 Gated-DeltaNet hybrid on CUDA, GitHub #3090, aprender 0.68.2 EPIC #3477) against origin/main. All 8 evaluation criteria (kernel correctness vs CPU reference, executor wrappers with graph recording, Qwen35CudaModel state and KV cache sizing, F2 hybrid guard validation, refusal site truthfulness and quant whitelisting, pinned float GEMV verification and DP4A falsification, test honesty and mutant coverage, and scope containment) have been rigorously verified and found fully compliant. Verdict: PASS.","findings":[{"claim":"Causal conv1d kernel correctly implements window shift (state[c][0..K-2] = state[c][1..K-1], state[c][K-2] = input[c]), pre-shift load order, and output SiLU activation exactly matching CPU reference.","file":"crates/aprender-gpu/src/kernels/gdn/causal_conv1d.rs","fix":"None required (verified correct).","grounding":"cited","line":118},{"claim":"Delta-rule recurrence kernel correctly maps state memory row j to S column j (s_h[j*D+i] = S[i][j]), assigns row j exclusively to thread j, computes decay, delta, update, and output projections in ascending i order with scale D^-0.5 matching CPU delta_rule_recurrence.","file":"crates/aprender-gpu/src/kernels/gdn/delta_rule.rs","fix":"None required (verified correct).","grounding":"cited","line":121},{"claim":"Gated RMSNorm kernel correctly normalizes the input only (gate is not normalized), computes rms_scale = 1/sqrt(mean(input^2) + eps), multiplies shared per-head weight, and applies silu(gate) matching CPU gated_rmsnorm.","file":"crates/aprender-gpu/src/kernels/gdn/gated_rmsnorm.rs","fix":"None required (verified correct).","grounding":"cited","line":118},{"claim":"Per-head L2 norm kernel correctly places epsilon on the sum of squares (sum(x_h^2) + eps) per head without dividing by head dimension, matching CPU l2_norm_per_head.","file":"crates/aprender-gpu/src/kernels/gdn/l2_norm.rs","fix":"None required (verified correct).","grounding":"cited","line":101},{"claim":"GDN gates kernel correctly computes dt[h] = softplus(alpha[h] + dt_bias[h]) * a[h] with x > 20 branching and beta[h] = sigmoid(beta_raw[h]) matching CPU reference.","file":"crates/aprender-gpu/src/kernels/gdn/gdn_gates.rs","fix":"None required (verified correct).","grounding":"cited","line":86},{"claim":"Single-query decode attention kernel correctly supports head_dim up to 256, implements online max-subtracted softmax with multi-pass running sum/max rescaling, avoiding the head_dim <= 128 truncation defect of kernels/attention.","file":"crates/aprender-gpu/src/kernels/gdn/decode_attention.rs","fix":"None required (verified correct).","grounding":"cited","line":240},{"claim":"Partial NEOX RoPE rotates only dimensions 0..n_rot across pairs (j, j + n_rot/2) using host-computed theta_scale = freq_base^(-2/n_rot) and Cody-Waite range reduction before hardware sine/cosine approximation.","file":"crates/aprender-gpu/src/kernels/gdn/partial_rope.rs","fix":"None required (verified correct).","grounding":"cited","line":115},{"claim":"Split interleaved kernel performs pure data movement separating interleaved [q | gate] attention projections into contiguous q and gate buffers matching CPU copy_from_slice.","file":"crates/aprender-gpu/src/kernels/gdn/split_interleave.rs","fix":"None required (verified correct).","grounding":"cited","line":80},{"claim":"All executor wrappers in gdn_ops.rs perform graph_recording RecordedKernel pushes with arg_data matching launch parameters in order, and use unique cache keys parameterized by dimensions and constants.","file":"crates/aprender-serve/src/cuda/executor/gdn_ops.rs","fix":"None required (verified correct).","grounding":"cited","line":75},{"claim":"Qwen35CudaModel dynamically sizes conv, recurrent, and KV cache states from config, matches CPU 256-wide head dimension in attention loop and KV cache, and resets device state cleanly between F2 validation and generation.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"None required (verified correct).","grounding":"cited","line":573},{"claim":"F2 hybrid runtime guard f2_validate_qwen35 uses CPU hybrid model (Qwen35Model::forward_single_qwen35) as reference, runs serial probe on throwaway states, maintains standard floors (0.95/0.98/0.90), and prints failure notices before fallback to CPU generation.","file":"crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs","fix":"None required (verified correct).","grounding":"cited","line":1384},{"claim":"Refusal sites (unsupported_architecture_reason, dense_loader_refusal, hybrid_ssm_verdict, hybrid_gpu_quant_refusal) accurately admit qwen35 on GPU under cuda feature while refusing unhandled SSM architectures and dense loader lookups with truthful messages.","file":"crates/aprender-serve/src/gguf/transformer.rs","fix":"None required (verified correct).","grounding":"cited","line":90},{"claim":"HYBRID_GPU_PROJECTIONS quant whitelist in dtype.rs strictly validates attn_qkv, ssm_alpha, ssm_beta, attn_gate, ssm_out, attention projections, and FFN projections against verified GPU GEMV kernels.","file":"crates/aprender-serve/src/gguf/dtype.rs","fix":"None required (verified correct).","grounding":"cited","line":58},{"claim":"Qwen35CudaModel pins float MWV GEMV kernels (q4k=Mwv, q6k=Mwv) across all layer and lm_head GEMV dispatches, verified by dp4a_gemv_is_catastrophic_through_the_recurrence falsifier test.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"None required (verified correct).","grounding":"cited","line":518}],"raw_bytes":12834,"err_bytes":404,"envelope_status":"SUCCESS","verdict_source":"structured_output","grounding_check":"parity","model":"gemini-3.7-flash-high","model_measured":"gemini-3.7-flash-high","model_source":"measured","family":"gemini"}],"agreed":false,"dissent":[{"lane":1,"verdict":"FAIL","summary":"The read-only review of PMAT-3477-gdn-gpu is complete. The GPU PTX implementations, F2 probe guarding, and GGUF refusal logic correctly reproduce the Qwen3.5 architecture and pass all checks (including the 256-wide KV attention loop quirk reproduction). However, the PR fails checklist item (3): the `forward_single` per-token loop contains an explicit `ex.sync_stream()?` at the end of every attention layer, and after the layer loop, it synchronizes again, downloads the intermediate `hidden` state to the host, and passes it to `hidden_to_logits` (which immediately re-uploads it to the GPU). This is a severe violation of the 'no host<->device sync inside the per-token loop beyond logits download' constraint.","findings":[{"claim":"State reset between F2 probe and generation is correct.","file":"crates/aprender-serve/src/infer/inference_result.rs","grounding":"asserted"},{"claim":"GPU `attention_layer_inner` introduces an unconditional host<->device sync inside the layer, stalling the GPU pipeline every 6th layer.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()?` from `attention_layer_inner`; `state.kv_len = state.kv_len.max(position + 1)` on the host does not require a device sync.","grounding":"cited","line":1041},{"claim":"GPU `forward_single` per-token loop syncs and downloads the hidden state to the CPU before the LM head, violating the 'no sync beyond logits download' rule.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Keep `dev` resident on the GPU and execute the final RMSNorm and LM head on the device buffer directly, instead of downloading it and passing it to `hidden_to_logits` (which just re-uploads it).","grounding":"cited","line":1371},{"claim":"Refusal sites and quant gates correctly identify the Gated DeltaNet architecture without masking dense models.","file":"crates/aprender-serve/src/gguf/load_verdict.rs","grounding":"asserted"},{"claim":"Float GEMV is properly pinned in Qwen35CudaModel::new to prevent DP4A int8 activation quantization errors from compounding through the recurrence.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","grounding":"cited","line":500}]},{"lane":2,"verdict":"FAIL","summary":"Pre-merge review of branch PMAT-3477-gdn-gpu (PR: Qwen3.5 Gated-DeltaNet hybrid on CUDA, GitHub #3090, aprender 0.68.2 EPIC #3477) against origin/main. Verdict: PASS-WITH-CHANGES. The PTX kernels, delta-rule recurrence, partial NEOX RoPE, GQA decode attention, executor wrappers, F2 runtime guard, and refusal sites are mathematically correct and adhere to CPU reference specifications. However, per-layer stream synchronizations and an extraneous host DtoH/HtoD roundtrip in forward_single violate the zero-sync requirement and must be resolved before merge.","findings":[{"claim":"ex.sync_stream()?; in attention_layer_inner causes redundant host-device stream synchronization on every attention layer in the per-token loop.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()?;` from line 1041. Layers in the per-token loop should execute asynchronously on the CUDA stream without synchronizing until logits are downloaded.","grounding":"cited","line":1041},{"claim":"ex.sync_stream() in deltanet_layer_inner causes redundant host-device stream synchronization on every DeltaNet layer in the per-token loop.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"Remove `ex.sync_stream()` from line 1318 and return `Ok(())`. Streams must remain unsynchronized across layer boundaries.","grounding":"cited","line":1318},{"claim":"forward_single synchronizes the stream, copies the hidden buffer to host, and passes it to hidden_to_logits which immediately copies it back to device via pad_and_upload_input.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","fix":"In forward_single (lines 1370-1388), avoid copying `hidden` to host and back to device. Apply output RMSNorm and LM head GEMV directly on the device buffer `dev` on GPU and synchronize only at final logits download.","grounding":"cited","line":1370},{"claim":"qwen35_cuda_the_parity_floor_is_the_cpu_references_own_activation_quantization explicitly calls gpu.pin_reference_gemv(), which masks whether Qwen35CudaModel::with_max_seq_len properly defaults to pinned float GEMV.","file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs","fix":"Remove line 1172 (`gpu.pin_reference_gemv();`) so the test exercises the constructor's default float GEMV configuration rather than manually re-pinning it.","grounding":"cited","line":1172}]}],"dedup":[{"file":"crates/aprender-gpu/src/kernels/gdn/causal_conv1d.rs","line":118,"lanes_agreeing":[3],"claims":["Causal conv1d kernel correctly implements window shift (state[c][0..K-2] = state[c][1..K-1], state[c][K-2] = input[c]), pre-shift load order, and output SiLU activation exactly matching CPU reference."]},{"file":"crates/aprender-gpu/src/kernels/gdn/decode_attention.rs","line":240,"lanes_agreeing":[3],"claims":["Single-query decode attention kernel correctly supports head_dim up to 256, implements online max-subtracted softmax with multi-pass running sum/max rescaling, avoiding the head_dim <= 128 truncation defect of kernels/attention."]},{"file":"crates/aprender-gpu/src/kernels/gdn/delta_rule.rs","line":121,"lanes_agreeing":[3],"claims":["Delta-rule recurrence kernel correctly maps state memory row j to S column j (s_h[j*D+i] = S[i][j]), assigns row j exclusively to thread j, computes decay, delta, update, and output projections in ascending i order with scale D^-0.5 matching CPU delta_rule_recurrence."]},{"file":"crates/aprender-gpu/src/kernels/gdn/gated_rmsnorm.rs","line":118,"lanes_agreeing":[3],"claims":["Gated RMSNorm kernel correctly normalizes the input only (gate is not normalized), computes rms_scale = 1/sqrt(mean(input^2) + eps), multiplies shared per-head weight, and applies silu(gate) matching CPU gated_rmsnorm."]},{"file":"crates/aprender-gpu/src/kernels/gdn/gdn_gates.rs","line":86,"lanes_agreeing":[3],"claims":["GDN gates kernel correctly computes dt[h] = softplus(alpha[h] + dt_bias[h]) * a[h] with x > 20 branching and beta[h] = sigmoid(beta_raw[h]) matching CPU reference."]},{"file":"crates/aprender-gpu/src/kernels/gdn/l2_norm.rs","line":101,"lanes_agreeing":[3],"claims":["Per-head L2 norm kernel correctly places epsilon on the sum of squares (sum(x_h^2) + eps) per head without dividing by head dimension, matching CPU l2_norm_per_head."]},{"file":"crates/aprender-gpu/src/kernels/gdn/partial_rope.rs","line":115,"lanes_agreeing":[3],"claims":["Partial NEOX RoPE rotates only dimensions 0..n_rot across pairs (j, j + n_rot/2) using host-computed theta_scale = freq_base^(-2/n_rot) and Cody-Waite range reduction before hardware sine/cosine approximation."]},{"file":"crates/aprender-gpu/src/kernels/gdn/split_interleave.rs","line":80,"lanes_agreeing":[3],"claims":["Split interleaved kernel performs pure data movement separating interleaved [q | gate] attention projections into contiguous q and gate buffers matching CPU copy_from_slice."]},{"file":"crates/aprender-serve/src/cuda/executor/gdn_ops.rs","line":75,"lanes_agreeing":[3],"claims":["All executor wrappers in gdn_ops.rs perform graph_recording RecordedKernel pushes with arg_data matching launch parameters in order, and use unique cache keys parameterized by dimensions and constants."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":500,"lanes_agreeing":[1],"claims":["Float GEMV is properly pinned in Qwen35CudaModel::new to prevent DP4A int8 activation quantization errors from compounding through the recurrence."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":518,"lanes_agreeing":[3],"claims":["Qwen35CudaModel pins float MWV GEMV kernels (q4k=Mwv, q6k=Mwv) across all layer and lm_head GEMV dispatches, verified by dp4a_gemv_is_catastrophic_through_the_recurrence falsifier test."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":573,"lanes_agreeing":[3],"claims":["Qwen35CudaModel dynamically sizes conv, recurrent, and KV cache states from config, matches CPU 256-wide head dimension in attention loop and KV cache, and resets device state cleanly between F2 validation and generation."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":1041,"lanes_agreeing":[1,2],"claims":["GPU `attention_layer_inner` introduces an unconditional host<->device sync inside the layer, stalling the GPU pipeline every 6th layer.","ex.sync_stream()?; in attention_layer_inner causes redundant host-device stream synchronization on every attention layer in the per-token loop."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":1318,"lanes_agreeing":[2],"claims":["ex.sync_stream() in deltanet_layer_inner causes redundant host-device stream synchronization on every DeltaNet layer in the per-token loop."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":1370,"lanes_agreeing":[2],"claims":["forward_single synchronizes the stream, copies the hidden buffer to host, and passes it to hidden_to_logits which immediately copies it back to device via pad_and_upload_input."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda.rs","line":1371,"lanes_agreeing":[1],"claims":["GPU `forward_single` per-token loop syncs and downloads the hidden state to the CPU before the LM head, violating the 'no sync beyond logits download' rule."]},{"file":"crates/aprender-serve/src/gguf/cuda/forward_qwen35_cuda_tests.rs","line":1172,"lanes_agreeing":[2],"claims":["qwen35_cuda_the_parity_floor_is_the_cpu_references_own_activation_quantization explicitly calls gpu.pin_reference_gemv(), which masks whether Qwen35CudaModel::with_max_seq_len properly defaults to pinned float GEMV."]},{"file":"crates/aprender-serve/src/gguf/dtype.rs","line":58,"lanes_agreeing":[3],"claims":["HYBRID_GPU_PROJECTIONS quant whitelist in dtype.rs strictly validates attn_qkv, ssm_alpha, ssm_beta, attn_gate, ssm_out, attention projections, and FFN projections against verified GPU GEMV kernels."]},{"file":"crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs","line":1384,"lanes_agreeing":[3],"claims":["F2 hybrid runtime guard f2_validate_qwen35 uses CPU hybrid model (Qwen35Model::forward_single_qwen35) as reference, runs serial probe on throwaway states, maintains standard floors (0.95/0.98/0.90), and prints failure notices before fallback to CPU generation."]},{"file":"crates/aprender-serve/src/gguf/load_verdict.rs","line":null,"lanes_agreeing":[1],"claims":["Refusal sites and quant gates correctly identify the Gated DeltaNet architecture without masking dense models."]},{"file":"crates/aprender-serve/src/gguf/transformer.rs","line":90,"lanes_agreeing":[3],"claims":["Refusal sites (unsupported_architecture_reason, dense_loader_refusal, hybrid_ssm_verdict, hybrid_gpu_quant_refusal) accurately admit qwen35 on GPU under cuda feature while refusing unhandled SSM architectures and dense loader lookups with truthful messages."]},{"file":"crates/aprender-serve/src/infer/inference_result.rs","line":null,"lanes_agreeing":[1],"claims":["State reset between F2 probe and generation is correct."]}],"uncovered":[],"coverage_source":"lanes","partial":false,"partial_reasons":[],"extra":{},"author":{"model":"Fable 5.1","family":"claude","source":"flag"}} \ No newline at end of file