diff --git a/contracts/qwen35-e2e-verification-v1.yaml b/contracts/qwen35-e2e-verification-v1.yaml index 883bf48435..6b688e3d9e 100644 --- a/contracts/qwen35-e2e-verification-v1.yaml +++ b/contracts/qwen35-e2e-verification-v1.yaml @@ -93,11 +93,33 @@ proof_obligations: property: Parameter count matches architecture formal: P(Qwen3.5-9B) ∈ [9.0B, 9.2B] applies_to: all + note: >- + #3091: the 9B instantiation is NOT proved, and is not asserted anywhere. This + contract's own model_parameter_count formula, evaluated on the in-tree + constants (contracts/model-families/qwen3_5.yaml size_variants.9b: V=248320, + d=4096, L=32, n_h=16, d_k=256, intermediate_dim=12288, untied lm_head), sums to + 8,208,535,552 = 8.209B -- outside [9.0B, 9.2B]. That descriptor's + tensor_template states no Gated DeltaNet tensors, so the ~0.8B difference has no + in-tree shape to come from, and whether the range or the descriptor is wrong is + open. qe2e_inv_001_parameter_sum_matches_tensor_inventory discharges the + model_parameter_count EQUATION (the config-derived sum equals the measured + tensor inventory) on the real Qwen3.5 layer structures; the range instantiation + stays unproved rather than being widened to fit. - id: QE2E-BND-002 type: bound property: FLOPs bounded by 2P - formal: F <= 2 * P + O(seq_len * d * L) + formal: F(s) <= 2*P + 4*L_attn*d_kv*s applies_to: all + note: >- + ont-delta (#3091): tightened from 'F <= 2 * P + O(seq_len * d * L)'. An O() + with no constant states no bound at all -- every F satisfies it, so no input + could falsify the obligation and FALSIFY-QE2E-002 had nothing to assert. The + constant is now named: s = cached positions, L_attn = full-attention layers + (a Gated DeltaNet layer carries a fixed-size recurrent state, so its cost does + not grow with s), d_kv = num_kv_heads * head_dim, and QK^T and A*V each cost + 2*d_kv per cached position per such layer. The slope is asserted as an + EQUALITY by qe2e_bnd_002_flops_bounded_by_2p_plus_attention_term, so the + constant cannot be lowered either. - id: QE2E-ORD-003 type: ordering property: Quantization memory ordering @@ -140,17 +162,17 @@ falsification_tests: - id: FALSIFY-QE2E-002 rule: FLOPs estimate prediction: 2P FLOPs per forward token - test: Deterministic with Qwen3.5-9B constants + test: qe2e_bnd_002_flops_bounded_by_2p_plus_attention_term if_fails: Missing layer in FLOP count - id: FALSIFY-QE2E-003 rule: Memory ordering prediction: Q4K < Q6K < F16 < F32 memory - test: proptest with random tensor dimensions + test: qe2e_ord_003_quantization_memory_ordering if_fails: Quantization byte formula wrong - id: FALSIFY-QE2E-004 rule: Throughput roofline prediction: tok/s bounded by bandwidth and compute - test: proptest with random hardware specs + test: qe2e_mon_004_throughput_is_monotone_in_bandwidth if_fails: Roofline formula error - id: FALSIFY-QE2E-005 rule: Coverage completeness @@ -160,12 +182,12 @@ falsification_tests: - id: FALSIFY-QE2E-006 rule: Compositional proof structure prediction: 'Each block preserves shape: shape(block_l(x)) = shape(x)' - test: 'proptest: verify shape(block(x)) = shape(x) for random blocks' + test: qe2e_inv_006_every_block_preserves_d_model if_fails: Block l breaks shape invariant - id: FALSIFY-QE2E-007 rule: End-to-end shape conservation prediction: tokens → [seq_len, d] → ... → [seq_len, V] - test: 'proptest: trace shapes through mock pipeline' + test: qe2e_con_007_tokens_in_logits_out_shape if_fails: Shape break in layer composition kani_harnesses: - id: KANI-QE2E-001 diff --git a/crates/apr-cli/src/commands/chat_generate_session_02.rs b/crates/apr-cli/src/commands/chat_generate_session_02.rs index 28bc2dc390..7761be9191 100644 --- a/crates/apr-cli/src/commands/chat_generate_session_02.rs +++ b/crates/apr-cli/src/commands/chat_generate_session_02.rs @@ -264,47 +264,8 @@ impl ChatSession { } } - // CPU path — create fresh OwnedQuantizedModel from cached or fresh mapped - let model = OwnedQuantizedModel::from_mapped(mapped) - .map_err(|e| format!("Failed to create GGUF model: {e}"))?; - - // APR-TRACE-001: Use traced generation when --trace is enabled - let output_tokens = if config.trace { - use realizar::{InferenceTracer, ModelInfo, TraceConfig}; - - let trace_config = TraceConfig { - enabled: true, - verbose: false, - output: config.trace_output.clone(), - ..Default::default() - }; - - let mut tracer = InferenceTracer::new(trace_config); - tracer.set_model_info(ModelInfo { - name: "GGUF Model (CPU)".to_string(), - num_layers: model.config().num_layers, - hidden_dim: model.config().hidden_dim, - vocab_size: model.config().vocab_size, - num_heads: model.config().num_heads, - quant_type: None, - }); - - eprintln!("Warning: CPU traced generation not implemented, using non-traced path"); - - let result = model - .generate_with_cache(&prompt_tokens, &gen_config) - .map_err(|e| format!("GGUF generate failed: {e}"))?; - - if let Err(e) = tracer.write_output() { - eprintln!("Warning: Failed to write trace output: {e}"); - } - - result - } else { - model - .generate_with_cache(&prompt_tokens, &gen_config) - .map_err(|e| format!("GGUF generate failed: {e}"))? - }; + let output_tokens = + Self::generate_gguf_cpu_tokens(mapped, &prompt_tokens, &gen_config, config)?; let new_tokens = if output_tokens.len() > prompt_len { &output_tokens[prompt_len..] @@ -316,6 +277,75 @@ impl ChatSession { Ok(decoded) } + /// CPU generation for one GGUF chat turn: the Qwen3.5 hybrid forward (#3091), which the + /// dense `OwnedQuantizedModel` loader refuses, or the dense model. Returns the prompt + /// followed by the generated tokens. + fn generate_gguf_cpu_tokens( + mapped: &realizar::gguf::MappedGGUFModel, + prompt_tokens: &[u32], + gen_config: &realizar::gguf::QuantizedGenerateConfig, + config: &ChatConfig, + ) -> Result, String> { + use realizar::gguf::OwnedQuantizedModel; + + if mapped.model.architecture() == Some("qwen35") { + let base = realizar::gguf::forward_qwen35::Qwen35Model::create_base_model( + &mapped.model, + mapped.data(), + ) + .map_err(|e| format!("Failed to load the Qwen3.5 base model: {e}"))?; + return realizar::gguf::forward_qwen35::run_qwen35_generate( + mapped, + &base, + prompt_tokens, + gen_config, + ) + .map_err(|e| format!("Qwen3.5 generate failed: {e}")); + } + + // CPU path — create fresh OwnedQuantizedModel from cached or fresh mapped + let model = OwnedQuantizedModel::from_mapped(mapped) + .map_err(|e| format!("Failed to create GGUF model: {e}"))?; + + // APR-TRACE-001: Use traced generation when --trace is enabled + if config.trace { + use realizar::{InferenceTracer, ModelInfo, TraceConfig}; + + let trace_config = TraceConfig { + enabled: true, + verbose: false, + output: config.trace_output.clone(), + ..Default::default() + }; + + let mut tracer = InferenceTracer::new(trace_config); + tracer.set_model_info(ModelInfo { + name: "GGUF Model (CPU)".to_string(), + num_layers: model.config().num_layers, + hidden_dim: model.config().hidden_dim, + vocab_size: model.config().vocab_size, + num_heads: model.config().num_heads, + quant_type: None, + }); + + eprintln!("Warning: CPU traced generation not implemented, using non-traced path"); + + let result = model + .generate_with_cache(prompt_tokens, gen_config) + .map_err(|e| format!("GGUF generate failed: {e}"))?; + + if let Err(e) = tracer.write_output() { + eprintln!("Warning: Failed to write trace output: {e}"); + } + + Ok(result) + } else { + model + .generate_with_cache(prompt_tokens, gen_config) + .map_err(|e| format!("GGUF generate failed: {e}")) + } + } + fn generate_apr( &mut self, prompt: &[u32], diff --git a/crates/apr-cli/src/commands/chat_load_tokenizers.rs b/crates/apr-cli/src/commands/chat_load_tokenizers.rs index 0368122731..4ba21c22ad 100644 --- a/crates/apr-cli/src/commands/chat_load_tokenizers.rs +++ b/crates/apr-cli/src/commands/chat_load_tokenizers.rs @@ -137,6 +137,13 @@ fn try_init_gguf_cuda( mapped: &realizar::gguf::MappedGGUFModel, ) -> Result<(Option, bool), crate::error::CliError> { use realizar::gguf::{OwnedQuantizedModel, OwnedQuantizedModelCuda}; + // #3091: Qwen3.5/Qwen3.8 hybrids have a CPU forward but no GPU one yet (#3090). Skip the + // CUDA attempt instead of failing it, so chat does not report a load error for a model + // it can run. + if mapped.model.architecture() == Some("qwen35") { + eprintln!("[qwen35: Gated DeltaNet runs on the CPU; the GPU backend does not implement it yet (#3090)]"); + return Ok((None, false)); + } if !OwnedQuantizedModelCuda::is_available() { return Ok((None, false)); } diff --git a/crates/aprender-serve/examples/qwen35_parity.rs b/crates/aprender-serve/examples/qwen35_parity.rs new file mode 100644 index 0000000000..c6fc75ba1f --- /dev/null +++ b/crates/aprender-serve/examples/qwen35_parity.rs @@ -0,0 +1,188 @@ +//! Qwen3.5 CPU parity against llama.cpp (#3091): greedy free-running and teacher-forced. +//! +//! Reads `PROMPT_TOKENS:` / `OUTPUT_TOKENS:` pairs that llama.cpp master produced greedily +//! from the evidence file, runs the same prompts through realizar's Qwen3.5 CPU forward, and +//! writes `parity.json` next to the evidence file. +//! +//! Gate: at every teacher-forced step llama.cpp's token must be realizar's argmax, or a +//! near-tie, meaning realizar's rank 2 with a logit gap of at most `QWEN35_PARITY_TAU` +//! (default 0.25). Exact greedy equality between two backends with different accumulation +//! precision is a coin flip at a near-tie (#2359), so the gaps are recorded, never hidden. +//! +//! Usage: `cargo run --release -p aprender-serve --example qwen35_parity -- [tokens.txt]` + +use realizar::gguf::forward_qwen35::{Qwen35Model, Qwen35State}; +use realizar::gguf::MappedGGUFModel; +use serde::Serialize; +use std::io::{BufRead, BufReader}; +use std::path::Path; + +#[derive(Serialize)] +struct Disagreement { + step: usize, + llama: u32, + realizar: u32, + gap: f32, + rank: usize, +} + +#[derive(Serialize)] +struct PromptParity { + prompt_len: usize, + llama_tokens: Vec, + greedy_tokens: Vec, + greedy_divergence: Option, + teacher_forced_agree: usize, + teacher_forced_steps: usize, + disagreements: Vec, + min_agreeing_margin: f32, +} + +#[derive(Serialize)] +struct Report { + model: String, + tau: f32, + pass: bool, + prompts: Vec, +} + +/// (argmax, best logit, second-best logit) +fn top2(logits: &[f32]) -> (u32, f32, f32) { + let (mut best, mut best_v, mut second) = (0usize, f32::MIN, f32::MIN); + for (j, &v) in logits.iter().enumerate() { + if v > best_v { + second = best_v; + best = j; + best_v = v; + } else if v > second { + second = v; + } + } + (u32::try_from(best).unwrap_or(u32::MAX), best_v, second) +} + +fn read_pairs(path: &Path) -> std::io::Result, Vec)>> { + let parse = |s: &str| { + s.split_whitespace() + .filter_map(|t| t.parse::().ok()) + .collect::>() + }; + let (mut prompts, mut outputs) = (Vec::new(), Vec::new()); + for line in BufReader::new(std::fs::File::open(path)?).lines() { + let line = line?; + if let Some(rest) = line.strip_prefix("PROMPT_TOKENS:") { + prompts.push(parse(rest)); + } else if let Some(rest) = line.strip_prefix("OUTPUT_TOKENS:") { + outputs.push(parse(rest)); + } + } + Ok(prompts.into_iter().zip(outputs).collect()) +} + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let model_path = args + .next() + .ok_or("usage: qwen35_parity [tokens.txt]")?; + let tokens_path = args + .next() + .unwrap_or_else(|| "evidence/parity/qwen35/lambda/llama_tokens.txt".to_string()); + let tau: f32 = std::env::var("QWEN35_PARITY_TAU") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0.25); + + let mapped = MappedGGUFModel::from_path(&model_path)?; + let base = Qwen35Model::create_base_model(&mapped.model, mapped.data())?; + let qwen = Qwen35Model::from_model_and_layers(&base, &mapped.model, mapped.data())?; + let new_state = |max_seq_len: usize| qwen.new_state(max_seq_len); + + let mut prompts = Vec::new(); + for (prompt, want) in read_pairs(Path::new(&tokens_path))? { + let steps = want.len().min(32); + let prefill = |state: &mut Qwen35State| -> realizar::error::Result> { + let mut logits = Vec::new(); + for (pos, &token) in prompt.iter().enumerate() { + logits = qwen.forward_single_qwen35(token, state, pos)?; + } + Ok(logits) + }; + + // Free-running greedy: realizar feeds back its own argmax. + let mut state = new_state(prompt.len() + steps + 1); + let mut logits = prefill(&mut state)?; + let mut greedy = Vec::with_capacity(steps); + for i in 0..steps { + let (next, _, _) = top2(&logits); + greedy.push(next); + if i + 1 < steps { + logits = qwen.forward_single_qwen35(next, &mut state, prompt.len() + i)?; + } + } + let greedy_divergence = greedy.iter().zip(&want).position(|(a, b)| a != b); + + // Teacher-forced: realizar is fed llama.cpp's tokens, so every step is judged. + let mut state = new_state(prompt.len() + steps + 1); + let mut logits = prefill(&mut state)?; + let (mut agree, mut min_margin, mut disagreements) = (0usize, f32::MAX, Vec::new()); + for (i, &llama) in want.iter().take(steps).enumerate() { + let (best, best_v, second) = top2(&logits); + if best == llama { + agree += 1; + min_margin = min_margin.min(best_v - second); + } else { + let llama_v = logits.get(llama as usize).copied().unwrap_or(f32::MIN); + let rank = logits.iter().filter(|&&v| v > llama_v).count() + 1; + disagreements.push(Disagreement { + step: i, + llama, + realizar: best, + gap: best_v - llama_v, + rank, + }); + } + if i + 1 < steps { + logits = qwen.forward_single_qwen35(llama, &mut state, prompt.len() + i)?; + } + } + println!( + "prompt {} ({} tokens): greedy divergence {:?}; teacher-forced {agree}/{steps}; disagreements {:?}", + prompts.len(), + prompt.len(), + greedy_divergence, + disagreements.iter().map(|d| (d.step, d.rank, d.gap)).collect::>() + ); + prompts.push(PromptParity { + prompt_len: prompt.len(), + llama_tokens: want[..steps].to_vec(), + greedy_tokens: greedy, + greedy_divergence, + teacher_forced_agree: agree, + teacher_forced_steps: steps, + disagreements, + min_agreeing_margin: min_margin, + }); + } + + let pass = !prompts.is_empty() + && prompts + .iter() + .all(|p| p.disagreements.iter().all(|d| d.rank <= 2 && d.gap <= tau)); + let report = Report { + model: model_path, + tau, + pass, + prompts, + }; + let out = Path::new(&tokens_path).with_file_name("parity.json"); + std::fs::write(&out, serde_json::to_string_pretty(&report)?)?; + println!( + "{} (tau {tau}); wrote {}", + if pass { "PASS" } else { "FAIL" }, + out.display() + ); + if !pass { + std::process::exit(1); + } + Ok(()) +} diff --git a/crates/aprender-serve/src/api/effective_config.rs b/crates/aprender-serve/src/api/effective_config.rs index 6a27999365..81513d2a87 100644 --- a/crates/aprender-serve/src/api/effective_config.rs +++ b/crates/aprender-serve/src/api/effective_config.rs @@ -869,9 +869,6 @@ mod effective_config_tests { ); } - /// The counter's peak is a high-water mark, not the current value, and - /// `leave` must not wrap below zero. - #[test] /// `set` mirrors the scheduler's live slot count: the peak follows the /// highest value ever set, `now` follows the last one, and a set to zero /// (the batch ended) leaves the peak standing. @@ -892,6 +889,9 @@ mod effective_config_tests { ); } + /// The counter's peak is a high-water mark, not the current value, and + /// `leave` must not wrap below zero. + #[test] fn in_flight_counter_tracks_peak_and_saturates() { let counter = InFlightCounter::new(); assert_eq!(counter.enter(), 1); diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs new file mode 100644 index 0000000000..b17a115e0f --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35.rs @@ -0,0 +1,1139 @@ +use crate::error::Result; +use crate::gguf::quantized::{OwnedQuantizedTensor, QuantizedTensorRef}; +use crate::gguf::{GGUFConfig, GGUFModel, OwnedQuantizedModel}; +use std::f32::consts::E; + +/// SiLU activation function +pub fn silu(x: f32) -> f32 { + x / (1.0 + (-x as f32).exp()) +} + +/// Softplus activation function +pub fn softplus(x: f32) -> f32 { + if x > 20.0 { + x + } else { + (1.0 + x.exp()).ln() + } +} + +/// L2 Normalization with epsilon over the given slice. +pub fn l2_norm(x: &mut [f32], eps: f32) { + let mut sq_sum = 0.0; + for &v in x.iter() { + sq_sum += v * v; + } + let scale = 1.0 / (sq_sum + eps).sqrt(); + for v in x.iter_mut() { + *v *= scale; + } +} + +/// Scale each `head_dim`-wide head of `x` to unit L2 norm on its own, as llama.cpp's +/// `build_gdn_l2_norm` does for Gated `DeltaNet` q and k (`rms_norm(x, eps / n) / sqrt(n)` +/// over one head is `x_h / sqrt(sum(x_h^2) + eps)`). One norm over all heads scales every +/// head by the other heads' magnitudes. +pub fn l2_norm_per_head(x: &mut [f32], head_dim: usize, eps: f32) { + for head in x.chunks_exact_mut(head_dim) { + l2_norm(head, eps); + } +} + +/// Multiply `x` elementwise by `sigmoid(gate)`: the output gate of Qwen3.5's full-attention +/// layers (llama.cpp's `attn_gated` node), applied before the output projection. +pub fn apply_sigmoid_gate(x: &mut [f32], gate: &[f32]) { + for (o, g) in x.iter_mut().zip(gate) { + *o *= 1.0 / (1.0 + (-*g).exp()); + } +} + +/// Gated RMSNorm: (RMSNorm(input) * silu(gate)) +/// Norm is computed independently over chunks of size `head_v_dim`. +pub fn gated_rmsnorm( + input: &[f32], + gate: &[f32], + weight: &[f32], + eps: f32, + head_v_dim: usize, + output: &mut [f32], +) { + assert_eq!(input.len(), gate.len()); + assert_eq!(input.len(), output.len()); + assert_eq!(weight.len(), head_v_dim); + assert_eq!(input.len() % head_v_dim, 0); + + for (chunk_in, (chunk_gate, chunk_out)) in input.chunks_exact(head_v_dim).zip( + gate.chunks_exact(head_v_dim) + .zip(output.chunks_exact_mut(head_v_dim)), + ) { + let mut sq_sum = 0.0; + for &v in chunk_in { + sq_sum += v * v; + } + let rms_scale = 1.0 / ((sq_sum / head_v_dim as f32) + eps).sqrt(); + + for i in 0..head_v_dim { + let norm_v = chunk_in[i] * rms_scale * weight[i]; + chunk_out[i] = norm_v * silu(chunk_gate[i]); + } + } +} + +/// Causal Conv1d over seq for a single time step. +/// state is `(kernel_size - 1) * channels` floats. +/// For each channel, we shift the past inputs left and insert the new input. +pub fn causal_conv1d( + input: &[f32], + state: &mut [f32], + weight: &[f32], + kernel_size: usize, + channels: usize, + output: &mut [f32], +) { + assert_eq!(input.len(), channels); + assert_eq!(state.len(), (kernel_size - 1) * channels); + assert_eq!(weight.len(), kernel_size * channels); + assert_eq!(output.len(), channels); + + for c in 0..channels { + let mut sum = 0.0; + let s_offset = c * (kernel_size - 1); + let w_offset = c * kernel_size; + + for k in 0..(kernel_size - 1) { + sum += state[s_offset + k] * weight[w_offset + k]; + } + sum += input[c] * weight[w_offset + kernel_size - 1]; + + // shift left (oldest is at 0, newest is at kernel_size - 2) + for k in 0..(kernel_size - 2) { + state[s_offset + k] = state[s_offset + k + 1]; + } + if kernel_size > 1 { + state[s_offset + kernel_size - 2] = input[c]; + } + + output[c] = sum; + } +} + +/// The gated delta-rule recurrence for a single token exactly as delta-net-base.cpp computes it. +pub fn delta_rule_recurrence( + q: &[f32], + k: &[f32], + v: &[f32], + beta: &[f32], + gate: &[f32], + state: &mut [f32], + output: &mut [f32], + num_v_heads: usize, + head_v_dim: usize, +) { + assert_eq!(q.len(), num_v_heads * head_v_dim); + assert_eq!(k.len(), num_v_heads * head_v_dim); + assert_eq!(v.len(), num_v_heads * head_v_dim); + assert_eq!(beta.len(), num_v_heads); + assert_eq!(gate.len(), num_v_heads); + assert_eq!(state.len(), num_v_heads * head_v_dim * head_v_dim); + assert_eq!(output.len(), num_v_heads * head_v_dim); + + let scale = 1.0 / (head_v_dim as f32).sqrt(); + + for h in 0..num_v_heads { + let q_h = &q[h * head_v_dim..(h + 1) * head_v_dim]; + let k_h = &k[h * head_v_dim..(h + 1) * head_v_dim]; + let v_h = &v[h * head_v_dim..(h + 1) * head_v_dim]; + let beta_val = beta[h]; + let gate_val = gate[h]; + + let state_offset = h * head_v_dim * head_v_dim; + let s_h = &mut state[state_offset..state_offset + head_v_dim * head_v_dim]; + + // 1. S_h *= exp(gate_val) + let exp_gate = gate_val.exp(); + for s in s_h.iter_mut() { + *s *= exp_gate; + } + + // 2. delta = (v_h - S_h^T * k_h) * beta_val + // Note: s_h[j * head_v_dim + i] is S[i][j]. + // So row j of s_h in memory is column j of S. + // sum = dot(row j of s_h, k_h) + let mut delta = vec![0.0; head_v_dim]; + for j in 0..head_v_dim { + let row_j = &s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let mut sum = 0.0; + for i in 0..head_v_dim { + sum += row_j[i] * k_h[i]; + } + delta[j] = (v_h[j] - sum) * beta_val; + } + + // 3. S_h += k_h * delta^T + for j in 0..head_v_dim { + let row_j = &mut s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let d_j = delta[j]; + for i in 0..head_v_dim { + row_j[i] += k_h[i] * d_j; + } + } + + // 4. out_h = S_h^T * q_h * scale + for j in 0..head_v_dim { + let row_j = &s_h[j * head_v_dim..(j + 1) * head_v_dim]; + let mut sum = 0.0; + for i in 0..head_v_dim { + sum += row_j[i] * q_h[i]; + } + output[h * head_v_dim + j] = sum * scale; + } + } +} + +/// Partial NEOX RoPE, exactly as llama.cpp ggml_rope_multi for text input (every position stream is the +/// token position, so the sections do not change theta): for pair j in 0..n_rot/2 of each head, +/// (x[j], x[j + n_rot/2]) rotates by theta_j = pos * theta_scale^j with theta_scale = freq_base^(-2/n_rot), +/// computed iteratively as ggml does; dims n_rot..head_dim pass through unrotated. +pub fn apply_partial_neox_rope( + x: &mut [f32], + num_heads: usize, + head_dim: usize, + n_rot: usize, + pos: usize, + freq_base: f32, +) { + let half = n_rot / 2; + let theta_scale = freq_base.powf(-2.0 / n_rot as f32); + for h in 0..num_heads { + let base = h * head_dim; + let mut theta = pos as f32; + for j in 0..half { + let (sin, cos) = theta.sin_cos(); + let (a, b) = (x[base + j], x[base + j + half]); + x[base + j] = a * cos - b * sin; + x[base + j + half] = a * sin + b * cos; + theta *= theta_scale; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_silu() { + assert!((silu(0.0) - 0.0).abs() < 1e-6); + assert!((silu(1.0) - (1.0 / (1.0 + (-1.0_f32).exp()))).abs() < 1e-6); + } + + #[test] + fn test_l2_norm() { + let mut x = [1.0, 2.0, 2.0]; + l2_norm(&mut x, 0.0); + assert!((x[0] - 1.0 / 3.0).abs() < 1e-6); + assert!((x[1] - 2.0 / 3.0).abs() < 1e-6); + assert!((x[2] - 2.0 / 3.0).abs() < 1e-6); + } + + #[test] + fn test_causal_conv1d() { + let input = [1.0, 2.0]; + let mut state = [0.1, 0.2, 0.3, 0.4]; // 2 past states per channel. Channels = 2. + let weight = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // kernel_size = 3. format: [w_0, w_1, w_2] per channel. Wait, weight memory is [kernel_size * channels]. + // Is it channel outer or time outer? + // causal_conv1d uses `w_offset = c * kernel_size`, so channel outer. + // weight for c=0: [1.0, 2.0, 3.0]. weight for c=1: [4.0, 5.0, 6.0]. + // state for c=0: [0.1, 0.2]. state for c=1: [0.3, 0.4]. + + let mut output = [0.0, 0.0]; + causal_conv1d(&input, &mut state, &weight, 3, 2, &mut output); + + // c = 0: sum = 0.1 * 1.0 + 0.2 * 2.0 + 1.0 * 3.0 = 0.1 + 0.4 + 3.0 = 3.5 + assert!((output[0] - 3.5).abs() < 1e-5); + // c = 1: sum = 0.3 * 4.0 + 0.4 * 5.0 + 2.0 * 6.0 = 1.2 + 2.0 + 12.0 = 15.2 + assert!((output[1] - 15.2).abs() < 1e-5); + + // State should shift: [0.2, 1.0, 0.4, 2.0] + assert!((state[0] - 0.2).abs() < 1e-5); + assert!((state[1] - 1.0).abs() < 1e-5); + assert!((state[2] - 0.4).abs() < 1e-5); + assert!((state[3] - 2.0).abs() < 1e-5); + } + + #[test] + fn test_delta_rule_recurrence() { + let num_v_heads = 1; + let head_v_dim = 2; + let q = [1.0, 2.0]; + let k = [0.5, 0.5]; + let v = [1.0, -1.0]; + let beta = [0.5]; + let gate = [0.0]; // exp(0) = 1.0 + + // State S is 2x2. + // s_h[j * 2 + i] is S[i][j]. + // Memory: [S[0][0], S[1][0], S[0][1], S[1][1]] + // Let S be identity. + let mut state = [ + 1.0, 0.0, // column 0 + 0.0, 1.0, // column 1 + ]; + let mut output = [0.0, 0.0]; + + delta_rule_recurrence( + &q, + &k, + &v, + &beta, + &gate, + &mut state, + &mut output, + num_v_heads, + head_v_dim, + ); + + // 1. S_h *= 1.0 -> same + // 2. delta[j] = (v[j] - S^T * k) * 0.5 + // S^T * k = [1 0; 0 1] * [0.5, 0.5] = [0.5, 0.5] + // v - S^T k = [0.5, -1.5] + // delta = [0.25, -0.75] + // 3. S += k * delta^T + // k * delta^T = [0.5; 0.5] * [0.25, -0.75] = [0.125, -0.375; 0.125, -0.375] + // New S = [1.125, -0.375; 0.125, 0.625] + // 4. out = S^T * q * scale (scale = 1/sqrt(2) = 0.7071) + // S^T = [1.125, 0.125; -0.375, 0.625] + // S^T * [1.0, 2.0] = [1.125 + 0.25, -0.375 + 1.25] = [1.375, 0.875] + // out = [1.375 / sqrt(2), 0.875 / sqrt(2)] + + let s2 = 2.0_f32.sqrt(); + assert!((output[0] - 1.375 / s2).abs() < 1e-5); + assert!((output[1] - 0.875 / s2).abs() < 1e-5); + + // Check state + assert!((state[0] - 1.125).abs() < 1e-5); // S[0][0] + assert!((state[1] - 0.125).abs() < 1e-5); // S[1][0] + assert!((state[2] - -0.375).abs() < 1e-5); // S[0][1] + assert!((state[3] - 0.625).abs() < 1e-5); // S[1][1] + } +} + +/// Per-sequence decode state for [`Qwen35Model`]: the attention layers' KV cache plus each +/// Gated `DeltaNet` layer's causal-conv window and recurrent state. Build one with +/// [`Qwen35Model::new_state`]. +pub struct Qwen35State { + pub(crate) conv_states: Vec>, + pub(crate) ssm_states: Vec>, + pub(crate) kv_cache: crate::gguf::OwnedQuantizedKVCache, +} + +impl Qwen35State { + pub(crate) fn new( + num_layers: usize, + max_seq_len: usize, + head_dim: usize, + num_kv_heads: usize, + num_k_heads: usize, + head_k_dim: usize, + num_v_heads: usize, + head_value_dim: usize, + ) -> Self { + let convalue_dim = head_k_dim * num_k_heads * 2 + head_value_dim * num_v_heads; + Self { + conv_states: vec![vec![0.0; convalue_dim * 3]; num_layers], + ssm_states: vec![vec![0.0; num_v_heads * head_value_dim * head_value_dim]; num_layers], + kv_cache: crate::gguf::OwnedQuantizedKVCache::new( + num_layers, + num_kv_heads * head_dim, + max_seq_len, + ), + } + } +} + +pub(crate) struct Qwen35OwnedDeltaNetLayer { + pub(crate) attn_norm: Vec, + pub(crate) attn_qkv: OwnedQuantizedTensor, + pub(crate) attn_gate: OwnedQuantizedTensor, + pub(crate) ssm_alpha: OwnedQuantizedTensor, + pub(crate) ssm_beta: OwnedQuantizedTensor, + pub(crate) ssm_a: Vec, + pub(crate) ssm_dt_bias: Vec, + pub ssm_conv1d_weight: Vec, + pub(crate) ssm_norm_weight: Vec, + pub(crate) ssm_out: OwnedQuantizedTensor, + pub(crate) post_attention_norm: Vec, + pub(crate) ffn_gate: OwnedQuantizedTensor, + pub(crate) ffn_up: OwnedQuantizedTensor, + pub(crate) ffn_down: OwnedQuantizedTensor, +} + +pub(crate) struct Qwen35OwnedAttentionLayer { + pub(crate) attn_norm: Vec, + pub(crate) attn_q: OwnedQuantizedTensor, + pub(crate) attn_k: OwnedQuantizedTensor, + pub(crate) attn_v: OwnedQuantizedTensor, + pub(crate) attn_q_norm: Vec, + pub(crate) attn_k_norm: Vec, + pub(crate) attn_output: OwnedQuantizedTensor, + pub(crate) post_attention_norm: Vec, + pub(crate) ffn_gate: OwnedQuantizedTensor, + pub(crate) ffn_up: OwnedQuantizedTensor, + pub(crate) ffn_down: OwnedQuantizedTensor, +} + +pub(crate) enum Qwen35OwnedLayer { + DeltaNet(Qwen35OwnedDeltaNetLayer), + Attention(Qwen35OwnedAttentionLayer), +} + +/// Qwen3.5 / Qwen3.8 hybrid decoder on the CPU (#3091): Gated `DeltaNet` layers (short causal +/// conv, per-head recurrent state, gated delta rule) interleaved with gated full-attention +/// layers, on top of the dense model's embeddings, final norm and `lm_head`. +pub struct Qwen35Model<'a> { + pub(crate) base: &'a OwnedQuantizedModel, + pub(crate) layers: Vec, + pub(crate) head_dim: usize, + pub(crate) num_kv_heads: usize, + pub(crate) num_v_heads: usize, + pub(crate) head_v_dim: usize, + pub(crate) num_k_heads: usize, + pub(crate) head_k_dim: usize, + pub(crate) conv_kernel: usize, + pub(crate) rope_sections: [usize; 4], +} + +fn load_f32_vec(tensor_ref: &QuantizedTensorRef, data: &[u8]) -> Result> { + if tensor_ref.qtype != crate::gguf::types::GGUF_TYPE_F32 { + return Err(crate::error::RealizarError::FormatError { + reason: format!( + "qwen35: expected an F32 tensor, found GGUF type {}", + tensor_ref.qtype + ), + }); + } + let bytes = data + .get(tensor_ref.offset..tensor_ref.offset + tensor_ref.byte_size) + .ok_or_else(|| crate::error::RealizarError::FormatError { + reason: format!( + "qwen35: F32 tensor at byte {} (+{}) lies outside the file", + tensor_ref.offset, tensor_ref.byte_size + ), + })?; + // Decoded, not reinterpreted: an mmap offset carries no f32 alignment guarantee. + Ok(bytes + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect()) +} + +impl<'a> Qwen35Model<'a> { + /// Build the shared base (config, token embeddings, final norm, `lm_head`) without the + /// dense layer loader, which refuses hybrid files. + /// + /// # Errors + /// Missing or malformed base tensors or metadata. + pub fn create_base_model( + model: &crate::gguf::GGUFModel, + data: &[u8], + ) -> crate::error::Result { + let config = crate::gguf::config::ValidatedModelConfig::from_gguf(model)?.into_inner(); + + let token_embedding = model.get_tensor_f32("token_embd.weight", data)?; + let output_norm_weight = model.get_tensor_f32("output_norm.weight", data)?; + + let lm_head_ref = + crate::gguf::QuantizedGGUFTransformer::get_tensor_ref(model, data, "output.weight") + .or_else(|_| { + crate::gguf::QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + "token_embd.weight", + ) + })?; + let lm_head_weight = crate::gguf::OwnedQuantizedTensor::from_ref_with_dims( + &lm_head_ref, + data, + config.hidden_dim, + config.vocab_size, + ); + + Ok(crate::gguf::OwnedQuantizedModel { + config, + token_embedding, + position_embedding: None, + layers: vec![], + encoder_layers: vec![], + encoder_output_norm_weight: None, + encoder_output_norm_bias: None, + output_norm_weight, + output_norm_bias: None, + lm_head_weight, + lm_head_bias: None, + // A CUDA build carries these on every OwnedQuantizedModel (as in loading.rs); the + // Qwen3.5 base never uses them (the GPU path has no Gated DeltaNet yet, #3090). + #[cfg(feature = "cuda")] + cuda_executor: None, + #[cfg(feature = "cuda")] + cuda_kernel_count: std::sync::atomic::AtomicU64::new(0), + #[cfg(feature = "cuda")] + cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()), + }) + } + + /// Load every hybrid layer of `model` on top of `base`. + /// + /// # Errors + /// A layer tensor that is missing, has an unexpected type, or lies outside the file. + pub fn from_model_and_layers( + base: &'a OwnedQuantizedModel, + model: &GGUFModel, + data: &[u8], + ) -> Result { + let refs = crate::gguf::qwen35_load::load_qwen35_layers(model, data)?; + + let get_u32 = |key: &str| -> Result { + match model.metadata.get(key) { + Some(crate::gguf::types::GGUFValue::UInt32(v)) => Ok(*v as usize), + Some(crate::gguf::types::GGUFValue::Int32(v)) => Ok(*v as usize), + _ => Err(crate::RealizarError::InvalidShape { + reason: format!("Missing or invalid {}", key), + }), + } + }; + + let head_k_dim = get_u32("qwen2.ssm.state_size") + .or_else(|_| get_u32("qwen35.ssm.state_size")) + .unwrap_or(16); + let head_v_dim = head_k_dim; + let num_k_heads = get_u32("qwen2.ssm.group_count") + .or_else(|_| get_u32("qwen35.ssm.group_count")) + .unwrap_or(1); + let num_v_heads = get_u32("qwen2.ssm.time_step_rank") + .or_else(|_| get_u32("qwen35.ssm.time_step_rank")) + .unwrap_or(16); + let conv_kernel = get_u32("qwen2.ssm.conv_kernel") + .or_else(|_| get_u32("qwen35.ssm.conv_kernel")) + .unwrap_or(4); + let mut rope_sections = [16, 24, 24, 0]; + if let Some(crate::gguf::types::GGUFValue::Array(arr)) = model + .metadata + .get("qwen2.rope.dimension_sections") + .or_else(|| model.metadata.get("qwen35.rope.dimension_sections")) + { + for (i, val) in arr.iter().take(4).enumerate() { + if let crate::gguf::types::GGUFValue::UInt32(v) = val { + rope_sections[i] = *v as usize; + } else if let crate::gguf::types::GGUFValue::Int32(v) = val { + rope_sections[i] = *v as usize; + } + } + } + + let hidden_dim = base.config.hidden_dim; + let intermediate_dim = base.config.intermediate_dim; + let num_heads = base.config.num_heads; + let num_kv_heads = base.config.num_kv_heads; + let head_dim = hidden_dim / num_heads; + + let key_dim = head_k_dim * num_k_heads; + let value_dim = head_v_dim * num_v_heads; + let conv_dim = key_dim * 2 + value_dim; + + let mut owned = Vec::with_capacity(refs.len()); + for layer_ref in refs { + match layer_ref { + crate::gguf::qwen35_load::Qwen35Layer::DeltaNet(d) => { + owned.push(Qwen35OwnedLayer::DeltaNet(Qwen35OwnedDeltaNetLayer { + attn_norm: load_f32_vec(&d.attn_norm, data)?, + attn_qkv: OwnedQuantizedTensor::from_ref_with_dims( + &d.attn_qkv, + data, + hidden_dim, + conv_dim, + ), + attn_gate: OwnedQuantizedTensor::from_ref_with_dims( + &d.attn_gate, + data, + hidden_dim, + value_dim, + ), + ssm_alpha: OwnedQuantizedTensor::from_ref_with_dims( + &d.ssm_alpha, + data, + hidden_dim, + num_v_heads, + ), + ssm_beta: OwnedQuantizedTensor::from_ref_with_dims( + &d.ssm_beta, + data, + hidden_dim, + num_v_heads, + ), + ssm_a: load_f32_vec(&d.ssm_a, data)?, + ssm_dt_bias: load_f32_vec(&d.ssm_dt_bias, data)?, + ssm_conv1d_weight: load_f32_vec(&d.ssm_conv1d_weight, data)?, + ssm_norm_weight: load_f32_vec(&d.ssm_norm_weight, data)?, + ssm_out: OwnedQuantizedTensor::from_ref_with_dims( + &d.ssm_out, data, value_dim, hidden_dim, + ), + post_attention_norm: load_f32_vec(&d.post_attention_norm, data)?, + ffn_gate: OwnedQuantizedTensor::from_ref_with_dims( + &d.ffn_gate, + data, + hidden_dim, + intermediate_dim, + ), + ffn_up: OwnedQuantizedTensor::from_ref_with_dims( + &d.ffn_up, + data, + hidden_dim, + intermediate_dim, + ), + ffn_down: OwnedQuantizedTensor::from_ref_with_dims( + &d.ffn_down, + data, + intermediate_dim, + hidden_dim, + ), + })); + }, + crate::gguf::qwen35_load::Qwen35Layer::Attention(a) => { + let attn_q_norm = load_f32_vec(&a.attn_q_norm, data)?; + let true_head_dim = attn_q_norm.len(); // 256 for Qwen3.5 standard attention + + owned.push(Qwen35OwnedLayer::Attention(Qwen35OwnedAttentionLayer { + attn_norm: load_f32_vec(&a.attn_norm, data)?, + attn_q: OwnedQuantizedTensor::from_ref_with_dims( + &a.attn_q, + data, + hidden_dim, + num_heads * true_head_dim * 2, + ), + attn_k: OwnedQuantizedTensor::from_ref_with_dims( + &a.attn_k, + data, + hidden_dim, + num_kv_heads * true_head_dim, + ), + attn_v: OwnedQuantizedTensor::from_ref_with_dims( + &a.attn_v, + data, + hidden_dim, + num_kv_heads * true_head_dim, + ), + attn_q_norm, + attn_k_norm: load_f32_vec(&a.attn_k_norm, data)?, + attn_output: OwnedQuantizedTensor::from_ref_with_dims( + &a.attn_output, + data, + num_heads * true_head_dim, + hidden_dim, + ), + post_attention_norm: load_f32_vec(&a.post_attention_norm, data)?, + ffn_gate: OwnedQuantizedTensor::from_ref_with_dims( + &a.ffn_gate, + data, + hidden_dim, + intermediate_dim, + ), + ffn_up: OwnedQuantizedTensor::from_ref_with_dims( + &a.ffn_up, + data, + hidden_dim, + intermediate_dim, + ), + ffn_down: OwnedQuantizedTensor::from_ref_with_dims( + &a.ffn_down, + data, + intermediate_dim, + hidden_dim, + ), + })); + }, + } + } + + // Attention head width is key_length (= attn_q_norm width, 256), NOT hidden/heads (128): + // the KV cache and the attention loop used 128-wide heads over 256-wide q/k/v. + let attn_head_dim = owned + .iter() + .find_map(|l| match l { + Qwen35OwnedLayer::Attention(a) => Some(a.attn_q_norm.len()), + Qwen35OwnedLayer::DeltaNet(_) => None, + }) + .unwrap_or(head_dim); + Ok(Self { + base, + layers: owned, + head_dim: attn_head_dim, + num_kv_heads, + num_v_heads, + head_v_dim, + num_k_heads, + head_k_dim, + conv_kernel, + rope_sections, + }) + } + + /// A fresh decode state with room for `max_seq_len` positions. + #[must_use] + pub fn new_state(&self, max_seq_len: usize) -> Qwen35State { + Qwen35State::new( + self.layers.len(), + max_seq_len, + self.head_dim, + self.num_kv_heads, + self.num_k_heads, + self.head_k_dim, + self.num_v_heads, + self.head_v_dim, + ) + } + + /// Run one token at `position` through every layer, updating `cache`, and return the + /// logits. + /// + /// # Errors + /// A matmul or shape failure in any layer. + pub fn forward_single_qwen35( + &self, + token_id: u32, + cache: &mut Qwen35State, + position: usize, + ) -> Result> { + let mut hidden = self.base.token_embedding()[(token_id as usize) + * self.base.config.hidden_dim + ..(token_id as usize + 1) * self.base.config.hidden_dim] + .to_vec(); + + let mut normed = vec![0.0; self.base.config.hidden_dim]; + let mut post_attn_normed = vec![0.0; self.base.config.hidden_dim]; + + let fn_softplus = |x: f32| -> f32 { + if x > 20.0 { + x + } else { + (1.0 + x.exp()).ln() + } + }; + + for (il, layer) in self.layers.iter().enumerate() { + match layer { + Qwen35OwnedLayer::DeltaNet(d) => { + self.forward_deltanet( + d, + &mut hidden, + cache, + il, + position, + &mut normed, + &mut post_attn_normed, + )?; + }, + Qwen35OwnedLayer::Attention(a) => { + self.forward_attention( + a, + &mut hidden, + cache, + il, + position, + &mut normed, + &mut post_attn_normed, + )?; + }, + } + } + let mut out_normed = vec![0.0; self.base.config.hidden_dim]; + crate::gguf::ops::rms_norm_into( + &hidden, + self.base.output_norm_weight(), + self.base.config.eps, + &mut out_normed, + ); + + let mut logits = vec![0.0; self.base.config.vocab_size]; + self.base + .fused_matmul_into(&out_normed, self.base.lm_head_weight(), &mut logits)?; + + cache.kv_cache.advance(); + + Ok(logits) + } + + fn forward_deltanet( + &self, + d: &Qwen35OwnedDeltaNetLayer, + hidden: &mut [f32], + cache: &mut Qwen35State, + il: usize, + position: usize, + normed: &mut [f32], + post_attn_normed: &mut [f32], + ) -> Result<()> { + let fn_softplus = |x: f32| -> f32 { + if x > 20.0 { + x + } else { + (1.0 + x.exp()).ln() + } + }; + + crate::gguf::ops::rms_norm_into(hidden, &d.attn_norm, self.base.config.eps, normed); + let conv_dim = self.head_k_dim * self.num_k_heads * 2 + self.head_v_dim * self.num_v_heads; + let mut conv_in = vec![0.0; conv_dim]; + self.base + .fused_matmul_into(normed, &d.attn_qkv, &mut conv_in)?; + let mut conv_out = vec![0.0; conv_dim]; + causal_conv1d( + &conv_in, + &mut cache.conv_states[il][..], + &d.ssm_conv1d_weight, + self.conv_kernel, + conv_dim, + &mut conv_out, + ); + for x in conv_out.iter_mut() { + *x = *x / (1.0 + (-*x).exp()); // silu/sigmoid + } + // SiLU ONCE, as llama.cpp (ggml_silu on conv_output_raw). A second identical loop applied silu(silu(x)). + + let k_dim = self.head_k_dim * self.num_k_heads; + let v_dim = self.head_v_dim * self.num_v_heads; + let mut q = conv_out[0..k_dim].to_vec(); + let mut k = conv_out[k_dim..k_dim * 2].to_vec(); + let mut v = conv_out[k_dim * 2..conv_dim].to_vec(); + + // Per-head L2 normalisation of q and k, as llama.cpp build_gdn_l2_norm: + // rms_norm(x, eps/n) * 1/sqrt(n) over ne[0] = head_k_dim, i.e. x_h / sqrt(sum(x_h^2) + eps) + // for EACH head. The earlier global norm over all heads (applied twice) scaled every head wrong. + l2_norm_per_head(&mut q, self.head_k_dim, self.base.config.eps); + l2_norm_per_head(&mut k, self.head_k_dim, self.base.config.eps); + + // Gated DeltaNet applies NO RoPE: llama.cpp build_layer_attn_linear has none (position comes from the causal conv). + + let mut dt_raw = vec![0.0; self.num_v_heads]; + self.base + .fused_matmul_into(normed, &d.ssm_alpha, &mut dt_raw)?; + let mut dt = vec![0.0; self.num_v_heads]; + for (i, val) in dt_raw.iter().enumerate() { + dt[i] = fn_softplus(val + d.ssm_dt_bias[i]) * d.ssm_a[i]; + } + + let mut beta = vec![0.0; self.num_v_heads]; + self.base + .fused_matmul_into(normed, &d.ssm_beta, &mut beta)?; + // sigmoid ONCE, as llama.cpp: beta = sigmoid(ssm_beta . x). (It was applied twice.) + for x in beta.iter_mut() { + *x = 1.0 / (1.0 + (-*x).exp()); + } + + let mut gate = vec![0.0; v_dim]; + self.base + .fused_matmul_into(normed, &d.attn_gate, &mut gate)?; + + let mut out_h = vec![0.0; v_dim]; + delta_rule_recurrence( + &q, + &k, + &v, + &beta, + &dt, + &mut cache.ssm_states[il][..], + &mut out_h, + self.num_v_heads, + self.head_v_dim, + ); + + let mut ssm_out_in = vec![0.0; v_dim]; + gated_rmsnorm( + &out_h, + &gate, + &d.ssm_norm_weight, + self.base.config.eps, + self.head_v_dim, + &mut ssm_out_in, + ); + // No normalisation after the gated RMS norm: llama.cpp goes from build_norm_gated straight to ssm_out. + // (Two extra global L2 norms here forced every DeltaNet output to unit length.) + + let mut ssm_out = vec![0.0; self.base.config.hidden_dim]; + self.base + .fused_matmul_into(&ssm_out_in, &d.ssm_out, &mut ssm_out)?; + + for i in 0..self.base.config.hidden_dim { + hidden[i] += ssm_out[i]; + } + + crate::gguf::ops::rms_norm_into( + hidden, + &d.post_attention_norm, + self.base.config.eps, + post_attn_normed, + ); + let mut ffn_gate = vec![0.0; d.ffn_gate.out_dim]; + self.base + .fused_matmul_into(post_attn_normed, &d.ffn_gate, &mut ffn_gate)?; + let mut ffn_up = vec![0.0; d.ffn_up.out_dim]; + self.base + .fused_matmul_into(post_attn_normed, &d.ffn_up, &mut ffn_up)?; + + for i in 0..ffn_gate.len() { + let x = ffn_gate[i]; + let silu = x / (1.0 + (-x as f32).exp()); + ffn_up[i] *= silu; + } + let mut ffn_down = vec![0.0; d.ffn_down.out_dim]; + self.base + .fused_matmul_into(&ffn_up, &d.ffn_down, &mut ffn_down)?; + for i in 0..self.base.config.hidden_dim { + hidden[i] += ffn_down[i]; + } + Ok(()) + } + + fn forward_attention( + &self, + a: &Qwen35OwnedAttentionLayer, + hidden: &mut [f32], + cache: &mut Qwen35State, + il: usize, + position: usize, + normed: &mut [f32], + post_attn_normed: &mut [f32], + ) -> Result<()> { + crate::gguf::ops::rms_norm_into(hidden, &a.attn_norm, self.base.config.eps, normed); + + let mut q_full = vec![0.0; a.attn_q.out_dim]; + let mut k = vec![0.0; a.attn_k.out_dim]; + let mut v = vec![0.0; a.attn_v.out_dim]; + self.base + .fused_matmul_into(normed, &a.attn_q, &mut q_full)?; + self.base.fused_matmul_into(normed, &a.attn_k, &mut k)?; + self.base.fused_matmul_into(normed, &a.attn_v, &mut v)?; + + let num_heads = self.base.config.num_heads; + let head_dim = a.attn_q_norm.len(); // 256 + let mut q = vec![0.0; num_heads * head_dim]; + let mut gate = vec![0.0; num_heads * head_dim]; + + for h in 0..num_heads { + let offset_q_full = h * head_dim * 2; + let offset_q = h * head_dim; + + // Split into Q and gate + q[offset_q..offset_q + head_dim] + .copy_from_slice(&q_full[offset_q_full..offset_q_full + head_dim]); + gate[offset_q..offset_q + head_dim] + .copy_from_slice(&q_full[offset_q_full + head_dim..offset_q_full + head_dim * 2]); + } + + crate::gguf::ops::apply_per_head_rms_norm( + &mut q, + &a.attn_q_norm, + num_heads, + self.base.config.eps, + ); + crate::gguf::ops::apply_per_head_rms_norm( + &mut k, + &a.attn_k_norm, + self.base.config.num_kv_heads, + self.base.config.eps, + ); + + // Partial NEOX RoPE over n_rot = 2 * sum(rope.dimension_sections) = 64 of each 256-wide head, as + // llama.cpp ggml_rope_multi for text input. The base apply_rope assumed head_dim = hidden/heads = 128 + // and a full rotation, both wrong for Qwen3.5 attention. + let n_rot = 2 * self.rope_sections.iter().sum::(); + let freq_base = self.base.config.rope_theta; + apply_partial_neox_rope(&mut q, num_heads, head_dim, n_rot, position, freq_base); + apply_partial_neox_rope( + &mut k, + self.base.config.num_kv_heads, + head_dim, + n_rot, + position, + freq_base, + ); + + cache.kv_cache.append(il, &k, &v); + let k_cache = cache.kv_cache.get_k(il); + let v_cache = cache.kv_cache.get_v(il); + + let mut attn_out_in = vec![0.0; q.len()]; + let num_kv_heads = self.base.config.num_kv_heads; + let num_heads = self.base.config.num_heads; + let group_size = num_heads / num_kv_heads; + let head_dim = self.head_dim; + + for h in 0..num_heads { + let kv_h = h / group_size; + let q_h = &q[h * head_dim..(h + 1) * head_dim]; + + let mut scores = vec![0.0; position + 1]; + for p in 0..=position { + let mut dot = 0.0; + let k_p = &k_cache[p * (num_kv_heads * head_dim) + kv_h * head_dim + ..p * (num_kv_heads * head_dim) + (kv_h + 1) * head_dim]; + for i in 0..head_dim { + dot += q_h[i] * k_p[i]; + } + scores[p] = dot / (head_dim as f32).sqrt(); + } + crate::gguf::ops::softmax(&mut scores); + + let out_h = &mut attn_out_in[h * head_dim..(h + 1) * head_dim]; + for p in 0..=position { + let w = scores[p]; + let v_p = &v_cache[p * (num_kv_heads * head_dim) + kv_h * head_dim + ..p * (num_kv_heads * head_dim) + (kv_h + 1) * head_dim]; + for i in 0..head_dim { + out_h[i] += w * v_p[i]; + } + } + } + // Output gate, as llama.cpp: attn_output * sigmoid(gate) before the output projection + // (the attn_gated node). The gate split off the joint Q projection was never applied. + apply_sigmoid_gate(&mut attn_out_in, &gate); + let mut attn_out = vec![0.0; self.base.config.hidden_dim]; + self.base + .fused_matmul_into(&attn_out_in, &a.attn_output, &mut attn_out)?; + for i in 0..self.base.config.hidden_dim { + hidden[i] += attn_out[i]; + } + + crate::gguf::ops::rms_norm_into( + hidden, + &a.post_attention_norm, + self.base.config.eps, + post_attn_normed, + ); + let mut ffn_gate = vec![0.0; a.ffn_gate.out_dim]; + self.base + .fused_matmul_into(post_attn_normed, &a.ffn_gate, &mut ffn_gate)?; + let mut ffn_up = vec![0.0; a.ffn_up.out_dim]; + self.base + .fused_matmul_into(post_attn_normed, &a.ffn_up, &mut ffn_up)?; + for i in 0..ffn_gate.len() { + let x = ffn_gate[i]; + let silu = x / (1.0 + (-x as f32).exp()); + ffn_up[i] *= silu; + } + let mut ffn_down = vec![0.0; a.ffn_down.out_dim]; + self.base + .fused_matmul_into(&ffn_up, &a.ffn_down, &mut ffn_down)?; + for i in 0..self.base.config.hidden_dim { + hidden[i] += ffn_down[i]; + } + Ok(()) + } +} + +/// Prefill `input_tokens` through the Qwen3.5 CPU forward, then decode up to +/// `gen_config.max_tokens` more with the dense path's token choice (argmax at temperature 0 or +/// `top_k` 1, else seeded top-k/top-p). Returns the prompt followed by the new tokens. `apr run` +/// and `apr chat` dispatch `qwen35` GGUFs here (#3091). +/// +/// # Errors +/// An empty prompt, a layer the Qwen3.5 loader cannot read, or a forward-pass failure. +pub fn run_qwen35_generate( + mapped: &crate::gguf::MappedGGUFModel, + base: &OwnedQuantizedModel, + input_tokens: &[u32], + gen_config: &crate::gguf::QuantizedGenerateConfig, +) -> Result> { + use rand::SeedableRng; + if input_tokens.is_empty() { + return Err(crate::error::RealizarError::InvalidShape { + reason: "run_qwen35_generate: prompt cannot be empty".to_string(), + }); + } + let qwen = Qwen35Model::from_model_and_layers(base, &mapped.model, mapped.data())?; + let max_seq_len = input_tokens.len() + gen_config.max_tokens + 1; + let mut state = qwen.new_state(max_seq_len); + let mut rng = rand::rngs::StdRng::seed_from_u64(gen_config.seed); + + let mut logits = Vec::new(); + for (pos, &token) in input_tokens.iter().enumerate() { + logits = qwen.forward_single_qwen35(token, &mut state, pos)?; + } + let mut tokens = input_tokens.to_vec(); + for _ in 0..gen_config.max_tokens { + let next = if gen_config.temperature == 0.0 || gen_config.top_k == 1 { + crate::gguf::ops::argmax(&logits) + } else { + OwnedQuantizedModel::sample_topk_seeded( + &logits, + gen_config.temperature, + gen_config.top_k, + gen_config.top_p, + &mut rng, + ) + }; + tokens.push(next); + if gen_config.stop_tokens.contains(&next) || tokens.len() >= max_seq_len { + break; + } + logits = qwen.forward_single_qwen35(next, &mut state, tokens.len() - 1)?; + } + Ok(tokens) +} + +#[cfg(test)] +mod qwen35_math_tests { + use super::*; + + #[test] + fn test_l2_norm_per_head_normalises_each_head_on_its_own() { + // Two heads whose magnitudes differ 10x: each must come out as [0.6, 0.8]. + let mut x = [3.0, 4.0, 30.0, 40.0]; + l2_norm_per_head(&mut x, 2, 1e-12); + for (got, want) in x.iter().zip([0.6, 0.8, 0.6, 0.8]) { + assert!((got - want).abs() < 1e-6, "{x:?}"); + } + } + + #[test] + fn test_apply_sigmoid_gate_scales_by_sigmoid() { + let mut x = [2.0, 2.0, 2.0]; + apply_sigmoid_gate(&mut x, &[0.0, 40.0, -40.0]); + assert!((x[0] - 1.0).abs() < 1e-6, "sigmoid(0) = 0.5: {x:?}"); + assert!((x[1] - 2.0).abs() < 1e-6, "sigmoid(40) ~ 1: {x:?}"); + assert!(x[2].abs() < 1e-6, "sigmoid(-40) ~ 0: {x:?}"); + } + + #[test] + fn test_partial_neox_rope_is_identity_at_position_zero() { + let orig: Vec = (0..8u8).map(|i| f32::from(i) + 1.0).collect(); + let mut x = orig.clone(); + apply_partial_neox_rope(&mut x, 1, 8, 4, 0, 10_000.0); + assert_eq!(x, orig); + } + + #[test] + fn test_partial_neox_rope_rotates_half_pairs_and_leaves_the_tail() { + // head_dim 8, n_rot 4: the pairs are (0,2) and (1,3) (NEOX halves, not (0,1),(2,3)), + // and dims 4..8 are not rotated at all. + let base = 10_000.0f32; + let mut x = [0.0, 1.0, 0.0, 0.0, 5.0, 6.0, 7.0, 8.0]; + apply_partial_neox_rope(&mut x, 1, 8, 4, 3, base); + let theta1 = 3.0 * base.powf(-2.0 / 4.0); // pair j = 1 + assert!( + (x[1] - theta1.cos()).abs() < 1e-6 && (x[3] - theta1.sin()).abs() < 1e-6, + "{x:?}" + ); + assert_eq!( + (x[0], x[2]), + (0.0, 0.0), + "the (0,2) pair was all zeros: {x:?}" + ); + assert_eq!(&x[4..], &[5.0, 6.0, 7.0, 8.0]); + } +} + +/// Proof-obligation bodies for `contracts/qwen35-hybrid-forward-v1.yaml` (QHF-INV-001..004, +/// QHF-INV-006, QHF-CON-007). +#[cfg(test)] +#[path = "forward_qwen35_contract_tests.rs"] +mod qhf_contract_tests; diff --git a/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35_contract_tests.rs b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35_contract_tests.rs new file mode 100644 index 0000000000..5a00847acf --- /dev/null +++ b/crates/aprender-serve/src/gguf/inference/forward/forward_qwen35_contract_tests.rs @@ -0,0 +1,1300 @@ +//! Proof-obligation bodies for `contracts/qwen35-hybrid-forward-v1.yaml` (#3091). +//! +//! Every test drives the real Qwen3.5 sublayers (`Qwen35Model::forward_attention`, +//! `Qwen35Model::forward_deltanet`, `Qwen35Model::forward_single_qwen35`) on a tiny seeded +//! F32 model built in memory: no GGUF file. Obligation ids follow PR #3316. +//! +//! Isolating one sublayer: `forward_attention` and `forward_deltanet` each run the +//! mixer sublayer AND the FFN sublayer, with no entry point for either alone. A sublayer is +//! isolated by zeroing the OTHER sublayer's output projection (`ffn_down`, `ssm_out`, +//! `attn_output`), which makes that sublayer add exactly `0.0` to every coordinate. + +use super::{ + Qwen35Model, Qwen35OwnedAttentionLayer, Qwen35OwnedDeltaNetLayer, Qwen35OwnedLayer, Qwen35State, +}; +use crate::error::Result; +use crate::gguf::{GGUFConfig, OwnedQuantizedModel, OwnedQuantizedTensor}; + +/// QHF-CON-007 `tolerance: 1.0e-06`, the only tolerance the contract states. +const CONTRACT_TOL: f32 = 1.0e-6; + +/// Shapes of one synthetic model. Every inner width differs from `hidden`, so a projection +/// that fails to restore `d_model` cannot pass by coincidence. +#[derive(Clone, Copy)] +struct Dims { + hidden: usize, + num_heads: usize, + num_kv_heads: usize, + /// Attention head width (`attn_q_norm.len()`), deliberately != hidden / num_heads. + attn_head_dim: usize, + intermediate: usize, + /// Gated `DeltaNet` head width (k and v share it, as the loader does). + gdn_head_dim: usize, + /// Gated `DeltaNet` head count (k and v, so `q`, `k` and `v` have equal widths). + gdn_heads: usize, + vocab: usize, +} + +const DIMS: [Dims; 2] = [ + Dims { + hidden: 8, + num_heads: 2, + num_kv_heads: 1, + attn_head_dim: 6, + intermediate: 12, + gdn_head_dim: 4, + gdn_heads: 2, + vocab: 5, + }, + Dims { + hidden: 12, + num_heads: 4, + num_kv_heads: 2, + attn_head_dim: 4, + intermediate: 20, + gdn_head_dim: 2, + gdn_heads: 3, + vocab: 7, + }, +]; + +/// `Qwen35State` allocates `3 * conv_dim` of conv window, i.e. kernel 4 is baked in. +const CONV_KERNEL: usize = 4; +const SEQ_LEN: usize = 3; +const INPUT_SEEDS: [u64; 3] = [11, 23, 57]; + +impl Dims { + fn gdn_width(self) -> usize { + self.gdn_head_dim * self.gdn_heads + } + fn conv_dim(self) -> usize { + self.gdn_width() * 3 + } +} + +/// Deterministic xorshift stream of floats in [-0.5, 0.5). +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Self(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1) + } + fn next_f32(&mut self) -> f32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + // 24 high bits -> [0, 1), exactly representable in f32. + ((self.0 >> 40) as f32) / 16_777_216.0 - 0.5 + } + fn vec(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.next_f32()).collect() + } + /// Strictly positive norm weights in [0.5, 1.5). + fn norm_weight(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.next_f32() + 1.0).collect() + } +} + +fn f32_tensor(rng: &mut Rng, in_dim: usize, out_dim: usize) -> OwnedQuantizedTensor { + OwnedQuantizedTensor { + data: rng + .vec(in_dim * out_dim) + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(), + in_dim, + out_dim, + qtype: crate::gguf::types::GGUF_TYPE_F32, + } +} + +fn zero_tensor(in_dim: usize, out_dim: usize) -> OwnedQuantizedTensor { + OwnedQuantizedTensor { + data: vec![0u8; in_dim * out_dim * 4], + in_dim, + out_dim, + qtype: crate::gguf::types::GGUF_TYPE_F32, + } +} + +fn base_model(dims: Dims) -> OwnedQuantizedModel { + let mut rng = Rng::new(1); + let config = GGUFConfig { + architecture: "qwen35".to_string(), + constraints: crate::gguf::ArchConstraints::from_architecture("qwen35"), + hidden_dim: dims.hidden, + num_layers: 0, + num_heads: dims.num_heads, + num_kv_heads: dims.num_kv_heads, + vocab_size: dims.vocab, + intermediate_dim: dims.intermediate, + context_length: 32, + rope_theta: 10_000.0, + // RMSNorm is exactly scale-invariant only as eps -> 0; 1e-12 vanishes against the + // O(0.1) mean squares these tests feed it. + eps: 1.0e-12, + rope_type: 2, + explicit_head_dim: Some(dims.attn_head_dim), + query_pre_attn_scalar: None, + bos_token_id: None, + eos_token_id: None, + }; + OwnedQuantizedModel { + config, + token_embedding: rng.vec(dims.vocab * dims.hidden), + position_embedding: None, + layers: vec![], + encoder_layers: vec![], + encoder_output_norm_weight: None, + encoder_output_norm_bias: None, + output_norm_weight: rng.norm_weight(dims.hidden), + output_norm_bias: None, + lm_head_weight: f32_tensor(&mut rng, dims.hidden, dims.vocab), + lm_head_bias: None, + #[cfg(feature = "cuda")] + cuda_executor: None, + #[cfg(feature = "cuda")] + cuda_kernel_count: std::sync::atomic::AtomicU64::new(0), + #[cfg(feature = "cuda")] + cached_weight_names: std::sync::Mutex::new(std::collections::HashSet::new()), + } +} + +fn attention_layer(dims: Dims, seed: u64) -> Qwen35OwnedAttentionLayer { + let mut r = Rng::new(seed); + let (h, hd) = (dims.hidden, dims.attn_head_dim); + Qwen35OwnedAttentionLayer { + attn_norm: r.norm_weight(h), + attn_q: f32_tensor(&mut r, h, dims.num_heads * hd * 2), + attn_k: f32_tensor(&mut r, h, dims.num_kv_heads * hd), + attn_v: f32_tensor(&mut r, h, dims.num_kv_heads * hd), + attn_q_norm: r.norm_weight(hd), + attn_k_norm: r.norm_weight(hd), + attn_output: f32_tensor(&mut r, dims.num_heads * hd, h), + post_attention_norm: r.norm_weight(h), + ffn_gate: f32_tensor(&mut r, h, dims.intermediate), + ffn_up: f32_tensor(&mut r, h, dims.intermediate), + ffn_down: f32_tensor(&mut r, dims.intermediate, h), + } +} + +fn deltanet_layer(dims: Dims, seed: u64) -> Qwen35OwnedDeltaNetLayer { + let mut r = Rng::new(seed); + let h = dims.hidden; + let heads = dims.gdn_heads; + Qwen35OwnedDeltaNetLayer { + attn_norm: r.norm_weight(h), + attn_qkv: f32_tensor(&mut r, h, dims.conv_dim()), + attn_gate: f32_tensor(&mut r, h, dims.gdn_width()), + ssm_alpha: f32_tensor(&mut r, h, heads), + ssm_beta: f32_tensor(&mut r, h, heads), + // A = -exp(A_log) < 0, so the state decays. + ssm_a: r.vec(heads).iter().map(|v| -(v + 0.6)).collect(), + ssm_dt_bias: r.vec(heads), + ssm_conv1d_weight: r.vec(CONV_KERNEL * dims.conv_dim()), + ssm_norm_weight: r.norm_weight(dims.gdn_head_dim), + ssm_out: f32_tensor(&mut r, dims.gdn_width(), h), + post_attention_norm: r.norm_weight(h), + ffn_gate: f32_tensor(&mut r, h, dims.intermediate), + ffn_up: f32_tensor(&mut r, h, dims.intermediate), + ffn_down: f32_tensor(&mut r, dims.intermediate, h), + } +} + +fn model(base: &OwnedQuantizedModel, dims: Dims, layers: Vec) -> Qwen35Model<'_> { + Qwen35Model { + base, + layers, + head_dim: dims.attn_head_dim, + num_kv_heads: dims.num_kv_heads, + num_v_heads: dims.gdn_heads, + head_v_dim: dims.gdn_head_dim, + num_k_heads: dims.gdn_heads, + head_k_dim: dims.gdn_head_dim, + conv_kernel: CONV_KERNEL, + // n_rot = 2 * 2 = 4 <= every attn_head_dim above. + rope_sections: [1, 1, 0, 0], + } +} + +/// One whole block (mixer sublayer then FFN sublayer) of layer `il` at `position`. +fn run_block( + m: &Qwen35Model<'_>, + il: usize, + h: &[f32], + state: &mut Qwen35State, + position: usize, +) -> Result> { + let hd = m.base.config.hidden_dim; + let mut out = h.to_vec(); + let mut normed = vec![0.0; hd]; + let mut post = vec![0.0; hd]; + match &m.layers[il] { + Qwen35OwnedLayer::DeltaNet(d) => { + m.forward_deltanet(d, &mut out, state, il, position, &mut normed, &mut post)?; + }, + Qwen35OwnedLayer::Attention(a) => { + m.forward_attention(a, &mut out, state, il, position, &mut normed, &mut post)?; + }, + } + Ok(out) +} + +/// Run `inputs` (one per position) through the single layer of `m` from a fresh state and +/// return each position's residual delta `h_out - h_in`. +fn deltas_over_sequence(m: &Qwen35Model<'_>, inputs: &[Vec]) -> Result>> { + let mut state = m.new_state(inputs.len() + 1); + let mut deltas = Vec::with_capacity(inputs.len()); + for (pos, h) in inputs.iter().enumerate() { + let out = run_block(m, 0, h, &mut state, pos)?; + assert_eq!(out.len(), h.len(), "block output width at position {pos}"); + deltas.push(out.iter().zip(h).map(|(o, i)| o - i).collect()); + state.kv_cache.advance(); + } + Ok(deltas) +} + +/// Qwen3.5's layer schedule (every 4th layer is full attention) over two periods, each layer +/// seeded `seed + l`. +fn hybrid_layers(dims: Dims, seed: u64) -> Vec { + hybrid_schedule(dims, seed, 8) +} + +/// Qwen3.5's layer schedule over `n` layers: every 4th is full attention, each seeded `seed + l`. +fn hybrid_schedule(dims: Dims, seed: u64, n: u64) -> Vec { + (0..n) + .map(|l| { + if (l + 1) % 4 == 0 { + Qwen35OwnedLayer::Attention(attention_layer(dims, seed + l)) + } else { + Qwen35OwnedLayer::DeltaNet(deltanet_layer(dims, seed + l)) + } + }) + .collect() +} + +fn inputs(dims: Dims, seed: u64) -> Vec> { + let mut r = Rng::new(seed); + (0..SEQ_LEN).map(|_| r.vec(dims.hidden)).collect() +} + +/// The sublayer wrote a finite value into EVERY one of the `d_model` coordinates. A +/// sublayer whose output is narrower than `d_model` leaves a coordinate's delta at exactly +/// zero (or panics on the width mismatch); seeded dense weights make an honest exact zero +/// vanishingly unlikely. +fn assert_fills_d_model(deltas: &[Vec], dims: Dims, what: &str) { + for (pos, d) in deltas.iter().enumerate() { + assert_eq!(d.len(), dims.hidden, "{what}: width at position {pos}"); + for (i, v) in d.iter().enumerate() { + assert!( + v.is_finite(), + "{what}: coordinate {i} at position {pos} is {v}" + ); + assert!( + *v != 0.0, + "{what}: coordinate {i} of d_model={} never received the sublayer output \ + (position {pos}, deltas {d:?})", + dims.hidden + ); + } + } +} + +// --- independent reference math (QHF-CON-007 oracle) -------------------------------------- + +fn weights(t: &OwnedQuantizedTensor) -> Vec { + t.data + .chunks_exact(4) + .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() +} + +fn matvec(t: &OwnedQuantizedTensor, x: &[f32]) -> Vec { + let w = weights(t); + (0..t.out_dim) + .map(|r| (0..t.in_dim).map(|c| w[r * t.in_dim + c] * x[c]).sum()) + .collect() +} + +fn rmsnorm(x: &[f32], w: &[f32], eps: f32) -> Vec { + let inv = 1.0 / (x.iter().map(|v| v * v).sum::() / x.len() as f32 + eps).sqrt(); + x.iter().zip(w).map(|(v, g)| v * inv * g).collect() +} + +fn sigmoid(x: f32) -> f32 { + 1.0 / (1.0 + (-x).exp()) +} + +/// `swiglu(rmsnorm(x))`: down(silu(gate(n)) * up(n)). +fn ref_ffn( + h: &[f32], + norm: &[f32], + gate: &OwnedQuantizedTensor, + up: &OwnedQuantizedTensor, + down: &OwnedQuantizedTensor, + eps: f32, +) -> Vec { + let n = rmsnorm(h, norm, eps); + let g = matvec(gate, &n); + let u = matvec(up, &n); + let act: Vec = g.iter().zip(&u).map(|(g, u)| u * g * sigmoid(*g)).collect(); + matvec(down, &act) +} + +/// Gated attention sublayer at position 0 of a fresh cache: one key, so the softmax weight +/// is 1 and each head's output is its KV group's value, times sigmoid(gate), projected. +fn ref_attention_pos0(dims: Dims, a: &Qwen35OwnedAttentionLayer, h: &[f32], eps: f32) -> Vec { + let hd = dims.attn_head_dim; + let group = dims.num_heads / dims.num_kv_heads; + let n = rmsnorm(h, &a.attn_norm, eps); + let q_full = matvec(&a.attn_q, &n); + let v = matvec(&a.attn_v, &n); + let mut out = vec![0.0; dims.num_heads * hd]; + for head in 0..dims.num_heads { + let kv = head / group; + for i in 0..hd { + out[head * hd + i] = v[kv * hd + i] * sigmoid(q_full[head * hd * 2 + hd + i]); + } + } + matvec(&a.attn_output, &out) +} + +/// Gated `DeltaNet` sublayer at position 0 of a fresh state: the conv window and the +/// recurrent state are zero, so conv = last tap * input and `S' = k (beta v)^T`. +fn ref_deltanet_pos0(dims: Dims, d: &Qwen35OwnedDeltaNetLayer, h: &[f32], eps: f32) -> Vec { + let (hd, w) = (dims.gdn_head_dim, dims.gdn_width()); + let n = rmsnorm(h, &d.attn_norm, eps); + let conv_in = matvec(&d.attn_qkv, &n); + let conv: Vec = conv_in + .iter() + .enumerate() + .map(|(c, x)| { + let y = x * d.ssm_conv1d_weight[c * CONV_KERNEL + CONV_KERNEL - 1]; + y * sigmoid(y) + }) + .collect(); + let l2 = |x: &[f32]| -> Vec { + x.chunks_exact(hd) + .flat_map(|c| { + let inv = 1.0 / (c.iter().map(|v| v * v).sum::() + eps).sqrt(); + c.iter().map(move |v| v * inv) + }) + .collect() + }; + let (q, k, v) = (l2(&conv[..w]), l2(&conv[w..2 * w]), &conv[2 * w..]); + let beta: Vec = matvec(&d.ssm_beta, &n).into_iter().map(sigmoid).collect(); + let gate = matvec(&d.attn_gate, &n); + let scale = 1.0 / (hd as f32).sqrt(); + let mut normed = vec![0.0; w]; + for head in 0..dims.gdn_heads { + let r = head * hd..(head + 1) * hd; + let kq: f32 = k[r.clone()] + .iter() + .zip(&q[r.clone()]) + .map(|(a, b)| a * b) + .sum(); + let out: Vec = v[r.clone()] + .iter() + .map(|vj| vj * beta[head] * kq * scale) + .collect(); + let inv = 1.0 / (out.iter().map(|x| x * x).sum::() / hd as f32 + eps).sqrt(); + for (j, o) in out.iter().enumerate() { + let g = gate[head * hd + j]; + normed[head * hd + j] = o * inv * d.ssm_norm_weight[j] * g * sigmoid(g); + } + } + matvec(&d.ssm_out, &normed) +} + +fn assert_close(got: &[f32], want: &[f32], what: &str) { + assert_eq!(got.len(), want.len(), "{what}: width"); + for (i, (g, w)) in got.iter().zip(want).enumerate() { + assert!( + (g - w).abs() <= CONTRACT_TOL, + "{what}: coordinate {i}: h_(l+1) - h_l = {g}, sublayer(norm(h_l)) = {w} \ + (|diff| {} > {CONTRACT_TOL})\n got {got:?}\nwant {want:?}", + (g - w).abs() + ); + } +} + +// --- QHF-INV-001 / 002 / 003: shape preservation ------------------------------------------ + +/// QHF-INV-001 `∀x: shape(attention_sublayer(x)) = shape(x)` (FALSIFY-QHF-001), over two +/// `d_model`s, three seeded inputs and a three-token sequence (so the KV cache grows). The +/// attention inner width (`num_heads * attn_head_dim`) differs from `d_model` in both +/// configs; the FFN is zeroed so the delta is the attention sublayer alone. +#[test] +fn qhf_inv_001_attention_sublayer_preserves_d_model() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let mut layer = attention_layer(dims, 101); + layer.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let m = model(&base, dims, vec![Qwen35OwnedLayer::Attention(layer)]); + for seed in INPUT_SEEDS { + let deltas = deltas_over_sequence(&m, &inputs(dims, seed))?; + assert_fills_d_model(&deltas, dims, "QHF-INV-001 attention sublayer"); + } + } + Ok(()) +} + +/// QHF-INV-002 `∀x: shape(gdn_sublayer(x)) = shape(x)` (FALSIFY-QHF-002). The Gated +/// `DeltaNet` value width (`gdn_heads * gdn_head_dim`) differs from `d_model` in the second +/// config; the FFN is zeroed so the delta is the GDN sublayer alone. +#[test] +fn qhf_inv_002_gdn_sublayer_preserves_d_model() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let mut layer = deltanet_layer(dims, 202); + layer.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let m = model(&base, dims, vec![Qwen35OwnedLayer::DeltaNet(layer)]); + for seed in INPUT_SEEDS { + let deltas = deltas_over_sequence(&m, &inputs(dims, seed))?; + assert_fills_d_model(&deltas, dims, "QHF-INV-002 GDN sublayer"); + } + } + Ok(()) +} + +/// QHF-INV-003 `∀x: shape(ffn_sublayer(x)) = shape(x)` (FALSIFY-QHF-003), in BOTH layer +/// types (the contract's `ffn_sublayer` is shared). `intermediate != d_model`; the mixer's +/// output projection is zeroed so the delta is the FFN sublayer alone. +#[test] +fn qhf_inv_003_ffn_sublayer_preserves_d_model() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let mut attn = attention_layer(dims, 303); + attn.attn_output = zero_tensor(dims.num_heads * dims.attn_head_dim, dims.hidden); + let mut gdn = deltanet_layer(dims, 304); + gdn.ssm_out = zero_tensor(dims.gdn_width(), dims.hidden); + for (layer, what) in [ + ( + Qwen35OwnedLayer::Attention(attn), + "QHF-INV-003 FFN sublayer (attention layer)", + ), + ( + Qwen35OwnedLayer::DeltaNet(gdn), + "QHF-INV-003 FFN sublayer (GDN layer)", + ), + ] { + let m = model(&base, dims, vec![layer]); + for seed in INPUT_SEEDS { + let deltas = deltas_over_sequence(&m, &inputs(dims, seed))?; + assert_fills_d_model(&deltas, dims, what); + } + } + } + Ok(()) +} + +// --- QHF-INV-004: exactly one mixer per layer --------------------------------------------- + +/// QHF-INV-004 `∀l: is_attention(l) XOR is_gdn(l)` (FALSIFY-QHF-004), observed by EFFECT +/// through the full `forward_single_qwen35`: after a two-token run, a layer that ran +/// attention has written its KV cache, a layer that ran Gated `DeltaNet` has written its +/// conv window / recurrent state, and no layer has written both or neither. The schedule +/// is Qwen3.5's (every 4th layer is attention) over two periods. +#[test] +fn qhf_inv_004_each_layer_runs_exactly_one_mixer() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let m = model(&base, dims, hybrid_layers(dims, 400)); + let mut state = m.new_state(4); + for (pos, token) in [1u32, 3].into_iter().enumerate() { + let logits = m.forward_single_qwen35(token, &mut state, pos)?; + assert_eq!(logits.len(), dims.vocab); + assert!(logits.iter().all(|v| v.is_finite()), "{logits:?}"); + } + for (l, layer) in m.layers.iter().enumerate() { + let ran_attention = !state.kv_cache.get_k(l).is_empty(); + let ran_gdn = state.conv_states[l].iter().any(|v| *v != 0.0) + || state.ssm_states[l].iter().any(|v| *v != 0.0); + assert!( + ran_attention ^ ran_gdn, + "QHF-INV-004 layer {l}: ran attention = {ran_attention}, ran GDN = {ran_gdn}" + ); + let is_attention = matches!(layer, Qwen35OwnedLayer::Attention(_)); + assert_eq!( + ran_attention, is_attention, + "QHF-INV-004 layer {l}: the mixer that ran is not the layer's type" + ); + } + } + Ok(()) +} + +// --- QHF-INV-006: pre-norm order ---------------------------------------------------------- + +/// QHF-INV-006 check for one single-layer model: the residual delta for `c * h` equals the +/// delta for `h` at every position and coordinate, for c = 4 and c = 1/4. +fn assert_delta_scale_invariant(m: &Qwen35Model<'_>, xs: &[Vec], what: &str) -> Result<()> { + let reference = deltas_over_sequence(m, xs)?; + for c in [4.0f32, 0.25] { + let scaled: Vec> = xs + .iter() + .map(|x| x.iter().map(|v| v * c).collect()) + .collect(); + let got = deltas_over_sequence(m, &scaled)?; + for (pos, (g, r)) in got.iter().zip(&reference).enumerate() { + for (i, (a, b)) in g.iter().zip(r).enumerate() { + assert!( + (a - b).abs() <= CONTRACT_TOL, + "QHF-INV-006 {what}: delta for {c} * h differs from delta for h at position \ + {pos}, coordinate {i}: {a} vs {b}; the sublayer does not read RMSNorm(h)" + ); + } + } + } + Ok(()) +} + +/// QHF-INV-006 `pre-norm architecture: norm before attention/GDN and before FFN` +/// (FALSIFY-QHF-006), by the EFFECT of the order: RMSNorm is invariant to a positive scale +/// of its input, so a pre-normed sublayer adds the same delta for `c * h` as for `h`. A +/// sublayer that reads the raw residual (norm missing, or moved after it) adds a delta +/// that changes with `c`. Checked for each sublayer on its own (the other zeroed), in both +/// layer types, over a three-token sequence. c = 4 and 1/4 are exact in binary floating +/// point, so the norm of the scaled input is bit-identical; the bound is CONTRACT_TOL. +#[test] +fn qhf_inv_006_rmsnorm_precedes_each_sublayer() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let hd_attn = dims.num_heads * dims.attn_head_dim; + let mut attn_only = attention_layer(dims, 601); + attn_only.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let mut gdn_only = deltanet_layer(dims, 602); + gdn_only.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let mut ffn_in_attn = attention_layer(dims, 603); + ffn_in_attn.attn_output = zero_tensor(hd_attn, dims.hidden); + let mut ffn_in_gdn = deltanet_layer(dims, 604); + ffn_in_gdn.ssm_out = zero_tensor(dims.gdn_width(), dims.hidden); + for (layer, what) in [ + (Qwen35OwnedLayer::Attention(attn_only), "attention sublayer"), + (Qwen35OwnedLayer::DeltaNet(gdn_only), "GDN sublayer"), + ( + Qwen35OwnedLayer::Attention(ffn_in_attn), + "FFN sublayer (attention layer)", + ), + ( + Qwen35OwnedLayer::DeltaNet(ffn_in_gdn), + "FFN sublayer (GDN layer)", + ), + ] { + let m = model(&base, dims, vec![layer]); + for seed in INPUT_SEEDS { + assert_delta_scale_invariant(&m, &inputs(dims, seed), what)?; + } + } + } + Ok(()) +} + +// --- QHF-CON-007: residual identity ------------------------------------------------------- + +/// QHF-CON-007 `h_{l+1} - h_l = sublayer(norm(h_l))`, `tolerance: 1.0e-06` +/// (FALSIFY-QHF-007). The residual delta the code produces is compared with an +/// independent evaluation of `sublayer(norm(h_l))` from the contract's equations +/// (`attention_sublayer`, `gdn_sublayer`, `ffn_sublayer`): each sublayer isolated, then the +/// whole block `h + mixer(norm(h)) + ffn(norm(h + mixer(norm(h))))`. Position 0 of a fresh +/// state, where the attention softmax and the delta-rule state have closed forms. +#[test] +fn qhf_con_007_residual_identity() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let eps = base.config.eps; + for seed in INPUT_SEEDS { + let h = &inputs(dims, seed)[..1]; + let h0 = &h[0]; + + // Attention layer. + let attn = attention_layer(dims, 701); + let mixer = ref_attention_pos0(dims, &attn, h0, eps); + let mid: Vec = h0.iter().zip(&mixer).map(|(a, b)| a + b).collect(); + let ffn = ref_ffn( + &mid, + &attn.post_attention_norm, + &attn.ffn_gate, + &attn.ffn_up, + &attn.ffn_down, + eps, + ); + let ffn_alone = ref_ffn( + h0, + &attn.post_attention_norm, + &attn.ffn_gate, + &attn.ffn_up, + &attn.ffn_down, + eps, + ); + let block: Vec = mixer.iter().zip(&ffn).map(|(a, b)| a + b).collect(); + + let mut only_mixer = attention_layer(dims, 701); + only_mixer.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let mut only_ffn = attention_layer(dims, 701); + only_ffn.attn_output = zero_tensor(dims.num_heads * dims.attn_head_dim, dims.hidden); + for (layer, want, what) in [ + (only_mixer, &mixer, "attention sublayer"), + (only_ffn, &ffn_alone, "FFN sublayer (attention layer)"), + (attn, &block, "attention block"), + ] { + let m = model(&base, dims, vec![Qwen35OwnedLayer::Attention(layer)]); + let got = deltas_over_sequence(&m, h)?; + assert_close(&got[0], want, &format!("QHF-CON-007 {what}")); + } + + // Gated DeltaNet layer. + let gdn = deltanet_layer(dims, 702); + let mixer = ref_deltanet_pos0(dims, &gdn, h0, eps); + let mid: Vec = h0.iter().zip(&mixer).map(|(a, b)| a + b).collect(); + let ffn = ref_ffn( + &mid, + &gdn.post_attention_norm, + &gdn.ffn_gate, + &gdn.ffn_up, + &gdn.ffn_down, + eps, + ); + let ffn_alone = ref_ffn( + h0, + &gdn.post_attention_norm, + &gdn.ffn_gate, + &gdn.ffn_up, + &gdn.ffn_down, + eps, + ); + let block: Vec = mixer.iter().zip(&ffn).map(|(a, b)| a + b).collect(); + + let mut only_mixer = deltanet_layer(dims, 702); + only_mixer.ffn_down = zero_tensor(dims.intermediate, dims.hidden); + let mut only_ffn = deltanet_layer(dims, 702); + only_ffn.ssm_out = zero_tensor(dims.gdn_width(), dims.hidden); + for (layer, want, what) in [ + (only_mixer, &mixer, "GDN sublayer"), + (only_ffn, &ffn_alone, "FFN sublayer (GDN layer)"), + (gdn, &block, "GDN block"), + ] { + let m = model(&base, dims, vec![Qwen35OwnedLayer::DeltaNet(layer)]); + let got = deltas_over_sequence(&m, h)?; + assert_close(&got[0], want, &format!("QHF-CON-007 {what}")); + } + } + } + Ok(()) +} + +// === contracts/qwen35-e2e-verification-v1.yaml (QE2E) ====================================== +// +// Six of the seven obligations are discharged against this code: QE2E-ORD-003, +// QE2E-INV-006 and QE2E-CON-007 below, and QE2E-INV-001's parameter sum, QE2E-BND-002 and +// QE2E-MON-004 at the end of this file -- none of the three needs a device, because each is +// arithmetic over the model's own tensor inventory or over this contract's own equation. +// QE2E-BND-005 (coverage(qwen35_contracts) = 1.0) is NOT proved: it is false today, and +// asserting it would mean lowering it until it passed. QE2E-INV-001's 9B range is not +// asserted either -- see the note on that obligation in the contract (#3091). + +/// Tokens that visit the first and the last embedding row. +fn token_sequence(dims: Dims, seq_len: usize) -> Vec { + (0..seq_len) + .map(|p| u32::try_from((p * 3 + 1) % dims.vocab).unwrap_or(0)) + .collect() +} + +// --- QE2E-ORD-003: quantization memory ordering ------------------------------------------- + +/// QE2E-ORD-003 `M(Q4K) < M(Q6K) < M(F16) < M(F32)` (FALSIFY-QE2E-003, "Quantization byte +/// formula wrong"). The four byte counts are MEASURED, not read from block constants: the +/// same seeded tensor is encoded by the real encoders (`quantize_q4_k`, `quantize_q6_k`, +/// `f32_to_f16`, `f32::to_le_bytes`), the ordering is asserted on what they wrote, and then +/// each encoding is written into a GGUF and the loader's `get_tensor_ref` must reserve +/// exactly that many bytes, so the ordering holds for the bytes the serving path maps. +/// +/// Domain: whole 256-element super-blocks, the unit ggml's K-quant encoders are defined on. +/// The contract's formal states no domain; below one super-block the padded K-quant blocks +/// are LARGER than F16 (see #3091), so the formal is false for tiny tensors as written. +#[test] +fn qe2e_ord_003_quantization_memory_ordering() -> Result<()> { + use crate::gguf::test_factory::GGUFBuilder; + use crate::gguf::{GGUFModel, QuantizedGGUFTransformer}; + + for (i, dims) in [vec![256u64], vec![1024], vec![256, 512]] + .into_iter() + .enumerate() + { + let n: usize = dims + .iter() + .map(|&d| usize::try_from(d).unwrap_or(0)) + .product(); + let data = Rng::new(31 + i as u64).vec(n); + let q4k = trueno_quant::quantize_q4_k(&data); + let q6k = trueno_quant::quantize_q6_k(&data); + let f16: Vec = data + .iter() + .flat_map(|v| trueno_quant::f32_to_f16(*v).to_le_bytes()) + .collect(); + let f32_bytes: Vec = data.iter().flat_map(|v| v.to_le_bytes()).collect(); + + let measured = [ + ("Q4K", q4k.len()), + ("Q6K", q6k.len()), + ("F16", f16.len()), + ("F32", f32_bytes.len()), + ]; + for pair in measured.windows(2) { + assert!( + pair[0].1 < pair[1].1, + "QE2E-ORD-003 dims {dims:?}: M({}) = {} bytes is not < M({}) = {} bytes", + pair[0].0, + pair[0].1, + pair[1].0, + pair[1].1 + ); + } + + let file = GGUFBuilder::new() + .add_q4_k_tensor("q4k", &dims, &q4k) + .add_q6_k_tensor("q6k", &dims, &q6k) + .add_f16_tensor("f16", &dims, &f16) + .add_f32_tensor("f32", &dims, &data) + .build(); + let gguf = GGUFModel::from_bytes(&file)?; + for (name, (label, written)) in ["q4k", "q6k", "f16", "f32"].into_iter().zip(measured) { + let reserved = QuantizedGGUFTransformer::get_tensor_ref(&gguf, &file, name)?.byte_size; + assert_eq!( + reserved, written, + "QE2E-ORD-003 dims {dims:?}: the loader maps {reserved} bytes for the {label} \ + tensor, the encoder wrote {written}" + ); + } + } + Ok(()) +} + +// --- QE2E-INV-006: every block preserves shape --------------------------------------------- + +/// QE2E-INV-006 `∀l: shape(block_l(x)) = shape(x)` (FALSIFY-QE2E-006). The residual stream +/// of a real token is carried through the eight-layer hybrid stack ONE BLOCK AT A TIME: at +/// every layer `l` the block's output must be `d_model` wide, finite, and must have written +/// every one of the `d_model` coordinates (a block whose output is narrower than `x` leaves +/// a coordinate's delta at exactly zero). The hidden buffer is a slice, so its LENGTH cannot +/// change; the width a block actually writes is the observable shape. +/// +/// So that the `x` fed to each block is the model's real `h_l` and not a test artefact, the +/// block-by-block chain must reproduce `forward_single_qwen35`'s logits bit for bit (same +/// code, same inputs; no tolerance is involved). +#[test] +fn qe2e_inv_006_every_block_preserves_d_model() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let m = model(&base, dims, hybrid_layers(dims, 800)); + let tokens = token_sequence(dims, SEQ_LEN); + let mut by_block = m.new_state(SEQ_LEN); + let mut whole = m.new_state(SEQ_LEN); + for (pos, &token) in tokens.iter().enumerate() { + let t = token as usize; + let mut h = base.token_embedding[t * dims.hidden..(t + 1) * dims.hidden].to_vec(); + for l in 0..m.layers.len() { + let out = run_block(&m, l, &h, &mut by_block, pos)?; + assert_eq!(out.len(), dims.hidden, "QE2E-INV-006 block {l} width"); + for (i, (o, x)) in out.iter().zip(&h).enumerate() { + assert!( + o.is_finite() && o != x, + "QE2E-INV-006 block {l} (position {pos}) did not write coordinate {i} \ + of d_model={}: x = {x}, block(x) = {o}", + dims.hidden + ); + } + h = out; + } + by_block.kv_cache.advance(); + + let mut normed = vec![0.0; dims.hidden]; + crate::gguf::ops::rms_norm_into( + &h, + &base.output_norm_weight, + base.config.eps, + &mut normed, + ); + let mut chained = vec![0.0; dims.vocab]; + base.fused_matmul_into(&normed, &base.lm_head_weight, &mut chained)?; + let logits = m.forward_single_qwen35(token, &mut whole, pos)?; + assert_eq!( + chained, logits, + "QE2E-INV-006 position {pos}: the block-by-block chain is not the model" + ); + } + } + Ok(()) +} + +// --- QE2E-CON-007: tokens in, [seq_len, V] logits out -------------------------------------- + +/// QE2E-CON-007 `shape(model(tokens)) = [seq_len, V]`, `tolerance: 0.0` (FALSIFY-QE2E-007). +/// A token sequence is run through the full hybrid stack (`forward_single_qwen35` per +/// position, as `run_qwen35_generate` prefills) from a state sized for exactly `seq_len` +/// positions. The result must be `seq_len` rows of exactly `V` logits, each one finite and +/// written (a logit buffer wider than `V` leaves coordinates at exactly zero; a narrower one +/// fails the width). Sequence lengths 1, 3 and 6 over two vocabularies, every token id in +/// range including `V - 1`. +#[test] +fn qe2e_con_007_tokens_in_logits_out_shape() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let m = model(&base, dims, hybrid_layers(dims, 900)); + for seq_len in [1usize, 3, 6] { + let tokens = token_sequence(dims, seq_len); + let mut state = m.new_state(seq_len); + let logits = tokens + .iter() + .enumerate() + .map(|(pos, &t)| m.forward_single_qwen35(t, &mut state, pos)) + .collect::>>>()?; + assert_eq!(logits.len(), seq_len, "QE2E-CON-007 rows for {tokens:?}"); + for (pos, row) in logits.iter().enumerate() { + assert_eq!( + row.len(), + dims.vocab, + "QE2E-CON-007 position {pos}: logits width, V = {}", + dims.vocab + ); + for (v, x) in row.iter().enumerate() { + assert!( + x.is_finite() && *x != 0.0, + "QE2E-CON-007 position {pos}: logit {v} is {x} (never written or not \ + finite); row {row:?}" + ); + } + } + } + } + Ok(()) +} + +// --- QHF-BND-005: activation stability ------------------------------------------------------- + +/// FALSIFY-QHF-005 "No NaN/Inf after 48 layers": the depth the prediction names. +const BND_005_LAYERS: u64 = 48; +/// One more position than the conv window holds, so every tap of the causal conv state and +/// the `DeltaNet` recurrent state is overwritten at least once, and the KV cache grows. +const BND_005_POSITIONS: usize = CONV_KERNEL + 1; + +/// One adversarial-but-finite input sequence: `positions` hidden vectors of width `hidden`. +type SeqGen = fn(usize, usize) -> Vec>; + +fn signed(rng: &mut Rng, magnitude: f32) -> f32 { + if rng.next_f32() < 0.0 { + -magnitude + } else { + magnitude + } +} + +fn seq_constant(value: f32) -> impl Fn(usize, usize) -> Vec> { + move |hidden, positions| vec![vec![value; hidden]; positions] +} + +fn seq_signed(magnitude: f32, seed: u64) -> impl Fn(usize, usize) -> Vec> { + move |hidden, positions| { + let mut r = Rng::new(seed); + (0..positions) + .map(|_| (0..hidden).map(|_| signed(&mut r, magnitude)).collect()) + .collect() + } +} + +/// Every coordinate of one vector cycles through huge, underflowing, zero and O(1) values of +/// both signs, so the RMS is dominated by a few coordinates and the rest are ~0 after the norm. +fn seq_mixed(hidden: usize, positions: usize) -> Vec> { + let mut r = Rng::new(5); + (0..positions) + .map(|p| { + (0..hidden) + .map(|i| match (i + p) % 6 { + 0 => 1.0e4, + 1 => -1.0e-30, + 2 => 0.0, + 3 => -1.0e4, + 4 => 1.0e-30, + _ => r.next_f32(), + }) + .collect() + }) + .collect() +} + +/// Magnitude switches between positions: the recurrent and conv states built from a huge +/// token are then read by a zero token and an underflowing one. +fn seq_switching(hidden: usize, positions: usize) -> Vec> { + let mut r = Rng::new(9); + (0..positions) + .map(|p| match p % 4 { + 0 => (0..hidden).map(|_| signed(&mut r, 1.0e4)).collect(), + 1 => vec![0.0; hidden], + 2 => (0..hidden).map(|_| signed(&mut r, 1.0e-30)).collect(), + _ => vec![-1.0e4; hidden], + }) + .collect() +} + +fn assert_all_finite(h: &[f32], case: &str, hidden: usize, layer: usize, kind: &str, pos: usize) { + for (i, v) in h.iter().enumerate() { + assert!( + v.is_finite(), + "QHF-BND-005 violated: input `{case}` (d_model={hidden}), after layer {layer} \ + ({kind}), position {pos}, coordinate {i} = {v}" + ); + } +} + +/// QHF-BND-005 `∀l ∈ [0, L], ∀i: is_finite(h_l[i])` (FALSIFY-QHF-005). Drives the real +/// `forward_deltanet` / `forward_attention` blocks through a 48-layer Qwen3.5 schedule +/// (36 Gated `DeltaNet` + 12 attention), token by token over `BND_005_POSITIONS` positions +/// sharing one state, exactly as `forward_single_qwen35` chains them, and checks EVERY +/// coordinate of `h_0` and of `h_l` after each layer. +/// +/// Inputs are adversarial but finite. The all-zero vector is a fixed point of the whole +/// stack (no biases), so every RMSNorm, per-head L2 norm and gated RMSNorm of all 48 layers +/// sees `sum(x^2) = 0` and only its `eps` stands between `0 * (1/sqrt(0))` and NaN. The +/// `1e-30` input squares to `0` in f32 (underflow), reaching the same `eps` with nonzero +/// values (`x * inf = inf`). `±1e30` squares to `+inf`, so `inv_rms = 0`. +#[test] +fn qhf_bnd_005_hidden_states_stay_finite() -> Result<()> { + let cases: [(&str, &dyn Fn(usize, usize) -> Vec>); 9] = [ + ("all +1e4", &seq_constant(1.0e4)), + ("all -1e4", &seq_constant(-1.0e4)), + ("mixed sign ±1e4", &seq_signed(1.0e4, 3)), + ( + "mixed sign ±1e-30 (squares underflow)", + &seq_signed(1.0e-30, 4), + ), + ("all-zero", &seq_constant(0.0)), + ( + "mixed magnitude and sign within a vector", + &(seq_mixed as SeqGen), + ), + ( + "magnitude switching across positions", + &(seq_switching as SeqGen), + ), + ( + "mixed sign ±1e30 (squares overflow)", + &seq_signed(1.0e30, 6), + ), + ("seeded O(1) control", &seq_signed(0.25, 7)), + ]; + for dims in DIMS { + let base = base_model(dims); + let m = model( + &base, + dims, + hybrid_schedule(dims, dims.hidden as u64 * 1000, BND_005_LAYERS), + ); + assert_eq!(m.layers.len(), BND_005_LAYERS as usize, "stack depth"); + for (case, gen) in cases { + let seq = gen(dims.hidden, BND_005_POSITIONS); + let mut state = m.new_state(BND_005_POSITIONS + 1); + for (pos, h0) in seq.iter().enumerate() { + assert_all_finite(h0, case, dims.hidden, 0, "input h_0", pos); + let mut h = h0.clone(); + for il in 0..m.layers.len() { + h = run_block(&m, il, &h, &mut state, pos)?; + assert_eq!(h.len(), dims.hidden, "{case}: width after layer {}", il + 1); + let kind = match &m.layers[il] { + Qwen35OwnedLayer::DeltaNet(_) => "Gated DeltaNet", + Qwen35OwnedLayer::Attention(_) => "attention", + }; + assert_all_finite(&h, case, dims.hidden, il + 1, kind, pos); + } + state.kv_cache.advance(); + } + } + } + Ok(()) +} + +// === QE2E-INV-001 / QE2E-BND-002 / QE2E-MON-004: size, cost, throughput ==================== +// +// These three obligations are about the model's SIZE and COST rather than its outputs, so +// they are discharged against an inventory of the real `Qwen35Model` structures instead of a +// forward pass. Nothing here reads a device: `tok_s` is DEFINED by this contract's own +// `throughput_model` equation, and every constant is derived from the tensors the synthetic +// models actually carry. + +/// Elements a weight tensor carries, from its BYTES (F32 here), not from its declared shape: +/// a `predicted_parameters` term that disagrees with the tensor the fixture built is then a +/// falsification rather than two copies of the same typo. +fn tensor_elements(t: &OwnedQuantizedTensor) -> u64 { + u64::try_from(t.data.len() / 4).unwrap_or(0) +} + +fn as_u64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(0) +} + +/// Parameters of one full-attention layer, measured tensor by tensor. +fn attention_layer_elements(a: &Qwen35OwnedAttentionLayer) -> u64 { + as_u64(a.attn_norm.len()) + + tensor_elements(&a.attn_q) + + tensor_elements(&a.attn_k) + + tensor_elements(&a.attn_v) + + as_u64(a.attn_q_norm.len()) + + as_u64(a.attn_k_norm.len()) + + tensor_elements(&a.attn_output) + + as_u64(a.post_attention_norm.len()) + + tensor_elements(&a.ffn_gate) + + tensor_elements(&a.ffn_up) + + tensor_elements(&a.ffn_down) +} + +/// Parameters of one Gated `DeltaNet` layer, measured tensor by tensor. +fn deltanet_layer_elements(d: &Qwen35OwnedDeltaNetLayer) -> u64 { + as_u64(d.attn_norm.len()) + + tensor_elements(&d.attn_qkv) + + tensor_elements(&d.attn_gate) + + tensor_elements(&d.ssm_alpha) + + tensor_elements(&d.ssm_beta) + + as_u64(d.ssm_a.len()) + + as_u64(d.ssm_dt_bias.len()) + + as_u64(d.ssm_conv1d_weight.len()) + + as_u64(d.ssm_norm_weight.len()) + + tensor_elements(&d.ssm_out) + + as_u64(d.post_attention_norm.len()) + + tensor_elements(&d.ffn_gate) + + tensor_elements(&d.ffn_up) + + tensor_elements(&d.ffn_down) +} + +/// `P`, MEASURED: every parameter the model carries, counted from the tensors themselves. +fn measured_parameters(base: &OwnedQuantizedModel, layers: &[Qwen35OwnedLayer]) -> u64 { + let mut n = as_u64(base.token_embedding.len()) + + as_u64(base.output_norm_weight.len()) + + tensor_elements(&base.lm_head_weight); + for layer in layers { + n += match layer { + Qwen35OwnedLayer::Attention(a) => attention_layer_elements(a), + Qwen35OwnedLayer::DeltaNet(d) => deltanet_layer_elements(d), + }; + } + n +} + +/// `P`, PREDICTED from the architecture config alone — the contract's +/// `P = V*d + L*(d_attn + d_ffn + d_norm) + d_final + V*d` instantiated on `Dims`, with +/// `d_attn` split by layer type because Qwen3.5 is hybrid. `V*d` appears twice: the +/// embedding and the untied lm_head (`constraints.tied_embeddings = false`). +fn predicted_parameters(dims: Dims, layers: &[Qwen35OwnedLayer]) -> u64 { + let (h, hd, inter) = ( + as_u64(dims.hidden), + as_u64(dims.attn_head_dim), + as_u64(dims.intermediate), + ); + let v = as_u64(dims.vocab); + let (nh, nkv) = (as_u64(dims.num_heads), as_u64(dims.num_kv_heads)); + let (gdn_w, conv_d) = (as_u64(dims.gdn_width()), as_u64(dims.conv_dim())); + let (gdn_h, gdn_hd) = (as_u64(dims.gdn_heads), as_u64(dims.gdn_head_dim)); + // Every layer, whatever its mixer, carries two RMSNorms and the SwiGLU FFN. + let d_norm = 2 * h; + let d_ffn = 3 * h * inter; + // Gated full attention: q is DOUBLE width (the gate is packed into it, as the loader reads it). + let d_attn = h * nh * hd * 2 + 2 * (h * nkv * hd) + nh * hd * h + 2 * hd; + let d_gdn = h * conv_d + + h * gdn_w + + 2 * (h * gdn_h) + + 2 * gdn_h + + as_u64(CONV_KERNEL) * conv_d + + gdn_hd + + gdn_w * h; + let per_layer: u64 = layers + .iter() + .map(|l| match l { + Qwen35OwnedLayer::Attention(_) => d_attn + d_ffn + d_norm, + Qwen35OwnedLayer::DeltaNet(_) => d_gdn + d_ffn + d_norm, + }) + .sum(); + v * h + per_layer + h + v * h +} + +/// QE2E-INV-001's parameter SUM (FALSIFY-QE2E-001, "sum all parameter shapes"), over the +/// real `Qwen35Model` structures and both layer types: the count derived from the +/// architecture config must equal the count measured tensor by tensor from the model itself. +/// +/// This discharges the `model_parameter_count` EQUATION, not the obligation's 9B +/// instantiation `P(Qwen3.5-9B) ∈ [9.0B, 9.2B]`: see the note on QE2E-INV-001 in the +/// contract — this same formula on the in-tree 9B constants +/// (`contracts/model-families/qwen3_5.yaml`) gives 8.209B, outside the stated range, and no +/// in-tree descriptor states the Gated `DeltaNet` tensor shapes that the missing parameters +/// would have to come from (#3091). +#[test] +fn qe2e_inv_001_parameter_sum_matches_tensor_inventory() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + for layers in [hybrid_layers(dims, 1000), hybrid_schedule(dims, 1100, 4)] { + let measured = measured_parameters(&base, &layers); + let predicted = predicted_parameters(dims, &layers); + assert!( + measured > 0, + "QE2E-INV-001: an empty inventory would pass any sum" + ); + assert_eq!( + predicted, + measured, + "QE2E-INV-001 ({} layers, d_model={}): the parameter count derived from the \ + architecture config is {predicted}, the model carries {measured}", + layers.len(), + dims.hidden + ); + } + } + Ok(()) +} + +/// Weight FLOPs per decoded token: two per multiply-add of every projection the forward pass +/// runs. Norms and the embedding lookup are excluded — they are parameters, not matmuls. +fn weight_flops_per_token(base: &OwnedQuantizedModel, layers: &[Qwen35OwnedLayer]) -> u64 { + let mut mac = tensor_elements(&base.lm_head_weight); + for layer in layers { + mac += match layer { + Qwen35OwnedLayer::Attention(a) => { + tensor_elements(&a.attn_q) + + tensor_elements(&a.attn_k) + + tensor_elements(&a.attn_v) + + tensor_elements(&a.attn_output) + + tensor_elements(&a.ffn_gate) + + tensor_elements(&a.ffn_up) + + tensor_elements(&a.ffn_down) + }, + Qwen35OwnedLayer::DeltaNet(d) => { + tensor_elements(&d.attn_qkv) + + tensor_elements(&d.attn_gate) + + tensor_elements(&d.ssm_alpha) + + tensor_elements(&d.ssm_beta) + + tensor_elements(&d.ssm_out) + }, + }; + } + 2 * mac +} + +/// Full-attention layers, the only ones whose cost grows with the KV cache: a Gated +/// `DeltaNet` layer carries a fixed-size recurrent state. +fn full_attention_layers(layers: &[Qwen35OwnedLayer]) -> u64 { + as_u64( + layers + .iter() + .filter(|l| matches!(l, Qwen35OwnedLayer::Attention(_))) + .count(), + ) +} + +/// Attention FLOPs at cache depth `s`, counted TERM BY TERM rather than from a constant: in +/// every full-attention layer `QK^T` costs one multiply-add (2 FLOPs) per cached key element +/// and `A·V` one per cached value element, i.e. `2*d_kv*s` each. A Gated `DeltaNet` layer +/// contributes nothing here -- its recurrent state is fixed-size, which is why `L_attn` and +/// not `L` carries the growth. +fn attention_flops_at_depth(dims: Dims, layers: &[Qwen35OwnedLayer], s: u64) -> u64 { + let d_kv = as_u64(dims.num_kv_heads * dims.attn_head_dim); + let mut f = 0; + for layer in layers { + if matches!(layer, Qwen35OwnedLayer::Attention(_)) { + f += 2 * d_kv * s; // QK^T over the cached keys + f += 2 * d_kv * s; // A·V over the cached values + } + } + f +} + +/// FLOPs to decode one token at cache depth `s`. +fn flops_per_token( + dims: Dims, + base: &OwnedQuantizedModel, + layers: &[Qwen35OwnedLayer], + s: u64, +) -> u64 { + weight_flops_per_token(base, layers) + attention_flops_at_depth(dims, layers, s) +} + +/// QE2E-BND-002 `F(s) <= 2*P + 4*L_attn*d_kv*s` (FALSIFY-QE2E-002), the TIGHTENED form of +/// the contract's original `F <= 2*P + O(seq_len * d * L)` — an O() with no constant, which +/// no input can falsify. Both sides are measured from the same real model structures. +/// +/// Three claims, so that neither half of the bound can be dropped: +/// 1. at cache depth 0 the weight term alone is within `2*P` (every matmul weight is a +/// parameter, and `P` also holds the embedding and the norms); +/// 2. the bound holds at every depth up to the context the state allocates; +/// 3. the constant is EXACT, not merely sufficient: `F(s+1) - F(s) = 4*L_attn*d_kv`, so +/// lowering `4` or counting the `DeltaNet` layers into `L_attn` falsifies it. +#[test] +fn qe2e_bnd_002_flops_bounded_by_2p_plus_attention_term() -> Result<()> { + for dims in DIMS { + let base = base_model(dims); + let layers = hybrid_layers(dims, 1200); + let p = measured_parameters(&base, &layers); + // The bound's constant, written INDEPENDENTLY of the FLOP counter: 4 * L_attn * d_kv. + // Reading it back from the counter instead would make the slope assertion circular -- + // shrinking the counter would shrink the bound with it and nothing could fail (#3091). + let d_kv = as_u64(dims.num_kv_heads * dims.attn_head_dim); + let slope = 4 * full_attention_layers(&layers) * d_kv; + assert!( + slope > 0 && full_attention_layers(&layers) > 0, + "QE2E-BND-002: a schedule with no full-attention layer cannot exercise the bound" + ); + + let f0 = flops_per_token(dims, &base, &layers, 0); + assert!( + f0 <= 2 * p, + "QE2E-BND-002 (d_model={}): weight FLOPs {f0} exceed 2P = {} at cache depth 0", + dims.hidden, + 2 * p + ); + + for s in 0..=8u64 { + let f = flops_per_token(dims, &base, &layers, s); + let bound = 2 * p + slope * s; + assert!( + f <= bound, + "QE2E-BND-002 (d_model={}, s={s}): F = {f} exceeds 2P + 4*L_attn*d_kv*s = {bound}", + dims.hidden + ); + let next = flops_per_token(dims, &base, &layers, s + 1); + assert_eq!( + next - f, + slope, + "QE2E-BND-002 (d_model={}, s={s}): the attention term grows by {} per cached \ + position, the bound's constant claims {slope} — the bound is not tight", + dims.hidden, + next - f + ); + } + } + Ok(()) +} + +/// `tok/s = min(bandwidth / bytes_per_token, compute / flops_per_token)` — this contract's +/// own `throughput_model` equation, and the ONLY definition of `tok_s` QE2E-MON-004 is +/// written against. No device is involved. +fn tok_s(bandwidth: f64, bytes_per_token: f64, compute: f64, flops_per_token: f64) -> f64 { + (bandwidth / bytes_per_token).min(compute / flops_per_token) +} + +proptest::proptest! { + /// QE2E-MON-004 `bw1 < bw2 → tok_s(bw1) <= tok_s(bw2)` (FALSIFY-QE2E-004, "proptest with + /// random hardware specs"). The specs are random; the formula is the contract's. + #[test] + fn qe2e_mon_004_throughput_is_monotone_in_bandwidth( + bw1 in 1.0e6f64..1.0e13, + factor in 1.0f64..1.0e4, + bytes_per_token in 1.0e3f64..1.0e11, + flops_per_token in 1.0e3f64..1.0e13, + compute in 1.0e9f64..1.0e16, + ) { + let bw2 = bw1 * factor; + proptest::prop_assert!(bw1 <= bw2, "generator produced bw2 < bw1"); + let t1 = tok_s(bw1, bytes_per_token, compute, flops_per_token); + let t2 = tok_s(bw2, bytes_per_token, compute, flops_per_token); + proptest::prop_assert!( + t1 <= t2, + "QE2E-MON-004 falsified: bandwidth {bw1} -> {bw2} (bytes/token {bytes_per_token}, \ + flops/token {flops_per_token}, compute {compute}) took tok/s from {t1} to {t2}" + ); + } +} diff --git a/crates/aprender-serve/src/gguf/inference/forward/mod.rs b/crates/aprender-serve/src/gguf/inference/forward/mod.rs index 9abb357a8c..82f40f3d6f 100644 --- a/crates/aprender-serve/src/gguf/inference/forward/mod.rs +++ b/crates/aprender-serve/src/gguf/inference/forward/mod.rs @@ -8,6 +8,7 @@ mod batch; mod core; mod encoder_decoder; + mod forward_qwen3_moe; mod forward_qwen3_moe_gpu; mod forward_qwen3_moe_traced; diff --git a/crates/aprender-serve/src/gguf/mod.rs b/crates/aprender-serve/src/gguf/mod.rs index 2844f61d52..fd66403697 100644 --- a/crates/aprender-serve/src/gguf/mod.rs +++ b/crates/aprender-serve/src/gguf/mod.rs @@ -51,6 +51,7 @@ mod owned; #[cfg(feature = "cuda")] pub mod parity; mod quantized; +pub mod qwen35_load; pub mod qwen3_moe_load; mod runtime; mod transformer; @@ -120,3 +121,7 @@ mod io_tests; mod quantized_tests; #[cfg(test)] mod tests; + +/// Qwen3.5 / Qwen3.8 hybrid (Gated `DeltaNet` + gated attention) CPU forward (#3091). +#[path = "inference/forward/forward_qwen35.rs"] +pub mod forward_qwen35; diff --git a/crates/aprender-serve/src/gguf/qwen35_load.rs b/crates/aprender-serve/src/gguf/qwen35_load.rs new file mode 100644 index 0000000000..671ffc9f33 --- /dev/null +++ b/crates/aprender-serve/src/gguf/qwen35_load.rs @@ -0,0 +1,253 @@ +#![allow(missing_docs)] +use crate::error::Result; +use crate::gguf::quantized::QuantizedTensorRef; +use crate::gguf::types::GGUFValue; +use crate::gguf::GGUFModel; +use crate::gguf::QuantizedGGUFTransformer; + +#[derive(Clone, Debug)] +pub struct Qwen35DeltaNetLayer { + pub attn_norm: QuantizedTensorRef, + pub attn_qkv: QuantizedTensorRef, + pub attn_gate: QuantizedTensorRef, + pub ssm_alpha: QuantizedTensorRef, + pub ssm_beta: QuantizedTensorRef, + pub ssm_a: QuantizedTensorRef, + pub ssm_dt_bias: QuantizedTensorRef, + pub ssm_conv1d_weight: QuantizedTensorRef, + pub ssm_norm_weight: QuantizedTensorRef, + pub ssm_out: QuantizedTensorRef, + pub post_attention_norm: QuantizedTensorRef, + pub ffn_gate: QuantizedTensorRef, + pub ffn_up: QuantizedTensorRef, + pub ffn_down: QuantizedTensorRef, +} + +#[derive(Clone, Debug)] +pub struct Qwen35AttentionLayer { + pub attn_norm: QuantizedTensorRef, + pub attn_q: QuantizedTensorRef, + pub attn_k: QuantizedTensorRef, + pub attn_v: QuantizedTensorRef, + pub attn_q_norm: QuantizedTensorRef, + pub attn_k_norm: QuantizedTensorRef, + pub attn_output: QuantizedTensorRef, + pub post_attention_norm: QuantizedTensorRef, + pub ffn_gate: QuantizedTensorRef, + pub ffn_up: QuantizedTensorRef, + pub ffn_down: QuantizedTensorRef, +} + +#[derive(Clone, Debug)] +pub enum Qwen35Layer { + DeltaNet(Qwen35DeltaNetLayer), + Attention(Qwen35AttentionLayer), +} + +fn as_u32(v: &GGUFValue) -> Option { + match v { + GGUFValue::UInt32(x) => Some(*x), + _ => None, + } +} + +pub fn load_qwen35_layers(model: &GGUFModel, data: &[u8]) -> Result> { + let mut layers = Vec::new(); + let num_layers = model + .metadata + .get("block_count") + .and_then(as_u32) + .unwrap_or(24) as usize; + + let interval = model + .metadata + .get("full_attention_interval") + .and_then(as_u32) + .unwrap_or(4) as usize; + + for i in 0..num_layers { + if (i + 1) % interval == 0 { + // Full attention + layers.push(Qwen35Layer::Attention(Qwen35AttentionLayer { + attn_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_norm.weight", i), + )?, + attn_q: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_q.weight", i), + )?, + attn_k: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_k.weight", i), + )?, + attn_v: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_v.weight", i), + )?, + attn_q_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_q_norm.weight", i), + )?, + attn_k_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_k_norm.weight", i), + )?, + attn_output: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_output.weight", i), + )?, + post_attention_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.post_attention_norm.weight", i), + )?, + ffn_gate: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_gate.weight", i), + )?, + ffn_up: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_up.weight", i), + )?, + ffn_down: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_down.weight", i), + )?, + })); + } else { + // DeltaNet + layers.push(Qwen35Layer::DeltaNet(Qwen35DeltaNetLayer { + attn_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_norm.weight", i), + )?, + attn_qkv: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_qkv.weight", i), + )?, + attn_gate: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.attn_gate.weight", i), + )?, + ssm_alpha: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_alpha.weight", i), + )?, + ssm_beta: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_beta.weight", i), + )?, + ssm_a: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_a", i), + )?, + ssm_dt_bias: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_dt.bias", i), + )?, + ssm_conv1d_weight: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_conv1d.weight", i), + )?, + ssm_norm_weight: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_norm.weight", i), + )?, + ssm_out: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ssm_out.weight", i), + )?, + post_attention_norm: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.post_attention_norm.weight", i), + )?, + ffn_gate: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_gate.weight", i), + )?, + ffn_up: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_up.weight", i), + )?, + ffn_down: QuantizedGGUFTransformer::get_tensor_ref( + model, + data, + &format!("blk.{}.ffn_down.weight", i), + )?, + })); + } + } + + Ok(layers) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_qwen35_load_fixture() { + let fixture_path = std::path::PathBuf::from("/home/noah/models/Qwen3.5-0.8B-Q4_K_M.gguf"); + if !fixture_path.exists() { + println!("SKIP: {} absent", fixture_path.display()); + return; + } + + let data = std::fs::read(&fixture_path).expect("failed to read fixture file"); + let model = GGUFModel::from_bytes(&data).expect("failed to load fixture"); + let layers = load_qwen35_layers(&model, &data).expect("failed to extract layers"); + + assert_eq!(layers.len(), 24); + + let mut num_full_attn = 0; + let mut num_delta_net = 0; + + for (i, layer) in layers.iter().enumerate() { + match layer { + Qwen35Layer::Attention(attn) => { + num_full_attn += 1; + assert_eq!((i + 1) % 4, 0); + // Check some shapes + assert_eq!(attn.attn_q.num_elements, 1024 * 4096); + assert_eq!(attn.attn_k.num_elements, 1024 * 512); + assert_eq!(attn.attn_v.num_elements, 1024 * 512); + }, + Qwen35Layer::DeltaNet(delta) => { + num_delta_net += 1; + assert_ne!((i + 1) % 4, 0); + // Check some shapes + assert_eq!(delta.attn_qkv.num_elements, 1024 * 6144); + assert_eq!(delta.ssm_alpha.num_elements, 1024 * 16); + assert_eq!(delta.ssm_conv1d_weight.num_elements, 4 * 6144); + }, + } + } + + assert_eq!(num_delta_net, 18); + assert_eq!(num_full_attn, 6); + } +} diff --git a/crates/aprender-serve/src/infer/inference_result.rs b/crates/aprender-serve/src/infer/inference_result.rs index ac9684a4a4..ca8826b939 100644 --- a/crates/aprender-serve/src/infer/inference_result.rs +++ b/crates/aprender-serve/src/infer/inference_result.rs @@ -184,7 +184,15 @@ fn run_gguf_inference( let load_start = Instant::now(); let mapped = MappedGGUFModel::from_path(&config.model_path)?; prefault_mmap(mapped.data()); - let model = OwnedQuantizedModel::from_mapped(&mapped)?; + // #3091: Qwen3.5/Qwen3.8 hybrids (Gated DeltaNet) have their own CPU forward. The dense + // loader refuses them, so only the shared base (embeddings, final norm, lm_head) is built + // here and generation dispatches to `run_qwen35_generate` below. + let is_qwen35 = mapped.model.architecture() == Some("qwen35"); + let model = if is_qwen35 { + crate::gguf::forward_qwen35::Qwen35Model::create_base_model(&mapped.model, mapped.data())? + } else { + OwnedQuantizedModel::from_mapped(&mapped)? + }; let load_ms = load_start.elapsed().as_secs_f64() * 1000.0; // PMAT-109: Architecture from GGUF metadata (not filename) @@ -233,6 +241,17 @@ fn run_gguf_inference( &gen_config, )?; (tokens, false) // CPU-only path; GPU MoE wiring is M32d follow-up + } else if is_qwen35 { + if !config.no_gpu && cfg!(feature = "cuda") { + eprintln!("[qwen35: Gated DeltaNet runs on the CPU; the GPU backend does not implement it yet (#3090)]"); + } + let tokens = crate::gguf::forward_qwen35::run_qwen35_generate( + &mapped, + &model, + &input_tokens, + &gen_config, + )?; + (tokens, false) } else { run_gguf_generate(model, &input_tokens, &gen_config, config)? }; @@ -571,6 +590,29 @@ not CUDA." ) } +/// The F2 probe: the real prompt context (a peaked distribution) when there is one, else the +/// BOS token (batch model-init has no prompt yet), capped to bound the one-time CPU/GPU +/// prefill cost. BOS flows from GGUF metadata. Returns `(kv_dim, num_layers, probe)`, or +/// `None` when there is neither context nor a known BOS, so nothing to validate against. +#[cfg(feature = "cuda")] +fn gpu_probe( + model: &crate::gguf::OwnedQuantizedModel, + probe_context: &[u32], +) -> Option<(usize, usize, Vec)> { + const PROBE_MAX_CTX: usize = 64; + let kv_dim = model.config.num_kv_heads * (model.config.hidden_dim / model.config.num_heads); + let probe = if probe_context.is_empty() { + let Some(id) = model.config.bos_token_id else { + eprintln!("warning: no prompt context and no BOS token — skipping the GPU-vs-CPU output check"); + return None; + }; + vec![id] + } else { + probe_context[probe_context.len().saturating_sub(PROBE_MAX_CTX)..].to_vec() + }; + Some((kv_dim, model.config.num_layers, probe)) +} + #[cfg(feature = "cuda")] fn validate_gpu_first_token( cuda_model: &mut crate::gguf::OwnedQuantizedModelCuda, @@ -587,29 +629,8 @@ fn validate_gpu_first_token( return true; } - // Build the probe: real prompt context (peaked distribution) when available, - // else the BOS token (batch model-init has no prompt yet). Cap the context to - // bound the one-time CPU/GPU prefill cost. BOS flows from GGUF metadata; if it - // is unknown for a context-less probe there is nothing to validate against. - const PROBE_MAX_CTX: usize = 64; - let (kv_dim, num_layers, probe): (usize, usize, Vec) = { - let model = cuda_model.model(); - let kv_dim = - model.config.num_kv_heads * (model.config.hidden_dim / model.config.num_heads); - let num_layers = model.config.num_layers; - let probe: Vec = if probe_context.is_empty() { - match model.config.bos_token_id { - Some(id) => vec![id], - None => { - eprintln!("warning: no prompt context and no BOS token — skipping the GPU-vs-CPU output check"); - return true; - }, - } - } else { - let start = probe_context.len().saturating_sub(PROBE_MAX_CTX); - probe_context[start..].to_vec() - }; - (kv_dim, num_layers, probe) + let Some((kv_dim, num_layers, probe)) = gpu_probe(cuda_model.model(), probe_context) else { + return true; }; // A single-token probe (context-less BOS) has NO real position (≥1) to validate @@ -1189,7 +1210,7 @@ mod cuda_silent_fallback_tests { /// The rejection used `Err(_)` and returned a bare `false`. Its caller /// (gguf_gpu_generate.rs:303) can only convert that into `Err(Box::new(model))` /// for CPU fallback, so the reason had nowhere to go. Observed on an RTX 4090: - /// CUDA initialises, the run silently ends on CPU at ~20 tok/s instead of ~400, + /// CUDA initialises, the run silently ends on CPU far below the GPU decode rate, /// and nothing says why even under --verbose. /// /// That matters beyond ergonomics: throughput measured after a silent fallback diff --git a/evidence/parity/qwen35/lambda/llama_tokens.txt b/evidence/parity/qwen35/lambda/llama_tokens.txt new file mode 100644 index 0000000000..06d8e94792 --- /dev/null +++ b/evidence/parity/qwen35/lambda/llama_tokens.txt @@ -0,0 +1,6 @@ +PROMPT_TOKENS: 760 6511 314 9338 369 +OUTPUT_TOKENS: 7172 303 279 9514 314 279 3046 13 198 760 6511 314 9338 369 7172 303 279 9514 314 279 3046 13 198 760 6511 314 9338 369 +PROMPT_TOKENS: 1206 3165 264 6518 303 351 11 488 1220 1118 +OUTPUT_TOKENS: 6762 264 523 421 10551 279 30394 5426 13 1061 523 668 6435 279 5426 364 21621 279 1879 886 13 561 523 +PROMPT_TOKENS: 1421 25 21251 0 5044 3165 264 2716 31708 883 264 7993 13 198 69267 25 +OUTPUT_TOKENS: 271 248068 271 248069 271 332 760 65209 93630 332 271 32 2526 11 17024 281 4679 diff --git a/evidence/parity/qwen35/lambda/parity.json b/evidence/parity/qwen35/lambda/parity.json new file mode 100644 index 0000000000..1b59c2333f --- /dev/null +++ b/evidence/parity/qwen35/lambda/parity.json @@ -0,0 +1,195 @@ +{ + "model": "~/models/Qwen3.5-0.8B-Q4_K_M.gguf", + "tau": 0.25, + "pass": true, + "prompts": [ + { + "prompt_len": 5, + "llama_tokens": [ + 7172, + 303, + 279, + 9514, + 314, + 279, + 3046, + 13, + 198, + 760, + 6511, + 314, + 9338, + 369, + 7172, + 303, + 279, + 9514, + 314, + 279, + 3046, + 13, + 198, + 760, + 6511, + 314, + 9338, + 369 + ], + "greedy_tokens": [ + 279, + 6511, + 314, + 279, + 3046, + 13, + 198, + 760, + 6511, + 314, + 9338, + 369, + 279, + 6511, + 314, + 279, + 3046, + 13, + 198, + 760, + 6511, + 314, + 9338, + 369, + 279, + 6511, + 314, + 279 + ], + "greedy_divergence": 0, + "teacher_forced_agree": 27, + "teacher_forced_steps": 28, + "disagreements": [ + { + "step": 0, + "llama": 7172, + "realizar": 279, + "gap": 0.14469337, + "rank": 2 + } + ], + "min_agreeing_margin": 0.035121918 + }, + { + "prompt_len": 10, + "llama_tokens": [ + 6762, + 264, + 523, + 421, + 10551, + 279, + 30394, + 5426, + 13, + 1061, + 523, + 668, + 6435, + 279, + 5426, + 364, + 21621, + 279, + 1879, + 886, + 13, + 561, + 523 + ], + "greedy_tokens": [ + 6762, + 264, + 523, + 421, + 10551, + 279, + 30394, + 5426, + 13, + 1061, + 523, + 668, + 6435, + 279, + 5426, + 364, + 21621, + 279, + 1879, + 886, + 13, + 561, + 523 + ], + "greedy_divergence": null, + "teacher_forced_agree": 23, + "teacher_forced_steps": 23, + "disagreements": [], + "min_agreeing_margin": 0.009845734 + }, + { + "prompt_len": 16, + "llama_tokens": [ + 271, + 248068, + 271, + 248069, + 271, + 332, + 760, + 65209, + 93630, + 332, + 271, + 32, + 2526, + 11, + 17024, + 281, + 4679 + ], + "greedy_tokens": [ + 271, + 248068, + 271, + 248069, + 271, + 332, + 760, + 65209, + 93630, + 332, + 271, + 32, + 2526, + 17024, + 281, + 4679, + 68967 + ], + "greedy_divergence": 13, + "teacher_forced_agree": 16, + "teacher_forced_steps": 17, + "disagreements": [ + { + "step": 13, + "llama": 11, + "realizar": 17024, + "gap": 0.18462467, + "rank": 2 + } + ], + "min_agreeing_margin": 0.038477898 + } + ] +} \ No newline at end of file diff --git a/scripts/claim_literal_baseline.txt b/scripts/claim_literal_baseline.txt index 20e0cbce9e..563ada752a 100644 --- a/scripts/claim_literal_baseline.txt +++ b/scripts/claim_literal_baseline.txt @@ -249,7 +249,6 @@ crates/aprender-serve/src/gguf/cuda/speculative.rs:246 crates/aprender-serve/src/gguf/inference/cached/sync_owned_quantized_02.rs:25 crates/aprender-serve/src/gguf/inference/cached/sync_owned_quantized_02.rs:26 crates/aprender-serve/src/gguf/inference/forward/forward_qwen3_moe_gpu.rs:24 -crates/aprender-serve/src/infer/inference_result.rs:1192 crates/aprender-serve/src/infer/qwen3_moe_generate.rs:21 crates/aprender-serve/src/inference/thread.rs:10 crates/aprender-serve/src/inference/thread.rs:11