feat(qwen4): add Qwen3.8-Flash-Next PLE n-gram golden

.github/workflows/ci.yml

@@ -151,6 +151,10 @@ jobs:
         if: ${{ !cancelled() && steps.deps.outcome == 'success' }}
         run: cargo test --release --locked -p pegainfer-sim --lib
 
+      - name: Run pegainfer-qwen4 tests
+        if: ${{ !cancelled() && steps.deps.outcome == 'success' }}
+        run: cargo test --release --locked -p pegainfer-qwen4 --tests
+
       - name: Run kvbm-logical unit tests
         if: ${{ !cancelled() && steps.deps.outcome == 'success' }}
         run: cargo test --release --locked -p kvbm-logical --lib
@@ -188,6 +192,7 @@ jobs:
           cargo clippy --release --locked
           -p pegainfer-build
           -p pegainfer-frontend
+          -p pegainfer-qwen4
           -p pegainfer-sim
           -p kvbm-logical
           --all-targets -- -D warnings

docs/index.md

@@ -65,7 +65,7 @@ Organized by domain (model line / subsystem / playbook / lesson) instead of by l
 
 | Path | TL;DR |
 | --- | --- |
-| `models/qwen4/bring-up.md` | New line (`qwen4_exp`, first checkpoint Qwen3.8-Flash-Next), not a Qwen3.5 variant; frozen at `de4b8e4d`. Landed: an identity probe and one typed config validation that pins structure rather than widths, so a smaller `qwen4_exp` checkpoint passes it, models the router's upstream default (`norm_topk_prob`) and derives the n-gram table from five config fields via 16 per-head primes. Measured traps: untied LM head, **no final-norm tensor** (the readout is the hyper-connection mixer), `ple_layer_ids` is 1-indexed so the n-gram block is on `layers.1`, the n-gram table is **102.4 GB at BF16** (the quoted ~51 GB is fp8), `output_gate_type: "sigmoid"` is a **live** field selecting the GDN gated-norm activation (the inverse of the Qwen3.8-27B lesson), two RMSNorm conventions in one layer, and the PLE conv is dilated despite an identical tensor shape. Tracked in #1103 / #1104. |
+| `models/qwen4/bring-up.md` | New line (`qwen4_exp`, first checkpoint Qwen3.8-Flash-Next), not a Qwen3.5 variant; frozen at `de4b8e4d`. Landed: an identity probe and one typed config validation that pins structure rather than widths, so a smaller `qwen4_exp` checkpoint passes it, models the router's upstream default (`norm_topk_prob`) and derives the n-gram table from five config fields via 16 per-head primes, plus the PLE n-gram block golden and its two readers — a device-free gate (hash exact under i64 equality, block output bit-identical, four negative controls) whose hash half is production host code, and the block's first device kernels in `pegainfer-kernels` behind a new `qwen4` feature (204799 of the golden's 204800 stored outputs come back bit-for-bit, three device-side controls). Measured traps: untied LM head, **no final-norm tensor** (the readout is the hyper-connection mixer), `ple_layer_ids` is 1-indexed so the n-gram block is on `layers.1`, the n-gram table is **102.4 GB at BF16** (the quoted ~51 GB is fp8), `output_gate_type: "sigmoid"` is a **live** field selecting the GDN gated-norm activation (the inverse of the Qwen3.8-27B lesson), two RMSNorm conventions in one layer, and the PLE conv is dilated despite an identical tensor shape. Tracked in #1103 / #1104. |
 
 ## models / gemma4
 

pegainfer-kernels/src/ffi/qwen4.rs

@@ -31,3 +31,59 @@ unsafe extern "C" {
         stream: CUstream,
     ) -> CUresult;
 }
+
+unsafe extern "C" {
+    /// Row-major `[seq_len, width]` into branch-major
+    /// `[hc_count][seq_len][hidden]`, so the grouped norms run on contiguous
+    /// groups. `width` is the caller's row stride and is refused unless it is
+    /// the compiled one.
+    pub fn qwen4_ple_split_branches_cuda(
+        row_major: *const Half,
+        branch_major: *mut Half,
+        seq_len: i32,
+        width: i32,
+        hidden: i32,
+        hc_count: i32,
+        stream: CUstream,
+    ) -> CUresult;
+
+    /// One gate scalar per (row, branch) — the per-branch dot, its double
+    /// rounding around the `1 / sqrt_hidden` divide, the signed sqrt and the
+    /// sigmoid — times `value`, broadcast over the branches that share it.
+    ///
+    /// `key_normed`, `query_normed` and `gated` are branch-major; `value` is
+    /// `[seq_len, hidden]`. `gate_span` is `hidden` for the reference's
+    /// per-branch dot and `width` for the full-stream negative control;
+    /// `signed_sqrt` is 1 for the reference and 0 for the unsigned control.
+    pub fn qwen4_ple_gate_cuda(
+        key_normed: *const Half,
+        query_normed: *const Half,
+        value: *const Half,
+        gated: *mut Half,
+        seq_len: i32,
+        hidden: i32,
+        hc_count: i32,
+        gate_span: i32,
+        sqrt_hidden: f32,
+        signed_sqrt: i32,
+        stream: CUstream,
+    ) -> CUresult;
+
+    /// `out = gated + silu(bf16(dilated causal depthwise conv(conv_in)))`.
+    /// `conv_in` and `gated` are branch-major, `out` is row-major
+    /// `[seq_len, width]`, `conv_weight` is `[width, 1, conv_kernel]` flattened.
+    /// The zero initial state of `(conv_kernel - 1) * dilation` rows is a bound
+    /// inside the kernel, not a buffer.
+    pub fn qwen4_ple_conv_residual_cuda(
+        conv_in: *const Half,
+        gated: *const Half,
+        conv_weight: *const Half,
+        out: *mut Half,
+        seq_len: i32,
+        hidden: i32,
+        hc_count: i32,
+        conv_kernel: i32,
+        dilation: i32,
+        stream: CUstream,
+    ) -> CUresult;
+}

pegainfer-kernels/src/ops.rs

@@ -231,6 +231,14 @@ pub use norm::rms_norm_into;
 pub use norm::rms_norm_offset_into;
 pub use norm::rms_norm_rows_into;
 #[cfg(feature = "qwen4")]
+pub use qwen4::PleBlockGeometry;
+#[cfg(feature = "qwen4")]
+pub use qwen4::PleBlockVariant;
+#[cfg(feature = "qwen4")]
+pub use qwen4::ple_block_into;
+#[cfg(feature = "qwen4")]
+pub use qwen4::ple_block_variant_into;
+#[cfg(feature = "qwen4")]
 pub use qwen4::qsa_indexer_block_scores_into;
 #[cfg(feature = "qwen35")]
 pub use qwen35::GDN_AOT_KEY_HEAD_DIM;