qwen3_5 cuda: dense EXL3 checkpoints (Qwen3.8-27B), with DFlash2 drafts

CHANGELOG.md

@@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
 
 ## Unreleased
 
+- Qwen3.8-27B on CUDA reads turboderp's dense EXL3 checkpoints (`mul1` codebook, such as `turboderp/Qwen3.8-27B-exl3`), which used to be refused at load, and drafts them with DFlash2 as it drafts the MLX checkpoint. The dense MLP runs on the same EXL3 `Dense` projections the MoE checkpoints use, with no kernel changes; MoE EXL3 checkpoints still refuse a drafter, and the MLX and MoE EXL3 paths are unchanged. On a GB10 at 4.00 bpw, drafted replies equal plain ones and decode runs at 45.8 tok/s for chat and 71.9 for code with DFlash2, against 15.3 and 15.5 without it; prompts run at 338 to 359 tok/s from 2k to 64k tokens (#1465). Thanks to @BHCC2025, and to @alexderz for the RTX 3090 run.
+
 - HIP: on RDNA4 GPUs (gfx1201 and gfx1200) short routed prompts take the K-parallel tile further, bounded by the plan's items (now 507) and, new, its pairs per item (3.5). On an R9700, cold prefill on Qwen3.6-35B-A3B is 0.53 to 0.80 times main's time from 33 to 128 tokens, and on Nemotron 3.5 Lightning 0.82 to 0.97 times from 129 to 180 tokens; longer prompts and decode stay within 1% and replies are token-equal. Other GPUs' tables set no pairs bound, so their choices are unchanged; `hip-affine-tune` now also times Nemotron's and Flash Next's routed plans (#1607, fixes #1557). Thanks to @laudney.
 
 - CUDA: tree attention (head dims 128 and 256) and head dim 512 window decode and prompt rows gain kernels that read committed keys from a page table, so a family can keep its full-attention caches in the shared KV page pool (#1618; Gemma 4 and Kolibri-1 adopt them in follow-ups). Nothing launches them yet: the existing kernels' SASS is unchanged on sm_80, 86, 89, 120 and 121, and paged results are byte-equal to the contiguous cache over shuffled pages of 16, 37 and 256 positions. Thanks to @jschmied.

docs/recipes/qwen3.8-27b.md

@@ -4,7 +4,8 @@ The native engine serves `TensorFold/Qwen3.8-27B-MLX-4bit` on Metal, qualified o
 `zig/src/families/qwen27/`. It drafts with DFlash2 (`zig/src/families/qwen27/dflash/`) only when `--drafter` names
 the drafter; without it the model decodes plain. On CUDA the native engine serves the MLX affine checkpoints (2, 3,
 4, 5, 6 and 8 bits in groups of 64) from `zig/src/families/qwen3_5/cuda_*.zig`, with DFlash2 drafts and
-`--kv-cache fp8`; it reads no NVFP4 or EXL3 checkpoint yet.
+`--kv-cache fp8`. It also reads ModelOpt NVFP4 and FP8 exports, and turboderp's EXL3 packs (`mul1` codebook) with
+DFlash2 drafts through the shared EXL3 module, on one GPU.
 
 ```bash
 tensorfold pull TensorFold/Qwen3.8-27B-MLX-4bit
@@ -98,9 +99,9 @@ quality comparisons are TBD [release-0.3.5].
 
 ## CUDA
 
-This section is the Python engine's (0.6.6). The native CUDA engine reads only MLX affine checkpoints so far: the
-NVFP4 and EXL3 readers below have not been ported to it (1.1.0 carries the shared EXL3 module of #832, which the
-27B does not read through yet).
+This section is the Python engine's (0.6.6). The native CUDA engine reads the MLX affine checkpoints, ModelOpt
+NVFP4 and FP8 exports, and turboderp's EXL3 packs through the shared EXL3 module of #832; its own measurements are
+not the ones below.
 
 On CUDA the 27B serves NVFP4 and EXL3 checkpoints, and the MLX 4-bit checkpoint as the portable option: the same
 files a Mac serves, and the only format two ranks read. `tensorfold serve` loads the checkpoint you name; it picks

zig/qwen35_formats_host.zig

@@ -43,19 +43,23 @@ fn config(quant: []const u8, moe: bool, pocket: bool) ![]u8 {
 const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\"}";
 const ct = "\"quantization_config\":{\"quant_method\":\"compressed-tensors\"}";
 
-test "format coexistence: EXL3 stays MoE only and ModelOpt keeps its admission and refusals" {
+test "format coexistence: EXL3 admits MoE and dense and ModelOpt keeps its admission and refusals" {
     const x = try config(exl3, true, false);
     defer t.allocator.free(x);
     const gx = try geo.parse(t.allocator, x);
     try t.expect(gx.exl3);
     try t.expect(!gx.kinds.modelopt);
+    const xd = try config(exl3, false, false);
+    defer t.allocator.free(xd);
+    const gd = try geo.parse(t.allocator, xd);
+    try t.expect(gd.exl3 and !gd.moe() and !gd.kinds.modelopt);
     const m = try config(ct, false, false);
     defer t.allocator.free(m);
     const gm = try geo.parse(t.allocator, m);
     try t.expect(gm.kinds.modelopt and gm.kinds.ct and !gm.exl3);
     try t.expectEqual(@as(u8, 16), gm.bits.embed);
     for ([_]struct { q: []const u8, moe: bool, pocket: bool, err: anyerror }{
-        .{ .q = exl3, .moe = false, .pocket = false, .err = error.UnsupportedQwenQuantization },
+        .{ .q = exl3, .moe = false, .pocket = true, .err = error.UnsupportedQwenQuantization },
         .{ .q = exl3, .moe = true, .pocket = true, .err = error.UnsupportedQwenQuantization },
         .{ .q = ct, .moe = true, .pocket = false, .err = error.UnsupportedQwenQuantization },
         .{ .q = ct, .moe = false, .pocket = true, .err = error.UnsupportedLearningFormat },

zig/src/families/qwen3_5/cuda_exl3_admission_test.zig

@@ -1,4 +1,4 @@
-//! An EXL3 Qwen3.5-MoE config is admitted by its quant_method alone; anything else it cannot serve is refused.
+//! An EXL3 Qwen3.5 config, MoE or dense, is admitted by its quant_method alone; what it cannot serve is refused.
 const std = @import("std");
 const parse = @import("cuda_geometry.zig").parse;
 const a = std.testing.allocator;
@@ -32,3 +32,24 @@ test "a config with neither an MLX table nor quant_method exl3 is refused" {
         try std.testing.expectError(error.UnsupportedQwenQuantization, parse(a, c));
     }
 }
+
+/// Qwen3.8-27B's text tower, four layers of it: a dense MLP, no experts.
+const dense_text =
+    \\"text_config":{"model_type":"qwen3_5_text","attention_bias":false,"attn_output_gate":true,"hidden_act":"silu",
+    \\ "head_dim":256,"linear_key_head_dim":128,"linear_value_head_dim":128,"linear_conv_kernel_dim":4,
+    \\ "hidden_size":5120,"intermediate_size":17408,"num_hidden_layers":4,"vocab_size":248320,
+    \\ "linear_num_key_heads":16,"linear_num_value_heads":48,"num_attention_heads":24,"num_key_value_heads":4,
+    \\ "rms_norm_eps":1e-06,"max_position_embeddings":262144,
+    \\ "layer_types":["linear_attention","linear_attention","linear_attention","full_attention"],
+    \\ "rope_parameters":{"rope_type":"default","rope_theta":10000000,"partial_rotary_factor":0.25}}
+;
+
+test "a dense EXL3 config (Qwen3.8-27B's layout) is admitted, with no experts" {
+    const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\",\"bits\":4.0},";
+    const c = try std.fmt.allocPrint(a, "{{\"model_type\":\"qwen3_5\",{s}\"tie_word_embeddings\":false,{s}}}", .{ exl3, dense_text });
+    defer a.free(c);
+    const g = try parse(a, c);
+    try std.testing.expect(g.exl3);
+    try std.testing.expect(!g.moe());
+    try std.testing.expectEqual(@as(usize, 17408), g.intermediate);
+}

zig/src/families/qwen3_5/cuda_exl3_weights.zig

@@ -1,4 +1,4 @@
-//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5-MoE checkpoints on the CUDA engine, read under their HF tensor names.
+//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5 checkpoints, MoE and dense, on the CUDA engine under their HF tensor names.
 const std = @import("std");
 const cuda = @import("cuda");
 const core = @import("core");
@@ -31,6 +31,7 @@ pub const Kit = struct {
         k.b16 = try b16.Kernels.load(d);
         errdefer k.b16.unload();
         k.grouped = .{ .d = d, .kernels = &k.experts };
+        if (!g.moe()) return k; // a dense checkpoint's MLP is three Dense projections
         // the grouped experts' scratch grows on first use: allocate it before any graph capture, for the widest call
         const pairs = engine.max_rows * g.slots();
         const items = cuda.grouped.maxItems(pairs, g.experts + 1, 16);
@@ -276,7 +277,11 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void {
                 .k_norm = try norm1(s, ck, "{s}self_attn.k_norm.weight", .{p}, geometry.head_dim),
             } };
         }
-        l.mlp = .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } };
+        l.mlp = if (g.moe()) .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } } else .{ .dense = .{
+            .gate = try dense(s, kit, ck, p, "mlp.gate_proj", g.intermediate, g.hidden),
+            .up = try dense(s, kit, ck, p, "mlp.up_proj", g.intermediate, g.hidden),
+            .down = try dense(s, kit, ck, p, "mlp.down_proj", g.hidden, g.intermediate),
+        } };
     }
 }
 

zig/src/families/qwen3_5/cuda_geometry.zig

@@ -325,7 +325,7 @@ pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Geometry {
         return g;
     }
     if (exl3) {
-        if (!moe or g.pocket != 0) return error.UnsupportedQwenQuantization;
+        if (g.pocket != 0) return error.UnsupportedQwenQuantization; // dense (Qwen3.8-27B) and MoE checkpoints alike
         return g;
     }
     g.bits = try widths(gpa, root, try object(qv.?), g);

zig/src/families/qwen3_5/cuda_native.zig

@@ -164,7 +164,7 @@ fn reserve(p: *anyopaque, streams: u32) anyerror!void {
 pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void, prepare: struct { ctx: *anyopaque, run: *const fn (*anyopaque) anyerror!lanes.drafter.Facts, bytes: usize } } {
     if (bits != 4) return error.DrafterBitsUnsupported;
     const l: *Lanes = @ptrCast(@alignCast(p));
-    if (l.e.g.exl3) return error.Exl3DrafterUnsupported;
+    if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one
     if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams;
     const d = try Draft.init(gpa, io, l.e, dir);
     errdefer d.deinit();
@@ -180,7 +180,7 @@ pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []con
 pub fn openMtp(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void } {
     if (bits != 4) return error.DrafterBitsUnsupported;
     const l: *Lanes = @ptrCast(@alignCast(p));
-    if (l.e.g.exl3) return error.Exl3DrafterUnsupported;
+    if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one
     if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams;
     const d = try Mtp.init(gpa, io, l.e, dir, l.capacity);
     errdefer d.deinit();