From 70f784271dc3f9bfc6cca5f8b0969aa02bd2a1f9 Mon Sep 17 00:00:00 2001 From: BHCC2025 <241330250+BHCC2025@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:56:24 -0500 Subject: [PATCH 1/2] qwen3_5 cuda: dense EXL3 checkpoints (Qwen3.8-27B), with DFlash2 drafts The EXL3 reader admitted Qwen3.5-MoE checkpoints only. A dense checkpoint's MLP now loads as three Dense projections (gate, up, down) beside the attention, delta and head projections the reader already loads, and the grouped experts' scratch is sized only for MoE. A dense EXL3 checkpoint drafts as an MLX one does: the DFlash2 head already projects through Dense. MoE EXL3 checkpoints still refuse a drafter. No kernel or arithmetic changes. --- docs/recipes/qwen3.8-27b.md | 9 ++++---- zig/qwen35_formats_host.zig | 8 +++++-- .../qwen3_5/cuda_exl3_admission_test.zig | 23 ++++++++++++++++++- .../families/qwen3_5/cuda_exl3_weights.zig | 9 ++++++-- zig/src/families/qwen3_5/cuda_geometry.zig | 2 +- zig/src/families/qwen3_5/cuda_native.zig | 4 ++-- 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/docs/recipes/qwen3.8-27b.md b/docs/recipes/qwen3.8-27b.md index 4a4172baf2..da805186bc 100644 --- a/docs/recipes/qwen3.8-27b.md +++ b/docs/recipes/qwen3.8-27b.md @@ -4,7 +4,8 @@ The native engine serves `TensorFold/Qwen3.8-27B-MLX-4bit` on Metal, qualified o `zig/src/families/qwen27/`. It drafts with DFlash2 (`zig/src/families/qwen27/dflash/`) only when `--drafter` names the drafter; without it the model decodes plain. On CUDA the native engine serves the MLX affine checkpoints (2, 3, 4, 5, 6 and 8 bits in groups of 64) from `zig/src/families/qwen3_5/cuda_*.zig`, with DFlash2 drafts and -`--kv-cache fp8`; it reads no NVFP4 or EXL3 checkpoint yet. +`--kv-cache fp8`. It also reads ModelOpt NVFP4 and FP8 exports, and turboderp's EXL3 packs (`mul1` codebook) with +DFlash2 drafts through the shared EXL3 module, on one GPU. ```bash tensorfold pull TensorFold/Qwen3.8-27B-MLX-4bit @@ -98,9 +99,9 @@ quality comparisons are TBD [release-0.3.5]. ## CUDA -This section is the Python engine's (0.6.6). The native CUDA engine reads only MLX affine checkpoints so far: the -NVFP4 and EXL3 readers below have not been ported to it (1.1.0 carries the shared EXL3 module of #832, which the -27B does not read through yet). +This section is the Python engine's (0.6.6). The native CUDA engine reads the MLX affine checkpoints, ModelOpt +NVFP4 and FP8 exports, and turboderp's EXL3 packs through the shared EXL3 module of #832; its own measurements are +not the ones below. On CUDA the 27B serves NVFP4 and EXL3 checkpoints, and the MLX 4-bit checkpoint as the portable option: the same files a Mac serves, and the only format two ranks read. `tensorfold serve` loads the checkpoint you name; it picks diff --git a/zig/qwen35_formats_host.zig b/zig/qwen35_formats_host.zig index a70119e4a4..e72895961d 100644 --- a/zig/qwen35_formats_host.zig +++ b/zig/qwen35_formats_host.zig @@ -43,19 +43,23 @@ fn config(quant: []const u8, moe: bool, pocket: bool) ![]u8 { const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\"}"; const ct = "\"quantization_config\":{\"quant_method\":\"compressed-tensors\"}"; -test "format coexistence: EXL3 stays MoE only and ModelOpt keeps its admission and refusals" { +test "format coexistence: EXL3 admits MoE and dense and ModelOpt keeps its admission and refusals" { const x = try config(exl3, true, false); defer t.allocator.free(x); const gx = try geo.parse(t.allocator, x); try t.expect(gx.exl3); try t.expect(!gx.kinds.modelopt); + const xd = try config(exl3, false, false); + defer t.allocator.free(xd); + const gd = try geo.parse(t.allocator, xd); + try t.expect(gd.exl3 and !gd.moe() and !gd.kinds.modelopt); const m = try config(ct, false, false); defer t.allocator.free(m); const gm = try geo.parse(t.allocator, m); try t.expect(gm.kinds.modelopt and gm.kinds.ct and !gm.exl3); try t.expectEqual(@as(u8, 16), gm.bits.embed); for ([_]struct { q: []const u8, moe: bool, pocket: bool, err: anyerror }{ - .{ .q = exl3, .moe = false, .pocket = false, .err = error.UnsupportedQwenQuantization }, + .{ .q = exl3, .moe = false, .pocket = true, .err = error.UnsupportedQwenQuantization }, .{ .q = exl3, .moe = true, .pocket = true, .err = error.UnsupportedQwenQuantization }, .{ .q = ct, .moe = true, .pocket = false, .err = error.UnsupportedQwenQuantization }, .{ .q = ct, .moe = false, .pocket = true, .err = error.UnsupportedLearningFormat }, diff --git a/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig b/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig index c5dd01d4bd..23c48ff320 100644 --- a/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig +++ b/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig @@ -1,4 +1,4 @@ -//! An EXL3 Qwen3.5-MoE config is admitted by its quant_method alone; anything else it cannot serve is refused. +//! An EXL3 Qwen3.5 config, MoE or dense, is admitted by its quant_method alone; what it cannot serve is refused. const std = @import("std"); const parse = @import("cuda_geometry.zig").parse; const a = std.testing.allocator; @@ -32,3 +32,24 @@ test "a config with neither an MLX table nor quant_method exl3 is refused" { try std.testing.expectError(error.UnsupportedQwenQuantization, parse(a, c)); } } + +/// Qwen3.8-27B's text tower, four layers of it: a dense MLP, no experts. +const dense_text = + \\"text_config":{"model_type":"qwen3_5_text","attention_bias":false,"attn_output_gate":true,"hidden_act":"silu", + \\ "head_dim":256,"linear_key_head_dim":128,"linear_value_head_dim":128,"linear_conv_kernel_dim":4, + \\ "hidden_size":5120,"intermediate_size":17408,"num_hidden_layers":4,"vocab_size":248320, + \\ "linear_num_key_heads":16,"linear_num_value_heads":48,"num_attention_heads":24,"num_key_value_heads":4, + \\ "rms_norm_eps":1e-06,"max_position_embeddings":262144, + \\ "layer_types":["linear_attention","linear_attention","linear_attention","full_attention"], + \\ "rope_parameters":{"rope_type":"default","rope_theta":10000000,"partial_rotary_factor":0.25}} +; + +test "a dense EXL3 config (Qwen3.8-27B's layout) is admitted, with no experts" { + const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\",\"bits\":4.0},"; + const c = try std.fmt.allocPrint(a, "{{\"model_type\":\"qwen3_5\",{s}\"tie_word_embeddings\":false,{s}}}", .{ exl3, dense_text }); + defer a.free(c); + const g = try parse(a, c); + try std.testing.expect(g.exl3); + try std.testing.expect(!g.moe()); + try std.testing.expectEqual(@as(usize, 17408), g.intermediate); +} diff --git a/zig/src/families/qwen3_5/cuda_exl3_weights.zig b/zig/src/families/qwen3_5/cuda_exl3_weights.zig index 8c333de8b7..99dc9c8085 100644 --- a/zig/src/families/qwen3_5/cuda_exl3_weights.zig +++ b/zig/src/families/qwen3_5/cuda_exl3_weights.zig @@ -1,4 +1,4 @@ -//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5-MoE checkpoints on the CUDA engine, read under their HF tensor names. +//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5 checkpoints, MoE and dense, on the CUDA engine under their HF tensor names. const std = @import("std"); const cuda = @import("cuda"); const core = @import("core"); @@ -31,6 +31,7 @@ pub const Kit = struct { k.b16 = try b16.Kernels.load(d); errdefer k.b16.unload(); k.grouped = .{ .d = d, .kernels = &k.experts }; + if (!g.moe()) return k; // a dense checkpoint's MLP is three Dense projections // the grouped experts' scratch grows on first use: allocate it before any graph capture, for the widest call const pairs = engine.max_rows * g.slots(); const items = cuda.grouped.maxItems(pairs, g.experts + 1, 16); @@ -276,7 +277,11 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void { .k_norm = try norm1(s, ck, "{s}self_attn.k_norm.weight", .{p}, geometry.head_dim), } }; } - l.mlp = .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } }; + l.mlp = if (g.moe()) .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } } else .{ .dense = .{ + .gate = try dense(s, kit, ck, p, "mlp.gate_proj", g.intermediate, g.hidden), + .up = try dense(s, kit, ck, p, "mlp.up_proj", g.intermediate, g.hidden), + .down = try dense(s, kit, ck, p, "mlp.down_proj", g.hidden, g.intermediate), + } }; } } diff --git a/zig/src/families/qwen3_5/cuda_geometry.zig b/zig/src/families/qwen3_5/cuda_geometry.zig index 382bdc3f55..b61b113b16 100644 --- a/zig/src/families/qwen3_5/cuda_geometry.zig +++ b/zig/src/families/qwen3_5/cuda_geometry.zig @@ -325,7 +325,7 @@ pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Geometry { return g; } if (exl3) { - if (!moe or g.pocket != 0) return error.UnsupportedQwenQuantization; + if (g.pocket != 0) return error.UnsupportedQwenQuantization; // dense (Qwen3.8-27B) and MoE checkpoints alike return g; } g.bits = try widths(gpa, root, try object(qv.?), g); diff --git a/zig/src/families/qwen3_5/cuda_native.zig b/zig/src/families/qwen3_5/cuda_native.zig index dcfc7787f0..7d46f68b28 100644 --- a/zig/src/families/qwen3_5/cuda_native.zig +++ b/zig/src/families/qwen3_5/cuda_native.zig @@ -164,7 +164,7 @@ fn reserve(p: *anyopaque, streams: u32) anyerror!void { pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void, prepare: struct { ctx: *anyopaque, run: *const fn (*anyopaque) anyerror!lanes.drafter.Facts, bytes: usize } } { if (bits != 4) return error.DrafterBitsUnsupported; const l: *Lanes = @ptrCast(@alignCast(p)); - if (l.e.g.exl3) return error.Exl3DrafterUnsupported; + if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams; const d = try Draft.init(gpa, io, l.e, dir); errdefer d.deinit(); @@ -180,7 +180,7 @@ pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []con pub fn openMtp(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void } { if (bits != 4) return error.DrafterBitsUnsupported; const l: *Lanes = @ptrCast(@alignCast(p)); - if (l.e.g.exl3) return error.Exl3DrafterUnsupported; + if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams; const d = try Mtp.init(gpa, io, l.e, dir, l.capacity); errdefer d.deinit(); From b78c7af947212ba7843c731ce7a7d413a276af1e Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:36:31 +0000 Subject: [PATCH 2/2] changelog: dense EXL3 Qwen3.8-27B on CUDA with DFlash2 (#1465) --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff4c6f53b..6c168943b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel ## Unreleased +- Qwen3.8-27B on CUDA reads turboderp's dense EXL3 checkpoints (`mul1` codebook, such as `turboderp/Qwen3.8-27B-exl3`), which used to be refused at load, and drafts them with DFlash2 as it drafts the MLX checkpoint. The dense MLP runs on the same EXL3 `Dense` projections the MoE checkpoints use, with no kernel changes; MoE EXL3 checkpoints still refuse a drafter, and the MLX and MoE EXL3 paths are unchanged. On a GB10 at 4.00 bpw, drafted replies equal plain ones and decode runs at 45.8 tok/s for chat and 71.9 for code with DFlash2, against 15.3 and 15.5 without it; prompts run at 338 to 359 tok/s from 2k to 64k tokens (#1465). Thanks to @BHCC2025, and to @alexderz for the RTX 3090 run. + - HIP: on RDNA4 GPUs (gfx1201 and gfx1200) short routed prompts take the K-parallel tile further, bounded by the plan's items (now 507) and, new, its pairs per item (3.5). On an R9700, cold prefill on Qwen3.6-35B-A3B is 0.53 to 0.80 times main's time from 33 to 128 tokens, and on Nemotron 3.5 Lightning 0.82 to 0.97 times from 129 to 180 tokens; longer prompts and decode stay within 1% and replies are token-equal. Other GPUs' tables set no pairs bound, so their choices are unchanged; `hip-affine-tune` now also times Nemotron's and Flash Next's routed plans (#1607, fixes #1557). Thanks to @laudney. - CUDA: tree attention (head dims 128 and 256) and head dim 512 window decode and prompt rows gain kernels that read committed keys from a page table, so a family can keep its full-attention caches in the shared KV page pool (#1618; Gemma 4 and Kolibri-1 adopt them in follow-ups). Nothing launches them yet: the existing kernels' SASS is unchanged on sm_80, 86, 89, 120 and 121, and paged results are byte-equal to the contiguous cache over shuffled pages of 16, 37 and 256 positions. Thanks to @jschmied.