diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff4c6f53b..6c168943b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel ## Unreleased +- Qwen3.8-27B on CUDA reads turboderp's dense EXL3 checkpoints (`mul1` codebook, such as `turboderp/Qwen3.8-27B-exl3`), which used to be refused at load, and drafts them with DFlash2 as it drafts the MLX checkpoint. The dense MLP runs on the same EXL3 `Dense` projections the MoE checkpoints use, with no kernel changes; MoE EXL3 checkpoints still refuse a drafter, and the MLX and MoE EXL3 paths are unchanged. On a GB10 at 4.00 bpw, drafted replies equal plain ones and decode runs at 45.8 tok/s for chat and 71.9 for code with DFlash2, against 15.3 and 15.5 without it; prompts run at 338 to 359 tok/s from 2k to 64k tokens (#1465). Thanks to @BHCC2025, and to @alexderz for the RTX 3090 run. + - HIP: on RDNA4 GPUs (gfx1201 and gfx1200) short routed prompts take the K-parallel tile further, bounded by the plan's items (now 507) and, new, its pairs per item (3.5). On an R9700, cold prefill on Qwen3.6-35B-A3B is 0.53 to 0.80 times main's time from 33 to 128 tokens, and on Nemotron 3.5 Lightning 0.82 to 0.97 times from 129 to 180 tokens; longer prompts and decode stay within 1% and replies are token-equal. Other GPUs' tables set no pairs bound, so their choices are unchanged; `hip-affine-tune` now also times Nemotron's and Flash Next's routed plans (#1607, fixes #1557). Thanks to @laudney. - CUDA: tree attention (head dims 128 and 256) and head dim 512 window decode and prompt rows gain kernels that read committed keys from a page table, so a family can keep its full-attention caches in the shared KV page pool (#1618; Gemma 4 and Kolibri-1 adopt them in follow-ups). Nothing launches them yet: the existing kernels' SASS is unchanged on sm_80, 86, 89, 120 and 121, and paged results are byte-equal to the contiguous cache over shuffled pages of 16, 37 and 256 positions. Thanks to @jschmied. diff --git a/docs/recipes/qwen3.8-27b.md b/docs/recipes/qwen3.8-27b.md index 4a4172baf2..da805186bc 100644 --- a/docs/recipes/qwen3.8-27b.md +++ b/docs/recipes/qwen3.8-27b.md @@ -4,7 +4,8 @@ The native engine serves `TensorFold/Qwen3.8-27B-MLX-4bit` on Metal, qualified o `zig/src/families/qwen27/`. It drafts with DFlash2 (`zig/src/families/qwen27/dflash/`) only when `--drafter` names the drafter; without it the model decodes plain. On CUDA the native engine serves the MLX affine checkpoints (2, 3, 4, 5, 6 and 8 bits in groups of 64) from `zig/src/families/qwen3_5/cuda_*.zig`, with DFlash2 drafts and -`--kv-cache fp8`; it reads no NVFP4 or EXL3 checkpoint yet. +`--kv-cache fp8`. It also reads ModelOpt NVFP4 and FP8 exports, and turboderp's EXL3 packs (`mul1` codebook) with +DFlash2 drafts through the shared EXL3 module, on one GPU. ```bash tensorfold pull TensorFold/Qwen3.8-27B-MLX-4bit @@ -98,9 +99,9 @@ quality comparisons are TBD [release-0.3.5]. ## CUDA -This section is the Python engine's (0.6.6). The native CUDA engine reads only MLX affine checkpoints so far: the -NVFP4 and EXL3 readers below have not been ported to it (1.1.0 carries the shared EXL3 module of #832, which the -27B does not read through yet). +This section is the Python engine's (0.6.6). The native CUDA engine reads the MLX affine checkpoints, ModelOpt +NVFP4 and FP8 exports, and turboderp's EXL3 packs through the shared EXL3 module of #832; its own measurements are +not the ones below. On CUDA the 27B serves NVFP4 and EXL3 checkpoints, and the MLX 4-bit checkpoint as the portable option: the same files a Mac serves, and the only format two ranks read. `tensorfold serve` loads the checkpoint you name; it picks diff --git a/zig/qwen35_formats_host.zig b/zig/qwen35_formats_host.zig index a70119e4a4..e72895961d 100644 --- a/zig/qwen35_formats_host.zig +++ b/zig/qwen35_formats_host.zig @@ -43,19 +43,23 @@ fn config(quant: []const u8, moe: bool, pocket: bool) ![]u8 { const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\"}"; const ct = "\"quantization_config\":{\"quant_method\":\"compressed-tensors\"}"; -test "format coexistence: EXL3 stays MoE only and ModelOpt keeps its admission and refusals" { +test "format coexistence: EXL3 admits MoE and dense and ModelOpt keeps its admission and refusals" { const x = try config(exl3, true, false); defer t.allocator.free(x); const gx = try geo.parse(t.allocator, x); try t.expect(gx.exl3); try t.expect(!gx.kinds.modelopt); + const xd = try config(exl3, false, false); + defer t.allocator.free(xd); + const gd = try geo.parse(t.allocator, xd); + try t.expect(gd.exl3 and !gd.moe() and !gd.kinds.modelopt); const m = try config(ct, false, false); defer t.allocator.free(m); const gm = try geo.parse(t.allocator, m); try t.expect(gm.kinds.modelopt and gm.kinds.ct and !gm.exl3); try t.expectEqual(@as(u8, 16), gm.bits.embed); for ([_]struct { q: []const u8, moe: bool, pocket: bool, err: anyerror }{ - .{ .q = exl3, .moe = false, .pocket = false, .err = error.UnsupportedQwenQuantization }, + .{ .q = exl3, .moe = false, .pocket = true, .err = error.UnsupportedQwenQuantization }, .{ .q = exl3, .moe = true, .pocket = true, .err = error.UnsupportedQwenQuantization }, .{ .q = ct, .moe = true, .pocket = false, .err = error.UnsupportedQwenQuantization }, .{ .q = ct, .moe = false, .pocket = true, .err = error.UnsupportedLearningFormat }, diff --git a/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig b/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig index c5dd01d4bd..23c48ff320 100644 --- a/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig +++ b/zig/src/families/qwen3_5/cuda_exl3_admission_test.zig @@ -1,4 +1,4 @@ -//! An EXL3 Qwen3.5-MoE config is admitted by its quant_method alone; anything else it cannot serve is refused. +//! An EXL3 Qwen3.5 config, MoE or dense, is admitted by its quant_method alone; what it cannot serve is refused. const std = @import("std"); const parse = @import("cuda_geometry.zig").parse; const a = std.testing.allocator; @@ -32,3 +32,24 @@ test "a config with neither an MLX table nor quant_method exl3 is refused" { try std.testing.expectError(error.UnsupportedQwenQuantization, parse(a, c)); } } + +/// Qwen3.8-27B's text tower, four layers of it: a dense MLP, no experts. +const dense_text = + \\"text_config":{"model_type":"qwen3_5_text","attention_bias":false,"attn_output_gate":true,"hidden_act":"silu", + \\ "head_dim":256,"linear_key_head_dim":128,"linear_value_head_dim":128,"linear_conv_kernel_dim":4, + \\ "hidden_size":5120,"intermediate_size":17408,"num_hidden_layers":4,"vocab_size":248320, + \\ "linear_num_key_heads":16,"linear_num_value_heads":48,"num_attention_heads":24,"num_key_value_heads":4, + \\ "rms_norm_eps":1e-06,"max_position_embeddings":262144, + \\ "layer_types":["linear_attention","linear_attention","linear_attention","full_attention"], + \\ "rope_parameters":{"rope_type":"default","rope_theta":10000000,"partial_rotary_factor":0.25}} +; + +test "a dense EXL3 config (Qwen3.8-27B's layout) is admitted, with no experts" { + const exl3 = "\"quantization_config\":{\"quant_method\":\"exl3\",\"bits\":4.0},"; + const c = try std.fmt.allocPrint(a, "{{\"model_type\":\"qwen3_5\",{s}\"tie_word_embeddings\":false,{s}}}", .{ exl3, dense_text }); + defer a.free(c); + const g = try parse(a, c); + try std.testing.expect(g.exl3); + try std.testing.expect(!g.moe()); + try std.testing.expectEqual(@as(usize, 17408), g.intermediate); +} diff --git a/zig/src/families/qwen3_5/cuda_exl3_weights.zig b/zig/src/families/qwen3_5/cuda_exl3_weights.zig index 8c333de8b7..99dc9c8085 100644 --- a/zig/src/families/qwen3_5/cuda_exl3_weights.zig +++ b/zig/src/families/qwen3_5/cuda_exl3_weights.zig @@ -1,4 +1,4 @@ -//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5-MoE checkpoints on the CUDA engine, read under their HF tensor names. +//! ExLlamaV3 (EXL3, mul1 codebook) Qwen3.5 checkpoints, MoE and dense, on the CUDA engine under their HF tensor names. const std = @import("std"); const cuda = @import("cuda"); const core = @import("core"); @@ -31,6 +31,7 @@ pub const Kit = struct { k.b16 = try b16.Kernels.load(d); errdefer k.b16.unload(); k.grouped = .{ .d = d, .kernels = &k.experts }; + if (!g.moe()) return k; // a dense checkpoint's MLP is three Dense projections // the grouped experts' scratch grows on first use: allocate it before any graph capture, for the widest call const pairs = engine.max_rows * g.slots(); const items = cuda.grouped.maxItems(pairs, g.experts + 1, 16); @@ -276,7 +277,11 @@ fn all(s: *Sink, ck: *core.Checkpoint) !void { .k_norm = try norm1(s, ck, "{s}self_attn.k_norm.weight", .{p}, geometry.head_dim), } }; } - l.mlp = .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } }; + l.mlp = if (g.moe()) .{ .moe = .{ .router = try router(s, ck, p), .experts = try experts(s, ck, p) } } else .{ .dense = .{ + .gate = try dense(s, kit, ck, p, "mlp.gate_proj", g.intermediate, g.hidden), + .up = try dense(s, kit, ck, p, "mlp.up_proj", g.intermediate, g.hidden), + .down = try dense(s, kit, ck, p, "mlp.down_proj", g.hidden, g.intermediate), + } }; } } diff --git a/zig/src/families/qwen3_5/cuda_geometry.zig b/zig/src/families/qwen3_5/cuda_geometry.zig index 382bdc3f55..b61b113b16 100644 --- a/zig/src/families/qwen3_5/cuda_geometry.zig +++ b/zig/src/families/qwen3_5/cuda_geometry.zig @@ -325,7 +325,7 @@ pub fn parse(gpa: std.mem.Allocator, bytes: []const u8) !Geometry { return g; } if (exl3) { - if (!moe or g.pocket != 0) return error.UnsupportedQwenQuantization; + if (g.pocket != 0) return error.UnsupportedQwenQuantization; // dense (Qwen3.8-27B) and MoE checkpoints alike return g; } g.bits = try widths(gpa, root, try object(qv.?), g); diff --git a/zig/src/families/qwen3_5/cuda_native.zig b/zig/src/families/qwen3_5/cuda_native.zig index dcfc7787f0..7d46f68b28 100644 --- a/zig/src/families/qwen3_5/cuda_native.zig +++ b/zig/src/families/qwen3_5/cuda_native.zig @@ -164,7 +164,7 @@ fn reserve(p: *anyopaque, streams: u32) anyerror!void { pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void, prepare: struct { ctx: *anyopaque, run: *const fn (*anyopaque) anyerror!lanes.drafter.Facts, bytes: usize } } { if (bits != 4) return error.DrafterBitsUnsupported; const l: *Lanes = @ptrCast(@alignCast(p)); - if (l.e.g.exl3) return error.Exl3DrafterUnsupported; + if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams; const d = try Draft.init(gpa, io, l.e, dir); errdefer d.deinit(); @@ -180,7 +180,7 @@ pub fn openDrafter(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []con pub fn openMtp(gpa: std.mem.Allocator, io: std.Io, p: *anyopaque, dir: []const u8, bits: u8) !struct { drafter: lanes.drafter.Drafter, ctx: *anyopaque, close: *const fn (*anyopaque) void, reserve: *const fn (*anyopaque, u32) anyerror!void } { if (bits != 4) return error.DrafterBitsUnsupported; const l: *Lanes = @ptrCast(@alignCast(p)); - if (l.e.g.exl3) return error.Exl3DrafterUnsupported; + if (l.e.g.exl3 and l.e.g.moe()) return error.Exl3DrafterUnsupported; // a dense checkpoint drafts as an MLX one if (l.caches.count() != 0 or l.pool.items.len != 0) return error.TapsAfterStreams; const d = try Mtp.init(gpa, io, l.e, dir, l.capacity); errdefer d.deinit();