From 3faba0321f6f786ae4f7618491ccacb17c6faa23 Mon Sep 17 00:00:00 2001 From: gilby <880013+gilby@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:55:22 +0100 Subject: [PATCH 1/6] qwen3_5 cuda: the server sends message starts, so a later turn resumes where the last one's history ended --- zig/src/families/qwen3_5/cuda_native.zig | 3 +++ zig/src/families/qwen3_5/cuda_native_moe.zig | 2 ++ zig/src/native/cuda.zig | 2 +- zig/src/native/cuda_snaps.zig | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/zig/src/families/qwen3_5/cuda_native.zig b/zig/src/families/qwen3_5/cuda_native.zig index 1e77ed4943..1dc8342ecb 100644 --- a/zig/src/families/qwen3_5/cuda_native.zig +++ b/zig/src/families/qwen3_5/cuda_native.zig @@ -49,6 +49,9 @@ pub const snap_grid: u32 = E.max_rows; /// --learn-conversations: the kept states' disk image, the probe their identity hashes, the facts they depend on. pub const kept_disk = @import("cuda_disk.zig"); +/// cuda_prefill cuts a cold pass at the request's chunk starts too, so the server may send its message starts. +pub const request_starts = true; + /// explain() words open refusals too (--vision on a checkpoint without a tower it reads). pub const explains_open = true; diff --git a/zig/src/families/qwen3_5/cuda_native_moe.zig b/zig/src/families/qwen3_5/cuda_native_moe.zig index e9770197d4..a1e39959ef 100644 --- a/zig/src/families/qwen3_5/cuda_native_moe.zig +++ b/zig/src/families/qwen3_5/cuda_native_moe.zig @@ -27,6 +27,8 @@ pub const snap_host = native.snap_host; pub const snap_lookahead = native.snap_lookahead; pub const snap_grid = native.snap_grid; pub const kept_disk = native.kept_disk; + +pub const request_starts = native.request_starts; pub const explains_open = native.explains_open; pub const vision = native.vision; diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index c03b30d3fc..1eee57bab1 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -432,7 +432,6 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O std.log.info("recording each stream's rows under {s}/{s}", .{ dir, r.recorder.name }); } errdefer if (h.recording) |r| r.free(gpa); - // the family cuts its own prompt grid from position 0, as `tensorfold run` does: prefill_step 0 const admit: ?api.Admit = if (comptime !@hasDecl(F, "fits")) null else if (paged != null) .{ .ctx = loaded.ctx, .fits = F.fits } else null; h.kv = kv_pages.watchOf(F, loaded, paged); h.kv_count = if (admit != null) kv_pages.ledgerPool(F, loaded) else null; @@ -443,6 +442,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O .lanes = streams, .draft_sources = .{ .drafter = served.source }, .context_window = @intCast(window), + .prefill_step = snaps.requestStep(F), .startup = h.startup, .policy = if (@hasField(@TypeOf(loaded), "policy")) loaded.policy else "", .prompt_cache = h.store != null, diff --git a/zig/src/native/cuda_snaps.zig b/zig/src/native/cuda_snaps.zig index 5faaf2b5c9..1b6f4baf16 100644 --- a/zig/src/native/cuda_snaps.zig +++ b/zig/src/native/cuda_snaps.zig @@ -25,6 +25,11 @@ pub fn passRules(comptime F: type, prompt_rows: u32) api.prompt_cache.Rules { return rules; } +/// The server's chunk step: a family whose cold passes cut at request starts gets message starts on its grid (0: none). +pub fn requestStep(comptime F: type) u32 { + return if (@hasDecl(F, "request_starts") and F.request_starts) F.snap_grid else 0; +} + /// The cache's bytes less what the family sets aside for its states (`snap_ready`, as Nemotron's spare buffers). pub fn cacheBudget(comptime F: type, target: anytype, cache: u64) !u64 { return if (@hasDecl(F, "snap_ready")) cache - try F.snap_ready(target, cache) else cache; @@ -108,3 +113,16 @@ test "a family with snap_grid keeps on its chunk starts, any other at any row as test { _ = @import("cuda_snaps_test.zig"); } + +test "only a family whose passes cut at request starts asks the server for them" { + const Glm = struct { + pub const snap_grid: u32 = 2048; + }; + const Qwen = struct { + pub const snap_grid: u32 = 4096; + pub const request_starts = true; + }; + try std.testing.expectEqual(@as(u32, 0), requestStep(Glm)); + try std.testing.expectEqual(@as(u32, 0), requestStep(struct {})); + try std.testing.expectEqual(@as(u32, 4096), requestStep(Qwen)); +} From 4336c339eaf4ea63971782a257ba291b1faa9399 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:11:31 +0000 Subject: [PATCH 2/6] Gate CUDA message starts behind an explicit opt-in --- zig/build/cuda_family_host.zig | 1 + zig/src/core/profile.zig | 2 ++ zig/src/native/cuda_snaps.zig | 41 +++++++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/zig/build/cuda_family_host.zig b/zig/build/cuda_family_host.zig index ae207f06b6..756a0d7838 100644 --- a/zig/build/cuda_family_host.zig +++ b/zig/build/cuda_family_host.zig @@ -165,6 +165,7 @@ pub fn steps(b: *std.Build, cuda: *std.Build.Module, mods: anytype, step: *std.B pub fn registerCopyFamilies(b: *std.Build, native: *std.Build.Module, step: *std.Build.Step) void { b.step("test-glm-formats-host", "CUDA GLM EXL3 offer reaches the published family registry without a GPU").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"GLM capabilities:"} })).step); + b.step("test-request-starts-host", "Opt-in CUDA message starts without GPU execution").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"request starts:"} })).step); const family_copies = b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"CUDA lane copies:"} })); b.step("test-family-copies-host", "Actual CUDA family copy policies through the lane core").dependOn(&family_copies.step); b.step("test-glm-copies-host", "GLM copy facts, confidence and costs through the production lane core").dependOn(&family_copies.step); diff --git a/zig/src/core/profile.zig b/zig/src/core/profile.zig index cb8f82b02c..4805c93209 100644 --- a/zig/src/core/profile.zig +++ b/zig/src/core/profile.zig @@ -7,6 +7,7 @@ pub const Switch = enum { TF_QWEN_KV_PAGES, TF_QWEN_KV_PAGES_ALL, TF_QWEN_KV_SHARE, + TF_QWEN_REQUEST_STARTS, TF_KV_SHARE, TF_FLASHNEXT_KV_PAGES, TF_FLASHNEXT_KV_PAGES_ALL, @@ -96,6 +97,7 @@ const slide_cuda: []const []const u8 = &.{ "nemotron_h", "nemotron_h_omni", "qwe const flash: []const []const u8 = &.{"qwen4_exp"}; pub const entries = [_]Entry{ + .{ .id = .TF_QWEN_REQUEST_STARTS, .families = qwen, .class = .arithmetic, .syntax = .one, .effect = "Opt into CUDA message-start prompt cuts", .evidence = "#1334; default-off, device qualification pending" }, .{ .id = .TF_QWEN_KV_PAGES, .families = qwen, .class = .identical, .effect = "KV pool page size; memory and workload tradeoffs", .evidence = "#1357" }, .{ .id = .TF_QWEN_KV_PAGES_ALL, .families = qwen, .class = .debug, .syntax = .one, .effect = "Put the engine's own stream on pages too", .evidence = "#1357" }, .{ .id = .TF_QWEN_KV_SHARE, .families = qwen, .class = .identical, .effect = "Keep prompt pages and bounded recurrent snapshots", .evidence = "#1357" }, diff --git a/zig/src/native/cuda_snaps.zig b/zig/src/native/cuda_snaps.zig index 1b6f4baf16..3bd1298373 100644 --- a/zig/src/native/cuda_snaps.zig +++ b/zig/src/native/cuda_snaps.zig @@ -27,6 +27,13 @@ pub fn passRules(comptime F: type, prompt_rows: u32) api.prompt_cache.Rules { /// The server's chunk step: a family whose cold passes cut at request starts gets message starts on its grid (0: none). pub fn requestStep(comptime F: type) u32 { + return requestStepFor(F, @import("profile").get(.TF_QWEN_REQUEST_STARTS)); +} + +/// Only exact 1 opts in; a family must also declare that its cold passes honor request starts. +pub fn requestStepFor(comptime F: type, value: ?[]const u8) u32 { + const v = value orelse return 0; + if (!std.mem.eql(u8, v, "1")) return 0; return if (@hasDecl(F, "request_starts") and F.request_starts) F.snap_grid else 0; } @@ -122,7 +129,35 @@ test "only a family whose passes cut at request starts asks the server for them" pub const snap_grid: u32 = 4096; pub const request_starts = true; }; - try std.testing.expectEqual(@as(u32, 0), requestStep(Glm)); - try std.testing.expectEqual(@as(u32, 0), requestStep(struct {})); - try std.testing.expectEqual(@as(u32, 4096), requestStep(Qwen)); + try std.testing.expectEqual(@as(u32, 0), requestStepFor(Glm, "1")); + try std.testing.expectEqual(@as(u32, 0), requestStepFor(struct {}, "1")); + try std.testing.expectEqual(@as(u32, 4096), requestStepFor(Qwen, "1")); +} + +test "request starts: unset keeps CUDA's main prefill contract" { + const Qwen = struct { + pub const snap_grid: u32 = 4096; + pub const request_starts = true; + }; + try std.testing.expectEqual(@as(u32, 0), requestStep(Qwen)); +} + +test "request starts: opt in is explicit, eligible and independent of cache rules" { + const Qwen = struct { + pub const snap_grid: u32 = 4096; + pub const request_starts = true; + }; + const Disabled = struct { + pub const snap_grid: u32 = 4096; + pub const request_starts = false; + }; + for ([_]?[]const u8{ null, "", "0", "off", "on", "true", "2" }) |v| { + try std.testing.expectEqual(@as(u32, 0), requestStepFor(Qwen, v)); + } + try std.testing.expectEqual(@as(u32, 4096), requestStepFor(Qwen, "1")); + try std.testing.expectEqual(@as(u32, 0), requestStepFor(Disabled, "1")); + const before = cacheRules(Qwen); + _ = requestStepFor(Qwen, "1"); + try std.testing.expectEqualDeep(before, cacheRules(Qwen)); + try std.testing.expectEqual(@import("profile").Class.arithmetic, @import("profile").entry(.TF_QWEN_REQUEST_STARTS).class); } From d50bea872b7807aab10cdaf00565f3decf22a784 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:46:19 +0000 Subject: [PATCH 3/6] Refuse CUDA message-start cuts with shared KV at startup --- zig/build/cuda_family_host.zig | 1 + zig/src/families/qwen3_5/cuda.zig | 4 ++ zig/src/families/qwen3_5/cuda_native.zig | 1 + zig/src/families/qwen3_5/cuda_prefix.zig | 6 +- .../qwen3_5/cuda_request_starts_test.zig | 69 +++++++++++++++++++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 zig/src/families/qwen3_5/cuda_request_starts_test.zig diff --git a/zig/build/cuda_family_host.zig b/zig/build/cuda_family_host.zig index 756a0d7838..be3eafe259 100644 --- a/zig/build/cuda_family_host.zig +++ b/zig/build/cuda_family_host.zig @@ -10,6 +10,7 @@ pub fn steps(b: *std.Build, cuda: *std.Build.Module, mods: anytype, step: *std.B b.step("test-cuda-persistence-source-host", "Captured CUDA source keys without loading a driver").dependOn(&captured.step); step.dependOn(&captured.step); + b.step("test-qwen35-request-starts-host", "Request-start and shared-KV startup refusal without devices").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = mods.qwen35, .filters = &.{"request starts:"} })).step); @import("minimax_cuda.zig").hostTests(b, host, cuda, mods.core, mods.lanes, step); @import("cuda_groups_host.zig").host(b, cuda, mods, step); const prefill = b.addRunArtifact(b.addTest(.{ .root_module = mods.qwen35, .filters = &.{"Qwen stepped prefill:"} })); diff --git a/zig/src/families/qwen3_5/cuda.zig b/zig/src/families/qwen3_5/cuda.zig index 60e6527cab..e84d68b6a5 100644 --- a/zig/src/families/qwen3_5/cuda.zig +++ b/zig/src/families/qwen3_5/cuda.zig @@ -77,3 +77,7 @@ test { test "Qwen pages default: the learner's and the prefix share's refusals stop before the weights load" { try @import("cuda_admit_test.zig").refusals(); } + +test { + _ = @import("cuda_request_starts_test.zig"); +} diff --git a/zig/src/families/qwen3_5/cuda_native.zig b/zig/src/families/qwen3_5/cuda_native.zig index 1dc8342ecb..0b6b20ce10 100644 --- a/zig/src/families/qwen3_5/cuda_native.zig +++ b/zig/src/families/qwen3_5/cuda_native.zig @@ -230,6 +230,7 @@ pub fn explain(ctx: ?*anyopaque, err: anyerror) ?[]const u8 { if (ctx == null) return switch (err) { error.NoQwenVision => "--vision: this checkpoint has no image tower (no vision_config): serve it without --vision", error.QwenPagesNoLearner => "TF_QWEN_KV_PAGES: the learner reads whole windows, not pages; drop --slide or unset it", + error.QwenRequestStartsWithSharedKv => "TF_QWEN_REQUEST_STARTS=1 cannot be used with TF_QWEN_KV_SHARE above 0; unset request starts or disable shared KV", error.QwenShareNeedsPages => "TF_QWEN_KV_SHARE keeps prompts as pool pages: set TF_QWEN_KV_PAGES to a divisor of the 4096-row prompt chunk", error.BadQwenKvShare => "TF_QWEN_KV_SHARE takes the most recurrent-state snapshots the prompt cache keeps (0 or unset: off)", error.QwenYarnNoLearner => "--slide: the learner's attention ropes without YaRN; serve without --slide, or without --rope-yarn and a YaRN config", diff --git a/zig/src/families/qwen3_5/cuda_prefix.zig b/zig/src/families/qwen3_5/cuda_prefix.zig index d93e4ac506..2100e16f0a 100644 --- a/zig/src/families/qwen3_5/cuda_prefix.zig +++ b/zig/src/families/qwen3_5/cuda_prefix.zig @@ -13,7 +13,11 @@ const Cache = caches.Cache; /// TF_QWEN_KV_SHARE=n: keep prompts as pool pages and at most n recurrent-state snapshots (0 or unset: off). pub fn sharing() !usize { const v = @import("core").profile.get(.TF_QWEN_KV_SHARE) orelse return 0; - return std.fmt.parseInt(usize, v, 10) catch error.BadQwenKvShare; + const snaps = std.fmt.parseInt(usize, v, 10) catch return error.BadQwenKvShare; + if (snaps > 0) if (@import("core").profile.get(.TF_QWEN_REQUEST_STARTS)) |starts| { + if (std.mem.eql(u8, starts, "1")) return error.QwenRequestStartsWithSharedKv; + }; + return snaps; } /// Below 4 snapshots a stream, live streams evict each other's before a resume (8 at 8 streams resumed none). diff --git a/zig/src/families/qwen3_5/cuda_request_starts_test.zig b/zig/src/families/qwen3_5/cuda_request_starts_test.zig new file mode 100644 index 0000000000..430a625a1f --- /dev/null +++ b/zig/src/families/qwen3_5/cuda_request_starts_test.zig @@ -0,0 +1,69 @@ +//! Startup compatibility for the explicit message-cut opt-in, through the actual dense and MoE entry points. +const std = @import("std"); +const cuda = @import("cuda"); +const native = @import("cuda_native.zig"); +const moe = @import("cuda_native_moe.zig"); +const prefix = @import("cuda_prefix.zig"); +const env = @import("sys").env; +const t = std.testing; + +const Saved = struct { + const names = [_][:0]const u8{ "TF_QWEN_REQUEST_STARTS", "TF_QWEN_KV_SHARE", "TF_QWEN_KV_PAGES" }; + values: [names.len]?[:0]u8 = @splat(null), + fn take() !Saved { + var s: Saved = .{}; + errdefer for (s.values) |v| if (v) |old| t.allocator.free(old); + for (names, 0..) |n, i| if (@import("sys").getenv(n)) |v| { + s.values[i] = try t.allocator.dupeSentinel(u8, v, 0); + }; + return s; + } + fn restore(s: *Saved) void { + for (names, s.values) |n, v| if (v) |old| { + _ = env.setenv(n, old, 1); + t.allocator.free(old); + } else { + _ = env.unsetenv(n); + }; + } +}; + +fn put(name: [:0]const u8, value: ?[:0]const u8) !void { + try t.expectEqual(@as(c_int, 0), if (value) |v| env.setenv(name, v, 1) else env.unsetenv(name)); +} + +test "request starts: shared KV refuses at actual dense and MoE startup before loading" { + var saved = try Saved.take(); + defer saved.restore(); + try put("TF_QWEN_REQUEST_STARTS", "1"); + try put("TF_QWEN_KV_SHARE", "32"); + try put("TF_QWEN_KV_PAGES", "256"); + var ctx: cuda.Context = undefined; + inline for (.{ native, moe }) |F| { + try t.expectError(error.QwenRequestStartsWithSharedKv, F.open(t.allocator, t.io, &ctx, "missing-request-starts-checkpoint", null, .{ .context = 8192, .drafts = false })); + const reason = F.explain(null, error.QwenRequestStartsWithSharedKv) orelse return error.MissingNamedRefusal; + try t.expect(std.mem.indexOf(u8, reason, "TF_QWEN_REQUEST_STARTS=1") != null); + try t.expect(std.mem.indexOf(u8, reason, "TF_QWEN_KV_SHARE") != null); + } +} + +test "request starts: shared-KV parser keeps off values and its existing count errors" { + var saved = try Saved.take(); + defer saved.restore(); + try put("TF_QWEN_KV_SHARE", "32"); + for ([_]?[:0]const u8{ null, "", "0", "off", "on", "true", "2" }) |v| { + try put("TF_QWEN_REQUEST_STARTS", v); + try t.expectEqual(@as(usize, 32), try prefix.sharing()); + } + try put("TF_QWEN_REQUEST_STARTS", "1"); + for ([_]?[:0]const u8{ null, "0", "00" }) |v| { + try put("TF_QWEN_KV_SHARE", v); + try t.expectEqual(@as(usize, 0), try prefix.sharing()); + } + for ([_][:0]const u8{ "1", "32", "00032" }) |v| { + try put("TF_QWEN_KV_SHARE", v); + try t.expectError(error.QwenRequestStartsWithSharedKv, prefix.sharing()); + } + try put("TF_QWEN_KV_SHARE", "bad"); + try t.expectError(error.BadQwenKvShare, prefix.sharing()); +} From 50658f70e91915bfe92b4c0a0190ced4d4f024e3 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:26:15 +0000 Subject: [PATCH 4/6] cuda: each family reads its own request-starts switch, so one family's opt-in sends no other family message starts requestStep reads the switch the family names (request_starts_switch) through profile.lookup; a family that names none gets no message starts. Qwen's dense and MoE entry points name TF_QWEN_REQUEST_STARTS, so its name, its refusal with TF_QWEN_KV_SHARE and its tests are unchanged, and unset is main's path. The shape follows #2404's (0279d373), without its GLM engine changes, so #2404 can drop its copy. A host test walks the real CUDA registry: Qwen's switch reaches only Qwen, a second family's switch only that family, unset none. --- zig/src/families/qwen3_5/cuda_native.zig | 1 + zig/src/families/qwen3_5/cuda_native_moe.zig | 1 + zig/src/native/cuda.zig | 2 +- zig/src/native/cuda_snaps.zig | 46 +++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/zig/src/families/qwen3_5/cuda_native.zig b/zig/src/families/qwen3_5/cuda_native.zig index 0b6b20ce10..34a1b6aac9 100644 --- a/zig/src/families/qwen3_5/cuda_native.zig +++ b/zig/src/families/qwen3_5/cuda_native.zig @@ -51,6 +51,7 @@ pub const kept_disk = @import("cuda_disk.zig"); /// cuda_prefill cuts a cold pass at the request's chunk starts too, so the server may send its message starts. pub const request_starts = true; +pub const request_starts_switch = "TF_QWEN_REQUEST_STARTS"; /// explain() words open refusals too (--vision on a checkpoint without a tower it reads). pub const explains_open = true; diff --git a/zig/src/families/qwen3_5/cuda_native_moe.zig b/zig/src/families/qwen3_5/cuda_native_moe.zig index a1e39959ef..a67ab34458 100644 --- a/zig/src/families/qwen3_5/cuda_native_moe.zig +++ b/zig/src/families/qwen3_5/cuda_native_moe.zig @@ -29,6 +29,7 @@ pub const snap_grid = native.snap_grid; pub const kept_disk = native.kept_disk; pub const request_starts = native.request_starts; +pub const request_starts_switch = native.request_starts_switch; pub const explains_open = native.explains_open; pub const vision = native.vision; diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 1eee57bab1..4e3dc942ac 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -24,7 +24,7 @@ const drafted = @import("cuda_drafted.zig"); const kv_pages = @import("cuda_kv_pool.zig"); const cuda_families = @import("cuda_families.zig"); /// CUDA families provide metadata, open their lane backend and explain their refusals. -const registry = cuda_families.registry; +pub const registry = cuda_families.registry; /// --record opens here (engines.zig refuses it for a backend without this). pub const records = true; pub const backends: []const []const u8 = &.{"cuda"}; diff --git a/zig/src/native/cuda_snaps.zig b/zig/src/native/cuda_snaps.zig index 3bd1298373..9607919294 100644 --- a/zig/src/native/cuda_snaps.zig +++ b/zig/src/native/cuda_snaps.zig @@ -27,7 +27,8 @@ pub fn passRules(comptime F: type, prompt_rows: u32) api.prompt_cache.Rules { /// The server's chunk step: a family whose cold passes cut at request starts gets message starts on its grid (0: none). pub fn requestStep(comptime F: type) u32 { - return requestStepFor(F, @import("profile").get(.TF_QWEN_REQUEST_STARTS)); + if (!@hasDecl(F, "request_starts_switch")) return 0; // each family names its own opt-in, so it turns on alone + return requestStepFor(F, @import("profile").lookup(F.request_starts_switch)); } /// Only exact 1 opts in; a family must also declare that its cold passes honor request starts. @@ -134,6 +135,49 @@ test "only a family whose passes cut at request starts asks the server for them" try std.testing.expectEqual(@as(u32, 4096), requestStepFor(Qwen, "1")); } +test "request starts: one family's switch leaves every other family without starts, and unset leaves all on main" { + const sys = @import("sys"); + const profile = @import("profile"); + const Other = struct { + pub const snap_grid: u32 = 2048; + pub const request_starts = true; + pub const request_starts_switch = "TF_SNAPS_TEST_REQUEST_STARTS"; + }; + const Unnamed = struct { + pub const snap_grid: u32 = 2048; + pub const request_starts = true; + }; + const names = [_][:0]const u8{ "TF_QWEN_REQUEST_STARTS", Other.request_starts_switch }; + var saved: [names.len]?[:0]u8 = @splat(null); + for (names, &saved) |n, *s| if (sys.getenv(n)) |v| { + s.* = try std.testing.allocator.dupeSentinel(u8, v, 0); + }; + defer for (names, saved) |n, s| if (s) |old| { + _ = sys.env.setenv(n, old, 1); + std.testing.allocator.free(old); + } else { + _ = sys.env.unsetenv(n); + }; + const registry = @import("cuda.zig").registry; + var qwens: usize = 0; + inline for (registry) |F| if (@hasDecl(F, "request_starts_switch")) { + try std.testing.expect(std.mem.startsWith(u8, F.model_type, "qwen3_5")); + const id = profile.registered(F.request_starts_switch) orelse return error.UnreviewedSwitch; + try std.testing.expect(profile.entry(id).applies(F.model_type, "cuda")); + qwens += 1; + }; + try std.testing.expectEqual(@as(usize, 2), qwens); // the dense and MoE entry points + for (0..names.len + 1) |on| { // on == names.len: every switch unset + for (names, 0..) |n, i| try std.testing.expectEqual(@as(c_int, 0), if (i == on) sys.env.setenv(n, "1", 1) else sys.env.unsetenv(n)); + inline for (registry ++ .{ Other, Unnamed }) |F| { + const qwen = @hasDecl(F, "request_starts_switch") and F != Other; + const grid: u32 = if (@hasDecl(F, "snap_grid")) F.snap_grid else 0; + try std.testing.expect(!qwen or grid > 0); + try std.testing.expectEqual(if ((on == 0 and qwen) or (on == 1 and F == Other)) grid else 0, requestStep(F)); + } + } +} + test "request starts: unset keeps CUDA's main prefill contract" { const Qwen = struct { pub const snap_grid: u32 = 4096; From 367762fdb44bf62d7111672b57cc8341b3722742 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Sat, 7 Nov 2026 05:16:19 +0000 Subject: [PATCH 5/6] Keep each opted-in CUDA chat turn's own history start Let a family with exact request starts name the minimum distance between message cuts. Qwen's opt-in uses one token; other paths retain 256. The existing store keeps one history state per turn within its budget. Cover six real chat preparations and a one-slot cache, plus registry switch isolation and the existing startup refusal. Refs #2176. --- CHANGELOG.md | 1 + docs/optimization-ledger.md | 4 + zig/build/cuda_server_host.zig | 2 +- zig/build/request_starts_host.zig | 7 + zig/src/core/engine_api.zig | 2 + zig/src/native/cuda.zig | 1 + zig/src/native/cuda_snaps.zig | 6 + zig/src/server/request_starts_turns_test.zig | 137 +++++++++++++++++++ zig/src/server/root.zig | 4 + zig/src/server/server.zig | 1 + 10 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 zig/build/request_starts_host.zig create mode 100644 zig/src/server/request_starts_turns_test.zig diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3ff00bc0..4ece09bfbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel - Developers: `python3 -m pytest` from the repository root passes on a machine with no GPU or optional packages (286 passed, the rest skipped with their reason): the release packaging check is renamed `tools/release/check_package.py`, and CI runs the root suite (#2471). - Docs and packaging: `docs/evaluating.md` says what to pin when testing TensorFold, how to check a release's signature, which tests need no GPU and which requests to replay. A root `Dockerfile` installs a Linux release archive once its checksum, its signature and every unpacked file's checksum match (#2472). +- Qwen CUDA with `TF_QWEN_REQUEST_STARTS=1` keeps later chat turns at their own history starts even when turns are less than 256 tokens apart. The unset path and other families keep their existing chunk minimum (#2176). Thanks to @gilby. - Internal: HIP record counts are pinned in their owning arrays, so additions in different record files no longer edit a shared numeric total; omissions still fail host checks (#2461). - Qwen3.8 Flash Next on CUDA: BF16 prompt projections take a 128 × 128 tile on sm_12x wherever it fills the GPU, with the same output bytes, so prompts are 4 to 6% faster on a GB10 (NVFP4 checkpoint) and replies are unchanged. `TF_FLASHNEXT_PROMPT_BF16=0` turns the tile off, and `TF_FLASHNEXT_PROMPT_ROWS` is gone (#1846, refs #1754). Thanks to @laudney. - Internal: `compile-cuda-tests` builds again: `contract_controls.cu` declares the prompt-row fragment's page settings (`P256`, `IDS`, `PN`), which its control kernel predated (#2463). diff --git a/docs/optimization-ledger.md b/docs/optimization-ledger.md index 97d827f749..1edd79f45d 100644 --- a/docs/optimization-ledger.md +++ b/docs/optimization-ledger.md @@ -282,6 +282,10 @@ below; these are estimates for planning a port, not results. `--learn-conversations` today. Flash Next already resumes a prefix from the host store's whole-state copies; on the shared store a kept 32k state would stop duplicating about 0.9 GiB of attention rows (27 KiB a token). + Qwen CUDA's host-copy path can instead opt into message-start cuts with `TF_QWEN_REQUEST_STARTS=1` + (#2176). Later turns closer than 256 tokens now keep their own history cuts; host checks pass and the six-turn + device gate is pending. This mode remains incompatible with `TF_QWEN_KV_SHARE`. + 4. **Prompt states in device memory on integrated GPUs.** On a GB10 the device and host share memory, so a kept state can stay where it is instead of being copied through pinned host memory on every save ([#2043]). On Qwen CUDA this takes 1.6–2.1 % off 8k and 32k prompts ([#2113]). Metal is unified already. diff --git a/zig/build/cuda_server_host.zig b/zig/build/cuda_server_host.zig index 24664277fe..1e78207cf3 100644 --- a/zig/build/cuda_server_host.zig +++ b/zig/build/cuda_server_host.zig @@ -59,7 +59,7 @@ pub fn steps(b: *std.Build, host: std.Build.ResolvedTarget, cuda: *std.Build.Mod b.step("test-server-recovery-host", "Incomplete replies, token allocation and bounded SSE storage without shaders").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"Server recovery:"} })).step); b.step("test-prompt-interior-host", "Last-user metadata excludes later messages without shaders").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"interior prompt:"} })).step); const server_test = b.addRunArtifact(b.addTest(.{ .root_module = server_tests })); - @import("server_seed_host.zig").steps(b, server_tests); + @import("request_starts_host.zig").steps(b, native_modules.engines, server_tests); const json_test = b.addRunArtifact(b.addTest(.{ .root_module = server_json })); const publication = b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"benchmark publication:"} })); b.step("test-benchmark-publication-host", "Benchmark command publication boundaries with inert effects").dependOn(&publication.step); diff --git a/zig/build/request_starts_host.zig b/zig/build/request_starts_host.zig new file mode 100644 index 0000000000..fd2b853711 --- /dev/null +++ b/zig/build/request_starts_host.zig @@ -0,0 +1,7 @@ +//! Exact message-start metadata and chat retention, without devices. +const std = @import("std"); +pub fn steps(b: *std.Build, native: *std.Build.Module, server: *std.Build.Module) void { + @import("server_seed_host.zig").steps(b, server); + b.step("test-cuda-message-starts-host", "Family-owned request-start metadata without devices").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"request starts:"} })).step); + b.step("test-request-starts-turns-host", "Six chat histories keep their own exact start within one snapshot slot").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server, .filters = &.{"turn starts:"} })).step); +} diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 49ea1f13e0..50b8b1dd2f 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -193,6 +193,8 @@ pub const Info = struct { prompt_logprobs: bool = false, /// Prompt rows a prefill chunk at most, for the server's chunk starts (0: the engine cuts prompts itself). prefill_step: u32 = 0, + /// The minimum distance between template message cuts; exact-start families may keep adjacent turns. + prefill_min_chunk: u32 = 256, /// The engine keeps prompt states between requests, so ``prefilled`` counts the tokens it restored. prompt_cache: bool = false, /// Startup cache budget and the smallest state admitted by the family's existing keep plan. diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index 4e3dc942ac..74fc160673 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -443,6 +443,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O .draft_sources = .{ .drafter = served.source }, .context_window = @intCast(window), .prefill_step = snaps.requestStep(F), + .prefill_min_chunk = snaps.requestMinimum(F), .startup = h.startup, .policy = if (@hasField(@TypeOf(loaded), "policy")) loaded.policy else "", .prompt_cache = h.store != null, diff --git a/zig/src/native/cuda_snaps.zig b/zig/src/native/cuda_snaps.zig index 9607919294..7e883e53f6 100644 --- a/zig/src/native/cuda_snaps.zig +++ b/zig/src/native/cuda_snaps.zig @@ -31,6 +31,11 @@ pub fn requestStep(comptime F: type) u32 { return requestStepFor(F, @import("profile").lookup(F.request_starts_switch)); } +/// An opted-in family keeps each message start, including turns closer than the shared default. +pub fn requestMinimum(comptime F: type) u32 { + return if (requestStep(F) > 0) 1 else 256; +} + /// Only exact 1 opts in; a family must also declare that its cold passes honor request starts. pub fn requestStepFor(comptime F: type, value: ?[]const u8) u32 { const v = value orelse return 0; @@ -174,6 +179,7 @@ test "request starts: one family's switch leaves every other family without star const grid: u32 = if (@hasDecl(F, "snap_grid")) F.snap_grid else 0; try std.testing.expect(!qwen or grid > 0); try std.testing.expectEqual(if ((on == 0 and qwen) or (on == 1 and F == Other)) grid else 0, requestStep(F)); + try std.testing.expectEqual(@as(u32, if ((on == 0 and qwen) or (on == 1 and F == Other)) 1 else 256), requestMinimum(F)); } } } diff --git a/zig/src/server/request_starts_turns_test.zig b/zig/src/server/request_starts_turns_test.zig new file mode 100644 index 0000000000..8b5cd6a9ae --- /dev/null +++ b/zig/src/server/request_starts_turns_test.zig @@ -0,0 +1,137 @@ +//! Six actual chat preparations keep each history end, even when turns are closer than the default chunk minimum. +const std = @import("std"); +const api = @import("engine_api"); +const json = @import("json"); +const errors = @import("errors.zig"); +const fixture = @import("control_tokens_fixture.zig"); +const HfText = @import("hf_text.zig").HfText; +const Server = @import("server.zig").Server; +const chat = @import("chat.zig"); +const openai = @import("openai.zig"); +const pc = api.prompt_cache; +const a = std.testing.allocator; +const t = std.testing; + +const Fake = struct { + at: u32 = 0, + sum: u64 = 0, + saves: usize = 0, + minimum: u32, + const Saved = struct { at: u32, sum: u64 }; + fn engine(f: *@This()) api.Engine { + return .{ .ctx = f, .vtable = &.{ .info = info, .submit = submit, .cancel = cancel, .status = status, .memory = memory } }; + } + fn info(ptr: *anyopaque) api.Info { + const f: *@This() = @ptrCast(@alignCast(ptr)); + var out: api.Info = .{ .context_window = 32768, .prefill_step = 4096, .prompt_cache = true }; + if (@hasField(api.Info, "prefill_min_chunk")) out.prefill_min_chunk = f.minimum; + return out; + } + fn submit(_: *anyopaque, _: api.Id, _: *const api.Request, _: api.Sink) api.SubmitError!void { + return error.Closed; + } + fn cancel(_: *anyopaque, _: api.Id) void {} + fn status(_: *anyopaque, out: *api.Status, _: []u32) void { + out.* = .{}; + } + fn memory(_: *anyopaque, _: bool) ?api.Memory { + return null; + } + fn bytes(_: *anyopaque, _: u32) u64 { + return @sizeOf(Saved); + } + fn save(ptr: *anyopaque, _: ?*anyopaque, at: u32) !pc.Saved { + const f: *@This() = @ptrCast(@alignCast(ptr)); + try t.expectEqual(at, f.at); + const s = try a.create(Saved); + s.* = .{ .at = at, .sum = f.sum }; + f.saves += 1; + return s; + } + fn restore(ptr: *anyopaque, _: ?*anyopaque, saved: pc.Saved) !void { + const f: *@This() = @ptrCast(@alignCast(ptr)); + const s: *Saved = @ptrCast(@alignCast(saved)); + f.at = s.at; + f.sum = s.sum; + } + fn drop(_: *anyopaque, saved: pc.Saved) void { + a.destroy(@as(*Saved, @ptrCast(@alignCast(saved)))); + } + fn snapshots(f: *@This()) pc.Snapshots { + return .{ .ptr = f, .vtable = &.{ .bytes = bytes, .save = save, .restore = restore, .drop = drop } }; + } +}; +const Never = struct { + pub fn check(_: @This()) bool { + return false; + } +}; +fn fresh(ids: []const u32) u64 { + var sum: u64 = 0; + for (ids) |id| sum = sum *% 31 +% id; + return sum; +} +fn message(alloc: std.mem.Allocator, role: []const u8, content: []const u8) !json.Value { + const out = try json.newObject(alloc); + try out.put(alloc, "role", .{ .string = role }); + try out.put(alloc, "content", .{ .string = content }); + return .{ .object = out }; +} + +test "turn starts: native metadata reaches six chat histories and a one-slot kept store" { + for ([_]u32{ 256, 1 }) |minimum| { + var arena = std.heap.ArenaAllocator.init(a); + defer arena.deinit(); + const alloc = arena.allocator(); + var files = try fixture.Fixture.init(alloc, 0); + defer files.deinit(); + var problem: []const u8 = ""; + const hf = try HfText.load(alloc, t.io, files.path, null, alloc, &problem); + defer hf.deinit(); + var f: Fake = .{ .minimum = minimum }; + const srv = try Server.init(alloc, t.io, f.engine(), hf.text(), .{ .served_name = "m", .model_ids = &.{"m"}, .enable_thinking = false }, null); + defer srv.deinit(); + var store = pc.Store.init(a, f.snapshots(), .{ .planned = true, .grid = 4096, .min_prompt = 0 }, @sizeOf(Fake.Saved)); + defer store.deinit(); + store.most = 1; + var messages: std.ArrayList(json.Value) = .empty; + const document = try alloc.alloc(u8, 1450); + @memset(document, 'x'); + try messages.append(alloc, try message(alloc, "user", document)); + var previous: u32 = 0; + var first: u32 = 0; + for (0..6) |turn| { + const body = try json.newObject(alloc); + try body.put(alloc, "messages", .{ .array = messages.items }); + try body.put(alloc, "max_tokens", try json.intValue(alloc, 1)); + var cx: errors.Cx = .{ .a = alloc }; + const input = try openai.chatInput(srv, &cx, .{ .object = body }); + const prepared = try chat.prepare(srv, &cx, input, Never{}); + defer srv.release(prepared.preparing, prepared.request); + const r = prepared.request; + if (turn == 0) first = r.history_len; + if (turn > 0) try t.expect(r.history_len > previous and r.history_len - first < 256); + const plan = try store.beginUser(alloc, r.prompt, r.history_len, r.rewind_len, r.user_end, r.shared_prefixes, r.chunks, null, r.decode_spans); + try t.expectEqual(if (turn == 0) 0 else if (minimum == 1) previous else first, plan.from); + const wanted = if (minimum == 1) r.history_len else first; + if (minimum == 1 or turn == 0) try t.expectEqualSlices(u32, &.{wanted}, plan.marks); + if (plan.from == 0) { + f.at = 0; + f.sum = 0; + } + while (f.at < r.prompt.len) { + f.sum = f.sum *% 31 +% r.prompt[f.at]; + f.at += 1; + if (std.mem.indexOfScalar(u32, plan.marks, f.at) != null) try t.expect(store.keep(r.prompt, f.at, null, r.chunks, r.decode_spans)); + } + try t.expectEqual(fresh(r.prompt), f.sum); + try t.expectEqual(@as(usize, 1), store.entries.items.len); + try t.expectEqual(wanted, store.entries.items[0].at); + try t.expect(store.held <= store.budget); + previous = r.history_len; + try messages.append(alloc, try message(alloc, "assistant", "Fixed reply.")); + try messages.append(alloc, try message(alloc, "user", "Next question?")); + } + try t.expectEqual(@as(usize, if (minimum == 1) 6 else 1), f.saves); + } +} diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig index ca3d4fa4d6..fc6b1d08ab 100644 --- a/zig/src/server/root.zig +++ b/zig/src/server/root.zig @@ -250,3 +250,7 @@ test { _ = @import("penalty_request_test.zig"); _ = @import("deepseek_fixture_test.zig"); } + +test { + _ = @import("request_starts_turns_test.zig"); +} diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig index fd84c91031..0b877b77c0 100644 --- a/zig/src/server/server.zig +++ b/zig/src/server/server.zig @@ -235,6 +235,7 @@ pub const Server = struct { } if (srv.info.prefill_step > 0) { srv.chunks = try chunk_plan.markers(srv, a, srv.info.prefill_step); + srv.chunks.min_chunk = srv.info.prefill_min_chunk; var step: [32]u8 = undefined; log.line("prompt chunks of up to {s} tokens, cut at replies {d}+ tokens apart", .{ live.commas(&step, srv.info.prefill_step), srv.chunks.min_chunk }); } From 126972d6c50cfb05da9bb0b8690673fd99167f64 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:22:08 +0000 Subject: [PATCH 6/6] request starts: the GB10 six-turn result in the changelog, ledger and profile evidence --- CHANGELOG.md | 1 + docs/optimization-ledger.md | 6 ++++-- zig/src/core/profile.zig | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ece09bfbb..855bcd4ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel - Docs and packaging: `docs/evaluating.md` says what to pin when testing TensorFold, how to check a release's signature, which tests need no GPU and which requests to replay. A root `Dockerfile` installs a Linux release archive once its checksum, its signature and every unpacked file's checksum match (#2472). - Qwen CUDA with `TF_QWEN_REQUEST_STARTS=1` keeps later chat turns at their own history starts even when turns are less than 256 tokens apart. The unset path and other families keep their existing chunk minimum (#2176). Thanks to @gilby. +- Qwen CUDA with `TF_QWEN_REQUEST_STARTS=1` keeps later chat turns at their own history starts even when turns are less than 256 tokens apart. On a GB10 with Qwen3.8-27B 4-bit and its DFlash2 drafter, turns 2 to 6 of a six-turn chat reuse 1,450 to 1,635 prompt tokens and take 0.72 to 1.33 s instead of 1.78 to 2.10 s. Prompt passes are cut at replies, so a reply can differ from the unset path's; resumed and fresh replies are the same with it set. The unset path and other families keep their existing chunk minimum (#2176). Thanks to @gilby. - Internal: HIP record counts are pinned in their owning arrays, so additions in different record files no longer edit a shared numeric total; omissions still fail host checks (#2461). - Qwen3.8 Flash Next on CUDA: BF16 prompt projections take a 128 × 128 tile on sm_12x wherever it fills the GPU, with the same output bytes, so prompts are 4 to 6% faster on a GB10 (NVFP4 checkpoint) and replies are unchanged. `TF_FLASHNEXT_PROMPT_BF16=0` turns the tile off, and `TF_FLASHNEXT_PROMPT_ROWS` is gone (#1846, refs #1754). Thanks to @laudney. - Internal: `compile-cuda-tests` builds again: `contract_controls.cu` declares the prompt-row fragment's page settings (`P256`, `IDS`, `PN`), which its control kernel predated (#2463). diff --git a/docs/optimization-ledger.md b/docs/optimization-ledger.md index 1edd79f45d..a6cd6c934a 100644 --- a/docs/optimization-ledger.md +++ b/docs/optimization-ledger.md @@ -283,8 +283,10 @@ below; these are estimates for planning a port, not results. shared store a kept 32k state would stop duplicating about 0.9 GiB of attention rows (27 KiB a token). Qwen CUDA's host-copy path can instead opt into message-start cuts with `TF_QWEN_REQUEST_STARTS=1` - (#2176). Later turns closer than 256 tokens now keep their own history cuts; host checks pass and the six-turn - device gate is pending. This mode remains incompatible with `TF_QWEN_KV_SHARE`. + (#2176). Later turns closer than 256 tokens now keep their own history cuts. On a GB10 with Qwen3.8-27B 4-bit and + its DFlash2 drafter, turns 2 to 6 of a six-turn chat reused 1,450 to 1,635 prompt tokens and took 0.72 to 1.33 s + against 1.78 to 2.10 s unset. Prompt passes are cut at replies, so one of the six replies differed from the unset + path; resumed and fresh replies matched with it set. This mode remains incompatible with `TF_QWEN_KV_SHARE`. 4. **Prompt states in device memory on integrated GPUs.** On a GB10 the device and host share memory, so a kept state can stay where it is instead of being copied through pinned host memory on every save ([#2043]). On Qwen diff --git a/zig/src/core/profile.zig b/zig/src/core/profile.zig index 4805c93209..f4e80ea715 100644 --- a/zig/src/core/profile.zig +++ b/zig/src/core/profile.zig @@ -97,7 +97,7 @@ const slide_cuda: []const []const u8 = &.{ "nemotron_h", "nemotron_h_omni", "qwe const flash: []const []const u8 = &.{"qwen4_exp"}; pub const entries = [_]Entry{ - .{ .id = .TF_QWEN_REQUEST_STARTS, .families = qwen, .class = .arithmetic, .syntax = .one, .effect = "Opt into CUDA message-start prompt cuts", .evidence = "#1334; default-off, device qualification pending" }, + .{ .id = .TF_QWEN_REQUEST_STARTS, .families = qwen, .class = .arithmetic, .syntax = .one, .effect = "Opt into CUDA message-start prompt cuts", .evidence = "#1334; default-off; GB10 six-turn: resumed equals fresh, one reply differs from unset" }, .{ .id = .TF_QWEN_KV_PAGES, .families = qwen, .class = .identical, .effect = "KV pool page size; memory and workload tradeoffs", .evidence = "#1357" }, .{ .id = .TF_QWEN_KV_PAGES_ALL, .families = qwen, .class = .debug, .syntax = .one, .effect = "Put the engine's own stream on pages too", .evidence = "#1357" }, .{ .id = .TF_QWEN_KV_SHARE, .families = qwen, .class = .identical, .effect = "Keep prompt pages and bounded recurrent snapshots", .evidence = "#1357" },