qwen3_5 cuda: a chat's next turn can resume at the previous turn's history (opt-in)

CHANGELOG.md

@@ -11,6 +11,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
 - Developers: `python3 -m pytest` from the repository root passes on a machine with no GPU or optional packages (286 passed, the rest skipped with their reason): the release packaging check is renamed `tools/release/check_package.py`, and CI runs the root suite (#2471).
 
 - Docs and packaging: `docs/evaluating.md` says what to pin when testing TensorFold, how to check a release's signature, which tests need no GPU and which requests to replay. A root `Dockerfile` installs a Linux release archive once its checksum, its signature and every unpacked file's checksum match (#2472).
+- Qwen CUDA with `TF_QWEN_REQUEST_STARTS=1` keeps later chat turns at their own history starts even when turns are less than 256 tokens apart. The unset path and other families keep their existing chunk minimum (#2176). Thanks to @gilby.
+- Qwen CUDA with `TF_QWEN_REQUEST_STARTS=1` keeps later chat turns at their own history starts even when turns are less than 256 tokens apart. On a GB10 with Qwen3.8-27B 4-bit and its DFlash2 drafter, turns 2 to 6 of a six-turn chat reuse 1,450 to 1,635 prompt tokens and take 0.72 to 1.33 s instead of 1.78 to 2.10 s. Prompt passes are cut at replies, so a reply can differ from the unset path's; resumed and fresh replies are the same with it set. The unset path and other families keep their existing chunk minimum (#2176). Thanks to @gilby.
 - Internal: HIP record counts are pinned in their owning arrays, so additions in different record files no longer edit a shared numeric total; omissions still fail host checks (#2461).
 - Qwen3.8 Flash Next on CUDA: BF16 prompt projections take a 128 × 128 tile on sm_12x wherever it fills the GPU, with the same output bytes, so prompts are 4 to 6% faster on a GB10 (NVFP4 checkpoint) and replies are unchanged. `TF_FLASHNEXT_PROMPT_BF16=0` turns the tile off, and `TF_FLASHNEXT_PROMPT_ROWS` is gone (#1846, refs #1754). Thanks to @laudney.
 - Internal: `compile-cuda-tests` builds again: `contract_controls.cu` declares the prompt-row fragment's page settings (`P256`, `IDS`, `PN`), which its control kernel predated (#2463).

docs/optimization-ledger.md

@@ -282,6 +282,12 @@ below; these are estimates for planning a port, not results.
    `--learn-conversations` today. Flash Next already resumes a prefix from the host store's whole-state copies; on the
    shared store a kept 32k state would stop duplicating about 0.9 GiB of attention rows (27 KiB a token).
 
+   Qwen CUDA's host-copy path can instead opt into message-start cuts with `TF_QWEN_REQUEST_STARTS=1`
+   (#2176). Later turns closer than 256 tokens now keep their own history cuts. On a GB10 with Qwen3.8-27B 4-bit and
+   its DFlash2 drafter, turns 2 to 6 of a six-turn chat reused 1,450 to 1,635 prompt tokens and took 0.72 to 1.33 s
+   against 1.78 to 2.10 s unset. Prompt passes are cut at replies, so one of the six replies differed from the unset
+   path; resumed and fresh replies matched with it set. This mode remains incompatible with `TF_QWEN_KV_SHARE`.
+
 4. **Prompt states in device memory on integrated GPUs.** On a GB10 the device and host share memory, so a kept
    state can stay where it is instead of being copied through pinned host memory on every save ([#2043]). On Qwen
    CUDA this takes 1.6–2.1 % off 8k and 32k prompts ([#2113]). Metal is unified already.

zig/build/cuda_family_host.zig

@@ -10,6 +10,7 @@ pub fn steps(b: *std.Build, cuda: *std.Build.Module, mods: anytype, step: *std.B
     b.step("test-cuda-persistence-source-host", "Captured CUDA source keys without loading a driver").dependOn(&captured.step);
     step.dependOn(&captured.step);
 
+    b.step("test-qwen35-request-starts-host", "Request-start and shared-KV startup refusal without devices").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = mods.qwen35, .filters = &.{"request starts:"} })).step);
     @import("minimax_cuda.zig").hostTests(b, host, cuda, mods.core, mods.lanes, step);
     @import("cuda_groups_host.zig").host(b, cuda, mods, step);
     const prefill = b.addRunArtifact(b.addTest(.{ .root_module = mods.qwen35, .filters = &.{"Qwen stepped prefill:"} }));
@@ -165,6 +166,7 @@ pub fn steps(b: *std.Build, cuda: *std.Build.Module, mods: anytype, step: *std.B
 
 pub fn registerCopyFamilies(b: *std.Build, native: *std.Build.Module, step: *std.Build.Step) void {
     b.step("test-glm-formats-host", "CUDA GLM EXL3 offer reaches the published family registry without a GPU").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"GLM capabilities:"} })).step);
+    b.step("test-request-starts-host", "Opt-in CUDA message starts without GPU execution").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"request starts:"} })).step);
     const family_copies = b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"CUDA lane copies:"} }));
     b.step("test-family-copies-host", "Actual CUDA family copy policies through the lane core").dependOn(&family_copies.step);
     b.step("test-glm-copies-host", "GLM copy facts, confidence and costs through the production lane core").dependOn(&family_copies.step);

zig/build/cuda_server_host.zig

@@ -59,7 +59,7 @@ pub fn steps(b: *std.Build, host: std.Build.ResolvedTarget, cuda: *std.Build.Mod
     b.step("test-server-recovery-host", "Incomplete replies, token allocation and bounded SSE storage without shaders").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"Server recovery:"} })).step);
     b.step("test-prompt-interior-host", "Last-user metadata excludes later messages without shaders").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"interior prompt:"} })).step);
     const server_test = b.addRunArtifact(b.addTest(.{ .root_module = server_tests }));
-    @import("server_seed_host.zig").steps(b, server_tests);
+    @import("request_starts_host.zig").steps(b, native_modules.engines, server_tests);
     const json_test = b.addRunArtifact(b.addTest(.{ .root_module = server_json }));
     const publication = b.addRunArtifact(b.addTest(.{ .root_module = server_tests, .filters = &.{"benchmark publication:"} }));
     b.step("test-benchmark-publication-host", "Benchmark command publication boundaries with inert effects").dependOn(&publication.step);

zig/build/request_starts_host.zig

@@ -0,0 +1,7 @@
+//! Exact message-start metadata and chat retention, without devices.
+const std = @import("std");
+pub fn steps(b: *std.Build, native: *std.Build.Module, server: *std.Build.Module) void {
+    @import("server_seed_host.zig").steps(b, server);
+    b.step("test-cuda-message-starts-host", "Family-owned request-start metadata without devices").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = native, .filters = &.{"request starts:"} })).step);
+    b.step("test-request-starts-turns-host", "Six chat histories keep their own exact start within one snapshot slot").dependOn(&b.addRunArtifact(b.addTest(.{ .root_module = server, .filters = &.{"turn starts:"} })).step);
+}

zig/src/core/engine_api.zig

@@ -193,6 +193,8 @@ pub const Info = struct {
     prompt_logprobs: bool = false,
     /// Prompt rows a prefill chunk at most, for the server's chunk starts (0: the engine cuts prompts itself).
     prefill_step: u32 = 0,
+    /// The minimum distance between template message cuts; exact-start families may keep adjacent turns.
+    prefill_min_chunk: u32 = 256,
     /// The engine keeps prompt states between requests, so ``prefilled`` counts the tokens it restored.
     prompt_cache: bool = false,
     /// Startup cache budget and the smallest state admitted by the family's existing keep plan.

zig/src/core/profile.zig

@@ -7,6 +7,7 @@ pub const Switch = enum {
     TF_QWEN_KV_PAGES,
     TF_QWEN_KV_PAGES_ALL,
     TF_QWEN_KV_SHARE,
+    TF_QWEN_REQUEST_STARTS,
     TF_KV_SHARE,
     TF_FLASHNEXT_KV_PAGES,
     TF_FLASHNEXT_KV_PAGES_ALL,
@@ -96,6 +97,7 @@ const slide_cuda: []const []const u8 = &.{ "nemotron_h", "nemotron_h_omni", "qwe
 const flash: []const []const u8 = &.{"qwen4_exp"};
 
 pub const entries = [_]Entry{
+    .{ .id = .TF_QWEN_REQUEST_STARTS, .families = qwen, .class = .arithmetic, .syntax = .one, .effect = "Opt into CUDA message-start prompt cuts", .evidence = "#1334; default-off; GB10 six-turn: resumed equals fresh, one reply differs from unset" },
     .{ .id = .TF_QWEN_KV_PAGES, .families = qwen, .class = .identical, .effect = "KV pool page size; memory and workload tradeoffs", .evidence = "#1357" },
     .{ .id = .TF_QWEN_KV_PAGES_ALL, .families = qwen, .class = .debug, .syntax = .one, .effect = "Put the engine's own stream on pages too", .evidence = "#1357" },
     .{ .id = .TF_QWEN_KV_SHARE, .families = qwen, .class = .identical, .effect = "Keep prompt pages and bounded recurrent snapshots", .evidence = "#1357" },

zig/src/families/qwen3_5/cuda.zig

@@ -77,3 +77,7 @@ test {
 test "Qwen pages default: the learner's and the prefix share's refusals stop before the weights load" {
     try @import("cuda_admit_test.zig").refusals();
 }
+
+test {
+    _ = @import("cuda_request_starts_test.zig");
+}

zig/src/families/qwen3_5/cuda_native.zig

@@ -49,6 +49,10 @@ pub const snap_grid: u32 = E.max_rows;
 /// --learn-conversations: the kept states' disk image, the probe their identity hashes, the facts they depend on.
 pub const kept_disk = @import("cuda_disk.zig");
 
+/// cuda_prefill cuts a cold pass at the request's chunk starts too, so the server may send its message starts.
+pub const request_starts = true;
+pub const request_starts_switch = "TF_QWEN_REQUEST_STARTS";
+
 /// explain() words open refusals too (--vision on a checkpoint without a tower it reads).
 pub const explains_open = true;
 
@@ -227,6 +231,7 @@ pub fn explain(ctx: ?*anyopaque, err: anyerror) ?[]const u8 {
     if (ctx == null) return switch (err) {
         error.NoQwenVision => "--vision: this checkpoint has no image tower (no vision_config): serve it without --vision",
         error.QwenPagesNoLearner => "TF_QWEN_KV_PAGES: the learner reads whole windows, not pages; drop --slide or unset it",
+        error.QwenRequestStartsWithSharedKv => "TF_QWEN_REQUEST_STARTS=1 cannot be used with TF_QWEN_KV_SHARE above 0; unset request starts or disable shared KV",
         error.QwenShareNeedsPages => "TF_QWEN_KV_SHARE keeps prompts as pool pages: set TF_QWEN_KV_PAGES to a divisor of the 4096-row prompt chunk",
         error.BadQwenKvShare => "TF_QWEN_KV_SHARE takes the most recurrent-state snapshots the prompt cache keeps (0 or unset: off)",
         error.QwenYarnNoLearner => "--slide: the learner's attention ropes without YaRN; serve without --slide, or without --rope-yarn and a YaRN config",

zig/src/families/qwen3_5/cuda_native_moe.zig

@@ -27,6 +27,9 @@ pub const snap_host = native.snap_host;
 pub const snap_lookahead = native.snap_lookahead;
 pub const snap_grid = native.snap_grid;
 pub const kept_disk = native.kept_disk;
+
+pub const request_starts = native.request_starts;
+pub const request_starts_switch = native.request_starts_switch;
 pub const explains_open = native.explains_open;
 pub const vision = native.vision;
 

zig/src/families/qwen3_5/cuda_prefix.zig

@@ -13,7 +13,11 @@ const Cache = caches.Cache;
 /// TF_QWEN_KV_SHARE=n: keep prompts as pool pages and at most n recurrent-state snapshots (0 or unset: off).
 pub fn sharing() !usize {
     const v = @import("core").profile.get(.TF_QWEN_KV_SHARE) orelse return 0;
-    return std.fmt.parseInt(usize, v, 10) catch error.BadQwenKvShare;
+    const snaps = std.fmt.parseInt(usize, v, 10) catch return error.BadQwenKvShare;
+    if (snaps > 0) if (@import("core").profile.get(.TF_QWEN_REQUEST_STARTS)) |starts| {
+        if (std.mem.eql(u8, starts, "1")) return error.QwenRequestStartsWithSharedKv;
+    };
+    return snaps;
 }
 
 /// Below 4 snapshots a stream, live streams evict each other's before a resume (8 at 8 streams resumed none).

zig/src/families/qwen3_5/cuda_request_starts_test.zig

@@ -0,0 +1,69 @@
+//! Startup compatibility for the explicit message-cut opt-in, through the actual dense and MoE entry points.
+const std = @import("std");
+const cuda = @import("cuda");
+const native = @import("cuda_native.zig");
+const moe = @import("cuda_native_moe.zig");
+const prefix = @import("cuda_prefix.zig");
+const env = @import("sys").env;
+const t = std.testing;
+
+const Saved = struct {
+    const names = [_][:0]const u8{ "TF_QWEN_REQUEST_STARTS", "TF_QWEN_KV_SHARE", "TF_QWEN_KV_PAGES" };
+    values: [names.len]?[:0]u8 = @splat(null),
+    fn take() !Saved {
+        var s: Saved = .{};
+        errdefer for (s.values) |v| if (v) |old| t.allocator.free(old);
+        for (names, 0..) |n, i| if (@import("sys").getenv(n)) |v| {
+            s.values[i] = try t.allocator.dupeSentinel(u8, v, 0);
+        };
+        return s;
+    }
+    fn restore(s: *Saved) void {
+        for (names, s.values) |n, v| if (v) |old| {
+            _ = env.setenv(n, old, 1);
+            t.allocator.free(old);
+        } else {
+            _ = env.unsetenv(n);
+        };
+    }
+};
+
+fn put(name: [:0]const u8, value: ?[:0]const u8) !void {
+    try t.expectEqual(@as(c_int, 0), if (value) |v| env.setenv(name, v, 1) else env.unsetenv(name));
+}
+
+test "request starts: shared KV refuses at actual dense and MoE startup before loading" {
+    var saved = try Saved.take();
+    defer saved.restore();
+    try put("TF_QWEN_REQUEST_STARTS", "1");
+    try put("TF_QWEN_KV_SHARE", "32");
+    try put("TF_QWEN_KV_PAGES", "256");
+    var ctx: cuda.Context = undefined;
+    inline for (.{ native, moe }) |F| {
+        try t.expectError(error.QwenRequestStartsWithSharedKv, F.open(t.allocator, t.io, &ctx, "missing-request-starts-checkpoint", null, .{ .context = 8192, .drafts = false }));
+        const reason = F.explain(null, error.QwenRequestStartsWithSharedKv) orelse return error.MissingNamedRefusal;
+        try t.expect(std.mem.indexOf(u8, reason, "TF_QWEN_REQUEST_STARTS=1") != null);
+        try t.expect(std.mem.indexOf(u8, reason, "TF_QWEN_KV_SHARE") != null);
+    }
+}
+
+test "request starts: shared-KV parser keeps off values and its existing count errors" {
+    var saved = try Saved.take();
+    defer saved.restore();
+    try put("TF_QWEN_KV_SHARE", "32");
+    for ([_]?[:0]const u8{ null, "", "0", "off", "on", "true", "2" }) |v| {
+        try put("TF_QWEN_REQUEST_STARTS", v);
+        try t.expectEqual(@as(usize, 32), try prefix.sharing());
+    }
+    try put("TF_QWEN_REQUEST_STARTS", "1");
+    for ([_]?[:0]const u8{ null, "0", "00" }) |v| {
+        try put("TF_QWEN_KV_SHARE", v);
+        try t.expectEqual(@as(usize, 0), try prefix.sharing());
+    }
+    for ([_][:0]const u8{ "1", "32", "00032" }) |v| {
+        try put("TF_QWEN_KV_SHARE", v);
+        try t.expectError(error.QwenRequestStartsWithSharedKv, prefix.sharing());
+    }
+    try put("TF_QWEN_KV_SHARE", "bad");
+    try t.expectError(error.BadQwenKvShare, prefix.sharing());
+}

zig/src/native/cuda.zig

@@ -24,7 +24,7 @@ const drafted = @import("cuda_drafted.zig");
 const kv_pages = @import("cuda_kv_pool.zig");
 const cuda_families = @import("cuda_families.zig");
 /// CUDA families provide metadata, open their lane backend and explain their refusals.
-const registry = cuda_families.registry;
+pub const registry = cuda_families.registry;
 /// --record opens here (engines.zig refuses it for a backend without this).
 pub const records = true;
 pub const backends: []const []const u8 = &.{"cuda"};
@@ -432,7 +432,6 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O
         std.log.info("recording each stream's rows under {s}/{s}", .{ dir, r.recorder.name });
     }
     errdefer if (h.recording) |r| r.free(gpa);
-    // the family cuts its own prompt grid from position 0, as `tensorfold run` does: prefill_step 0
     const admit: ?api.Admit = if (comptime !@hasDecl(F, "fits")) null else if (paged != null) .{ .ctx = loaded.ctx, .fits = F.fits } else null;
     h.kv = kv_pages.watchOf(F, loaded, paged);
     h.kv_count = if (admit != null) kv_pages.ledgerPool(F, loaded) else null;
@@ -443,6 +442,8 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O
         .lanes = streams,
         .draft_sources = .{ .drafter = served.source },
         .context_window = @intCast(window),
+        .prefill_step = snaps.requestStep(F),
+        .prefill_min_chunk = snaps.requestMinimum(F),
         .startup = h.startup,
         .policy = if (@hasField(@TypeOf(loaded), "policy")) loaded.policy else "",
         .prompt_cache = h.store != null,