From 4ebe636a39f9975969c6cfc6c9a8f119d6a3b5fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BCrgen=20Schmied?=
 <600316+jschmied@users.noreply.github.com>
Date: Thu, 5 Nov 2026 21:21:11 +0100
Subject: [PATCH 1/2] gemma4 cuda: TF_GEMMA4_CHUNK_SOLO, on a solo-chunk rule
 shared in lanes (#1554 option C)

A Gemma 4 prompt pass takes TF_GEMMA4_CHUNK_SOLO rows (a multiple of the 512-row grid, up to
4096) while no other stream decodes, and the 512-row grid chunk once one does. The sliding rings,
the prompt scratch and the stream byte count size on the larger value. Unset, nothing changes.
Not beside a drafter's taps, or past 8 images in a prompt.

The rule (the limits, the parse, "does another stream decode", the pass's rows) moves to
core/lanes/solo_chunk.zig, backend-free; Flash Next's TF_FLASHNEXT_CHUNK_SOLO and GLM's
TF_GLM_CHUNK_SOLO call it in place of their own copies.

GB10, gemma-4-31b-it-4bit, --parallel 3, cold prompts, medians of 6 (A-B-B-A):
3.9k 4.225 -> 4.041 s, 15.6k 19.079 -> 18.081 s, 29.2k 40.844 -> 38.751 s (-4..-5 %).
Decode gaps beside a filling 15.6k prompt: 0.6817 -> 0.6824 s mean. 4096 gains no more
and takes 1.6 GiB more a stream than 2048. Replies byte-equal to main on every arm.
---
 CHANGELOG.md                               |  1 +
 zig/src/core/lanes/lanes.zig               |  1 +
 zig/src/core/lanes/solo_chunk.zig          | 55 ++++++++++++++++++++++
 zig/src/families/flashnext/cuda_chunk.zig  | 24 ++++------
 zig/src/families/flashnext/cuda_lanes.zig  |  2 +-
 zig/src/families/gemma4/cuda.zig           |  1 +
 zig/src/families/gemma4/cuda_assistant.zig |  2 +-
 zig/src/families/gemma4/cuda_backend.zig   |  7 ++-
 zig/src/families/gemma4/cuda_forward.zig   | 33 ++++++++-----
 zig/src/families/gemma4/cuda_native.zig    |  6 ++-
 zig/src/families/gemma4/cuda_snap.zig      | 24 ++++++++++
 zig/src/families/gemma4/cuda_solo.zig      | 18 +++++++
 zig/src/families/glm/cuda_prompt_solo.zig  |  4 +-
 13 files changed, 143 insertions(+), 35 deletions(-)
 create mode 100644 zig/src/core/lanes/solo_chunk.zig
 create mode 100644 zig/src/families/gemma4/cuda_solo.zig

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7e5712b11..994b4f423e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
 - Internal: HIP record counts are pinned in their owning arrays, so additions in different record files no longer edit a shared numeric total; omissions still fail host checks (#2461).
 - Qwen3.8 Flash Next on CUDA: BF16 prompt projections take a 128 × 128 tile on sm_12x wherever it fills the GPU, with the same output bytes, so prompts are 4 to 6% faster on a GB10 (NVFP4 checkpoint) and replies are unchanged. `TF_FLASHNEXT_PROMPT_BF16=0` turns the tile off, and `TF_FLASHNEXT_PROMPT_ROWS` is gone (#1846, refs #1754). Thanks to @laudney.
 - Internal: `compile-cuda-tests` builds again: `contract_controls.cu` declares the prompt-row fragment's page settings (`P256`, `IDS`, `PN`), which its control kernel predated (#2463).
+- Gemma 4 on CUDA: `TF_GEMMA4_CHUNK_SOLO=2048` lets a prompt pass take 2,048 rows while no other stream decodes, and the usual 512 once one does. On a GB10 with the 31B 4-bit release, a cold prompt's first token comes 4-5% sooner at 4K, 16K and 29K tokens with the same replies, and decode beside a filling prompt keeps its pace; each stream's sliding rings take 1.2 GiB more. Unset, nothing changes. The rule now lives in `lanes.solo_chunk`, which Flash Next's `TF_FLASHNEXT_CHUNK_SOLO` and GLM's `TF_GLM_CHUNK_SOLO` also call.
 
 - Qwen3.5 and Qwen3.6 on CUDA: a stream's rollback copy shares its verify log's bytes, which are never in use together, so each pooled stream takes 147 MiB less on Qwen3.8-27B and 61 MiB less on the 35B-A3B; tokens and speed unchanged (#2447, part of #1943). Thanks to @jschmied.
 - LightOnOCR-3 on HIP: the native server reads `lightonai/LightOnOCR-3-0.8B`, `-1B` and `-4B` (BF16) and serves their
diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig
index 1aa939fe95..b394f681d5 100644
--- a/zig/src/core/lanes/lanes.zig
+++ b/zig/src/core/lanes/lanes.zig
@@ -12,6 +12,7 @@ pub const recall_store = @import("recall_store.zig");
 pub const sampling = @import("sampling.zig");
 pub const fill = @import("fill.zig");
 pub const prefill_widths = @import("prefill_widths.zig");
+pub const solo_chunk = @import("solo_chunk.zig");
 pub const shape = @import("shape.zig");
 pub const plan_lanes = @import("plan_lanes.zig");
 pub const cost_cache = @import("cost_cache.zig");
diff --git a/zig/src/core/lanes/solo_chunk.zig b/zig/src/core/lanes/solo_chunk.zig
new file mode 100644
index 0000000000..4c4e092e43
--- /dev/null
+++ b/zig/src/core/lanes/solo_chunk.zig
@@ -0,0 +1,55 @@
+//! Solo prompt passes (#1554 C), backend-free: wider chunks while no other stream decodes, the busy chunk else.
+const std = @import("std");
+
+/// A family's limits: its busy chunk, its widest solo pass, the grid a solo pass keeps (1: any row count).
+pub const Limits = struct { busy: usize, most: usize, grid: usize = 1 };
+
+/// A solo value's rows: unset, `l.busy` (off); `l.busy` to `l.most` on `l.grid`, the rest error.BadChunkSolo.
+pub fn parse(v: ?[]const u8, l: Limits) !usize {
+    const s = v orelse return l.busy;
+    const n = std.fmt.parseInt(usize, s, 10) catch return error.BadChunkSolo;
+    if (n < l.busy or n > l.most or n % l.grid != 0) return error.BadChunkSolo;
+    return n;
+}
+
+/// The named variable's solo rows (parse).
+pub fn rows(name: [:0]const u8, l: Limits) !usize {
+    return parse(if (std.c.getenv(name)) |v| std.mem.span(v) else null, l);
+}
+
+/// Whether a stream besides `own` is past its prompt pass: its lane's `fill` field (where its pass stands) is null.
+pub fn othersDecode(lanes: anytype, own: anytype, comptime fill: []const u8) bool {
+    var it = lanes.iterator();
+    while (it.next()) |kv| if (kv.key_ptr.* != own and @field(kv.value_ptr.*, fill) == null) return true;
+    return false;
+}
+
+/// The rows the pass of stream `own` takes as it starts: `solo` unless another stream decodes, then `busy`.
+pub fn passRows(lanes: anytype, own: anytype, comptime fill: []const u8, solo: usize, busy: usize) usize {
+    if (solo == busy) return busy;
+    return if (othersDecode(lanes, own, fill)) busy else solo;
+}
+
+test "a solo value is off unset, takes busy to most on the grid and refuses the rest" {
+    const l: Limits = .{ .busy = 512, .most = 4096, .grid = 512 };
+    try std.testing.expectEqual(@as(usize, 512), try parse(null, l));
+    try std.testing.expectEqual(@as(usize, 2048), try parse("2048", l));
+    try std.testing.expectEqual(@as(usize, 4096), try parse("4096", l));
+    const bad = [_][]const u8{ "0", "256", "1000", "4608", "-1", "2k", "" };
+    for (bad) |v| try std.testing.expectError(error.BadChunkSolo, parse(v, l));
+    try std.testing.expectEqual(@as(usize, 1537), try parse("1537", .{ .busy = 1536, .most = 16384 }));
+}
+
+test "a pass takes the solo rows unless another stream decodes; prompts still filling do not count" {
+    const L = struct { fill: ?usize };
+    var m: std.AutoHashMapUnmanaged(u32, L) = .empty;
+    defer m.deinit(std.testing.allocator);
+    try m.put(std.testing.allocator, 0, .{ .fill = 0 });
+    try std.testing.expectEqual(@as(usize, 2048), passRows(&m, 0, "fill", 2048, 512));
+    try m.put(std.testing.allocator, 1, .{ .fill = 1024 });
+    try std.testing.expectEqual(@as(usize, 2048), passRows(&m, 0, "fill", 2048, 512));
+    m.getPtr(1).?.fill = null;
+    try std.testing.expectEqual(@as(usize, 512), passRows(&m, 0, "fill", 2048, 512));
+    try std.testing.expectEqual(@as(usize, 2048), passRows(&m, 1, "fill", 2048, 512)); // its own lane never counts
+    try std.testing.expectEqual(@as(usize, 512), passRows(&m, 0, "fill", 512, 512));
+}
diff --git a/zig/src/families/flashnext/cuda_chunk.zig b/zig/src/families/flashnext/cuda_chunk.zig
index 5b71259714..e30bcdb67a 100644
--- a/zig/src/families/flashnext/cuda_chunk.zig
+++ b/zig/src/families/flashnext/cuda_chunk.zig
@@ -1,6 +1,7 @@
 //! Prompt rows a pass: TF_FLASHNEXT_CHUNK while another stream decodes, TF_FLASHNEXT_CHUNK_SOLO when none does.
 const std = @import("std");
 const batch = @import("cuda_batch.zig");
+const solo_chunk = @import("lanes").solo_chunk;
 
 /// TF_FLASHNEXT_CHUNK's rows a pass: unset, the default; otherwise 1 to batch.max_chunk, anything else refused.
 pub fn parseChunk(v: ?[]const u8) !usize {
@@ -19,22 +20,17 @@ pub const lone_rows: usize = 8192;
 
 /// TF_FLASHNEXT_CHUNK_SOLO's rows: unset, `lone_rows` when `lone`, else `busy` (off); below `busy`, refused.
 pub fn parseSolo(v: ?[]const u8, busy: usize, lone: bool) !usize {
-    const s = v orelse return if (lone) @max(busy, lone_rows) else busy;
-    const n = parseChunk(s) catch return error.BadChunk;
-    if (n < busy) return error.BadChunk;
-    return n;
+    if (v == null and lone) return @max(busy, lone_rows);
+    return solo_chunk.parse(v, .{ .busy = busy, .most = batch.max_chunk }) catch error.BadChunk;
 }
 
 pub fn soloRows(busy: usize, lone: bool) !usize {
     return parseSolo(if (std.c.getenv("TF_FLASHNEXT_CHUNK_SOLO")) |v| std.mem.span(v) else null, busy, lone);
 }
 
-/// The rows a pass of `own` takes when it starts: `solo` unless another lane's prompt is in (it decodes).
+/// The rows the pass of stream `own` takes when it starts: lanes.solo_chunk's rule on the lanes' `filled`.
 pub fn passRows(lanes: anytype, own: anytype, solo: usize, busy: usize) usize {
-    if (solo == busy) return busy;
-    var it = lanes.valueIterator();
-    while (it.next()) |l| if (l != own and l.filled == null) return busy;
-    return solo;
+    return solo_chunk.passRows(lanes, own, "filled", solo, busy);
 }
 
 var readout_rows: ?usize = null;
@@ -73,12 +69,10 @@ test "a pass takes the solo rows unless another lane decodes; prompts still fill
     var m: std.AutoHashMapUnmanaged(u32, L) = .empty;
     defer m.deinit(std.testing.allocator);
     try m.put(std.testing.allocator, 0, .{ .filled = 0 });
-    const own = m.getPtr(0).?;
-    try std.testing.expectEqual(@as(usize, 8192), passRows(&m, own, 8192, 1536));
+    try std.testing.expectEqual(@as(usize, 8192), passRows(&m, 0, 8192, 1536));
     try m.put(std.testing.allocator, 1, .{ .filled = 3072 });
-    const mine = m.getPtr(0).?;
-    try std.testing.expectEqual(@as(usize, 8192), passRows(&m, mine, 8192, 1536));
+    try std.testing.expectEqual(@as(usize, 8192), passRows(&m, 0, 8192, 1536));
     m.getPtr(1).?.filled = null;
-    try std.testing.expectEqual(@as(usize, 1536), passRows(&m, m.getPtr(0).?, 8192, 1536));
-    try std.testing.expectEqual(@as(usize, 1536), passRows(&m, m.getPtr(0).?, 1536, 1536));
+    try std.testing.expectEqual(@as(usize, 1536), passRows(&m, 0, 8192, 1536));
+    try std.testing.expectEqual(@as(usize, 1536), passRows(&m, 0, 1536, 1536));
 }
diff --git a/zig/src/families/flashnext/cuda_lanes.zig b/zig/src/families/flashnext/cuda_lanes.zig
index 7afdb74f6c..ee6c4ff3d0 100644
--- a/zig/src/families/flashnext/cuda_lanes.zig
+++ b/zig/src/families/flashnext/cuda_lanes.zig
@@ -338,7 +338,7 @@ pub fn Namespace(comptime Tri: type) type {
                     s.keepInterruptedPrefill(@intCast(at));
                     return error.Cancelled;
                 }
-                const rows = @import("cuda_chunk.zig").passRows(&self.lanes, l, self.engine.chunk, self.engine.busy_chunk);
+                const rows = @import("cuda_chunk.zig").passRows(&self.lanes, s, self.engine.chunk, self.engine.busy_chunk);
                 var marks: [prompt.batch.max_marks]prompt.batch.Mark = .{ .{}, .{} };
                 defer snaps.dropPrepared(self, l);
                 const cut = snaps.cut(self, s, l, ids, at, @min(ids.len, at + rows), &marks);
diff --git a/zig/src/families/gemma4/cuda.zig b/zig/src/families/gemma4/cuda.zig
index f090439e78..6e54624e1c 100644
--- a/zig/src/families/gemma4/cuda.zig
+++ b/zig/src/families/gemma4/cuda.zig
@@ -48,6 +48,7 @@ test {
     _ = @import("cuda_forward.zig");
     _ = @import("cuda_snap.zig");
     _ = @import("cuda_backend.zig");
+    _ = @import("cuda_solo.zig");
     _ = @import("cuda_shared.zig");
     _ = @import("cuda_assistant.zig");
     _ = @import("cuda_assistant_weights.zig");
diff --git a/zig/src/families/gemma4/cuda_assistant.zig b/zig/src/families/gemma4/cuda_assistant.zig
index 88b88a39b3..8139df0ccd 100644
--- a/zig/src/families/gemma4/cuda_assistant.zig
+++ b/zig/src/families/gemma4/cuda_assistant.zig
@@ -79,7 +79,7 @@ pub const Drafter = struct {
         const hid = c.hidden;
         const qw = c.heads * cfg.global_dim;
         const wide: Window = .{ .h = c.heads, .hk = geo.global_kv_heads, .d = cfg.global_dim, .win = 0, .ring = m.context, .scale = 1 };
-        const narrow: Window = .{ .h = c.heads, .hk = geo.sliding_kv_heads, .d = cfg.sliding_dim, .win = cfg.window, .ring = fwd.ring, .scale = 1 };
+        const narrow: Window = .{ .h = c.heads, .hk = geo.sliding_kv_heads, .d = cfg.sliding_dim, .win = cfg.window, .ring = m.ringRows(), .scale = 1 };
         const part = @max(cuda.window_attention.WindowAttention.scratchBytes(wide, 16), cuda.window_attention.WindowAttention.scratchBytes(narrow, 16));
         x.x = .{
             .ids = try x.alloc(n * 4),
diff --git a/zig/src/families/gemma4/cuda_backend.zig b/zig/src/families/gemma4/cuda_backend.zig
index 3541cb5d58..1781b440db 100644
--- a/zig/src/families/gemma4/cuda_backend.zig
+++ b/zig/src/families/gemma4/cuda_backend.zig
@@ -337,12 +337,15 @@ pub const Gemma = struct {
         return k.at;
     }
 
-    /// One chunk from `l.fill` to the next grid line, image or mark; true once the prompt's last row is in. `_: u32`.
+    /// One chunk from `l.fill` to a grid line, image or mark; true once the prompt's last row is in. `_: u32`.
     fn chunk(x: *Gemma, s: *Stream, l: *Lane, streamed: bool, _: u32) !bool {
         const p = s.prompt();
         const at = l.fill.?;
         if (s.isCancelled()) return error.Cancelled;
-        var end = @min(p.len, (at / prompt_rows + 1) * prompt_rows);
+        // solo passes (TF_GEMMA4_CHUNK_SOLO) only without a drafter's taps, at most `injects` images a pass
+        const solo = lanes.solo_chunk.passRows(&x.lanes, s, "fill", x.m.prompt_most, prompt_rows);
+        const rows = if (x.tapped or s.images.len > 8) prompt_rows else solo;
+        var end = @min(p.len, at / prompt_rows * prompt_rows + rows);
         for (if (l.shared) l.marks else s.reuse.marks) |mk| if (mk > at and mk < end and !insideImage(s, mk)) {
             end = mk;
         };
diff --git a/zig/src/families/gemma4/cuda_forward.zig b/zig/src/families/gemma4/cuda_forward.zig
index ed4938890b..522a3f89ec 100644
--- a/zig/src/families/gemma4/cuda_forward.zig
+++ b/zig/src/families/gemma4/cuda_forward.zig
@@ -5,6 +5,7 @@ const rope_freq = @import("core").rope_freq;
 const cfg = @import("cuda_config.zig");
 const wts = @import("cuda_weights.zig");
 const kern = @import("cuda_kernels.zig");
+const solo = @import("cuda_solo.zig");
 
 const DeviceBuffer = cuda.DeviceBuffer;
 const Window = cuda.window_attention.Window;
@@ -16,7 +17,7 @@ pub const prompt_rows = 512;
 pub const round_rows = 128;
 /// Rows a decode attention call takes: its partials' scratch is sized for this many (Metal's attend_rows).
 const attend_rows = 16;
-/// A sliding layer's ring: the window and a prompt chunk, so a chunk's writes never reach keys its first row reads.
+/// A sliding layer's ring: window + chunk, so a chunk never overwrites keys its first row reads (solo: ringRows).
 pub const ring = cfg.window + prompt_rows;
 
 /// A stream's rows in one forward: its cache slot, first position, tokens.
@@ -58,6 +59,7 @@ pub const Model = struct {
     s: Scratch,
     buffers: std.ArrayList(DeviceBuffer) = .empty,
     planned: ?usize = null, // a plan: the bytes counted, nothing allocated
+    prompt_most: usize = prompt_rows, // rows a prompt chunk takes at most: TF_GEMMA4_CHUNK_SOLO's, else prompt_rows
     /// Decode forwards of n rows as CUDA graphs, captured at their first use ([n]; null: not yet, or graphs off).
     graphs: [round_rows + 1]?cuda.graph.Exec = @splat(null),
     graphed: bool = true,
@@ -93,7 +95,12 @@ pub const Model = struct {
 
     /// A layer's cache positions: the ring, or every position of the context (global layers).
     pub fn cacheLen(m: *const Model, i: usize) usize {
-        return if (cfg.global(i)) m.context else ring;
+        return if (cfg.global(i)) m.context else m.ringRows();
+    }
+
+    /// A sliding layer's ring rows: the window and the widest prompt chunk.
+    pub fn ringRows(m: *const Model) usize {
+        return cfg.window + m.prompt_most;
     }
 
     pub fn pagesFor(m: *const Model, slot: ?usize) ?cuda.window_attention.Pages {
@@ -107,10 +114,10 @@ pub const Model = struct {
         };
     }
 
-    /// Bytes one stream's caches take: the owning layers' (a shared layer reads its owner's).
-    pub fn streamBytes(geo: cfg.Geometry, layers: usize, context: usize) usize {
+    /// Bytes one stream's caches take: the owning layers' (a shared layer reads its owner's), rings `rings` rows each.
+    pub fn streamBytes(geo: cfg.Geometry, layers: usize, context: usize, rings: usize) usize {
         var total: usize = 0;
-        for (0..@min(layers, geo.cacheLayers())) |i| total += 2 * (if (cfg.global(i)) context else ring) * geo.kvHeads(i) * cfg.headDim(i) * 2;
+        for (0..@min(layers, geo.cacheLayers())) |i| total += 2 * (if (cfg.global(i)) context else rings) * geo.kvHeads(i) * cfg.headDim(i) * 2;
         return total;
     }
 
@@ -121,6 +128,7 @@ pub const Model = struct {
     pub fn initPaged(gpa: std.mem.Allocator, d: *const cuda.Driver, s: cuda.Stream, w: *const wts.Weights, context: usize, slots: usize, paged: bool) !Model {
         var m = blank(gpa, d, w, context, if (paged) 1 else slots);
         m.paged = paged;
+        m.prompt_most = try solo.rows(); // refused before anything allocates
         errdefer m.deinit();
         try m.build(s);
         return m;
@@ -134,6 +142,7 @@ pub const Model = struct {
     pub fn openPagedBytes(gpa: std.mem.Allocator, d: *const cuda.Driver, s: cuda.Stream, w: *const wts.Weights, context: usize, slots: usize, paged: bool) !usize {
         var m = blank(gpa, d, w, context, if (paged) 1 else slots);
         m.paged = paged;
+        m.prompt_most = try solo.rows();
         defer m.deinit();
         m.planned = 0;
         try m.build(s);
@@ -159,18 +168,18 @@ pub const Model = struct {
             try (DeviceBuffer{ .d = m.d, .ptr = m.inv_sliding, .len = sl.len * 4, .borrowed = true }).upload(0, std.mem.sliceAsBytes(&sl));
             try (DeviceBuffer{ .d = m.d, .ptr = m.inv_global, .len = gl.len * 4, .borrowed = true }).upload(0, std.mem.sliceAsBytes(&gl));
         }
-        const r = @max(prompt_rows, round_rows);
+        const r = @max(m.prompt_most, round_rows);
         const h = geo.hidden;
         const qw = geo.queryWidth();
         const kw = geo.kvWidth();
         const most = @max(h, geo.intermediate);
         // the widest decode partials: attend_rows rows of a global layer over the context in 256-key chunks
         const wide: Window = .{ .h = geo.query_heads, .hk = geo.global_kv_heads, .d = cfg.global_dim, .win = 0, .ring = m.context, .scale = 1 };
-        const narrow: Window = .{ .h = geo.query_heads, .hk = geo.sliding_kv_heads, .d = cfg.sliding_dim, .win = geo.sliding_window, .ring = ring, .scale = 1 };
+        const narrow: Window = .{ .h = geo.query_heads, .hk = geo.sliding_kv_heads, .d = cfg.sliding_dim, .win = geo.sliding_window, .ring = m.ringRows(), .scale = 1 };
         const part = @max(cuda.window_attention.WindowAttention.scratchBytes(wide, attend_rows), cuda.window_attention.WindowAttention.scratchBytes(narrow, attend_rows));
         m.s = .{
             .ids = try m.alloc(r * 4),
-            .hi = try m.alloc(prompt_rows * 4),
+            .hi = try m.alloc(m.prompt_most * 4),
             .pos = try m.alloc(r * 4),
             .slot = try m.alloc(r * 4),
             .h = .{ try m.alloc(r * h * 2), try m.alloc(r * h * 2) },
@@ -275,7 +284,7 @@ pub const Model = struct {
             if (m.paged and c.p0 + c.tokens.len > m.caps[c.slot]) return error.PastPages;
             rows += c.tokens.len;
         }
-        if (rows > @as(usize, if (prompt) prompt_rows else round_rows) or (prompt and chains.len != 1)) return error.BadWindow;
+        if (rows > @as(usize, if (prompt) m.prompt_most else round_rows) or (prompt and chains.len != 1)) return error.BadWindow;
         if (heads > round_rows or head0 + heads > rows) return error.BadWindow;
         const ids = try m.gpa.alloc(i32, 3 * rows);
         defer m.gpa.free(ids);
@@ -428,9 +437,9 @@ test "a stream's caches count the owning layers only: the E4B's 24 of 42" {
     const g = cfg.e4b;
     var own: usize = 0;
     for (0..g.cacheLayers()) |i| own += 2 * (if (cfg.global(i)) @as(usize, 4096) else ring) * g.kvHeads(i) * cfg.headDim(i) * 2;
-    try std.testing.expectEqual(own, Model.streamBytes(g, g.layers, 4096));
-    try std.testing.expectEqual(Model.streamBytes(g, 24, 4096), Model.streamBytes(g, 42, 4096));
-    try std.testing.expect(Model.streamBytes(g, 23, 4096) < Model.streamBytes(g, 24, 4096));
+    try std.testing.expectEqual(own, Model.streamBytes(g, g.layers, 4096, ring));
+    try std.testing.expectEqual(Model.streamBytes(g, 24, 4096, ring), Model.streamBytes(g, 42, 4096, ring));
+    try std.testing.expect(Model.streamBytes(g, 23, 4096, ring) < Model.streamBytes(g, 24, 4096, ring));
     try std.testing.expectEqual(@as(?usize, 22), cfg.cacheOwner(40, g.cacheLayers()));
     try std.testing.expectEqual(@as(?usize, 23), cfg.cacheOwner(41, g.cacheLayers()));
 }
diff --git a/zig/src/families/gemma4/cuda_native.zig b/zig/src/families/gemma4/cuda_native.zig
index 4365bfc90d..f9e7d135b6 100644
--- a/zig/src/families/gemma4/cuda_native.zig
+++ b/zig/src/families/gemma4/cuda_native.zig
@@ -158,6 +158,9 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [
     };
     own.m = try fwd.Model.initPaged(gpa, ctx.d, own.s, &own.w, context, n, paged);
     errdefer own.m.deinit();
+    const most = own.m.prompt_most;
+    const busy = fwd.prompt_rows;
+    if (most != busy) std.log.info("gemma4: prompt passes of {d} rows, {d} beside a decode", .{ most, busy });
     own.m.graphed = !envOff("TENSORFOLD_GEMMA4_GRAPHS"); // =0: every decode forward eager
     own.gb = try be.Gemma.init(gpa, ctx.d, &own.m, &own.k, own.s);
     own.gb.priority_copies = priority_copies;
@@ -173,7 +176,7 @@ pub fn open(gpa: std.mem.Allocator, io: std.Io, ctx: *const cuda.Context, dir: [
         .backend = own.gb.backend(),
         .facts = own.gb.facts(),
         .rows = be.max_window,
-        .stream_bytes = fwd.Model.streamBytes(own.w.config.geo, own.w.layers.len, context),
+        .stream_bytes = fwd.Model.streamBytes(own.w.config.geo, own.w.layers.len, context, own.m.ringRows()),
         .free_streams = @intCast(n),
         .ctx = own,
         .deinit = release,
@@ -266,6 +269,7 @@ pub fn explain(_: ?*anyopaque, err: anyerror) ?[]const u8 {
         error.ParallelPastRound => std.fmt.comptimePrint("--parallel: Gemma 4 on CUDA holds at most {d} streams (a round's rows): serve --parallel {d} or fewer, or auto", .{ be.round_rows, be.round_rows }),
         error.ParallelPastMemory => "--parallel: the CUDA memory budget fits fewer Gemma 4 streams than asked (each stream's caches are allocated at load): lower --parallel or --context, serve --parallel auto, or adjust TENSORFOLD_MEMORY_RESERVE_GIB / TENSORFOLD_CUDA_MEMORY_LIMIT_GB",
         error.DrafterPastMemory => "--drafter: the CUDA memory budget leaves less room than the drafter's weights need after the streams: lower --parallel or --context, or adjust TENSORFOLD_MEMORY_RESERVE_GIB / TENSORFOLD_CUDA_MEMORY_LIMIT_GB",
+        error.BadChunkSolo => "TF_GEMMA4_CHUNK_SOLO: a solo prompt pass's rows, a multiple of 512 up to 4096",
         error.NoStreamFits => "the CUDA memory budget fits no Gemma 4 stream after the weights: lower --context, free device memory, or adjust TENSORFOLD_MEMORY_RESERVE_GIB / TENSORFOLD_CUDA_MEMORY_LIMIT_GB",
         else => null,
     };
diff --git a/zig/src/families/gemma4/cuda_snap.zig b/zig/src/families/gemma4/cuda_snap.zig
index afdddd0154..fca535a0a1 100644
--- a/zig/src/families/gemma4/cuda_snap.zig
+++ b/zig/src/families/gemma4/cuda_snap.zig
@@ -165,6 +165,7 @@ test "a state holds global layers' positions and each sliding ring's last window
     m.layers = 6; // layer 5 is global
     m.context = 4096;
     m.paged = false;
+    m.prompt_most = fwd.prompt_rows;
     m.kc = &k;
     m.vc = &v;
     const at = 2000; // sliding: positions 977..1999, ring rows 977..1535 then 0..463
@@ -182,6 +183,28 @@ test "a state holds global layers' positions and each sliding ring's last window
     try testing.expectEqual([3]usize{ (12 << 30) + 4096 * gb, 2 * 5 * 1023 * sb + at * gb, at * gb }, last);
 }
 
+test "a solo model's wider ring holds the same window - 1 positions, unwrapped where the grid ring wraps" {
+    var k = [_]u64{ 1 << 30, 2 << 30, 3 << 30, 4 << 30, 5 << 30, 6 << 30 };
+    var v = [_]u64{ 7 << 30, 8 << 30, 9 << 30, 10 << 30, 11 << 30, 12 << 30 };
+    var m: fwd.Model = undefined;
+    m.geo = cfg.unified12b;
+    m.layers = 6;
+    m.context = 4096;
+    m.paged = false;
+    m.prompt_most = 2048; // ring rows 3072: positions 977..1999 sit in rows 977..1999
+    m.kc = &k;
+    m.vc = &v;
+    const at = 2000;
+    const sb = 8 * 256 * 2;
+    const gb = 1 * 512 * 2;
+    try testing.expectEqual(@as(u64, 5 * 2 * 1023 * sb + 2 * at * gb), bytes(&m, at));
+    var runs: std.ArrayList([3]usize) = .empty;
+    defer runs.deinit(testing.allocator);
+    try each(&m, 1, at, 0, Count{ .runs = &runs }, Count.add);
+    try testing.expectEqual(@as(usize, 5 * 2 + 2), runs.items.len);
+    try testing.expectEqual([3]usize{ (1 << 30) + (3072 + 977) * sb, 0, 1023 * sb }, runs.items[0]);
+}
+
 test "a paged state takes a global layer's positions through the slot's pages, a run a page, in position order" {
     var k = [_]u64{ 1 << 30, 2 << 30, 3 << 30, 4 << 30, 5 << 30, 6 << 30 };
     var v = [_]u64{ 7 << 30, 8 << 30, 9 << 30, 10 << 30, 11 << 30, 12 << 30 };
@@ -191,6 +214,7 @@ test "a paged state takes a global layer's positions through the slot's pages, a
     m.layers = 6; // layer 5 is global
     m.context = 4 * fwd.pages.tokens;
     m.paged = true;
+    m.prompt_most = fwd.prompt_rows;
     m.ids = &ids;
     m.kc = &k;
     m.vc = &v;
diff --git a/zig/src/families/gemma4/cuda_solo.zig b/zig/src/families/gemma4/cuda_solo.zig
new file mode 100644
index 0000000000..3d7b01be54
--- /dev/null
+++ b/zig/src/families/gemma4/cuda_solo.zig
@@ -0,0 +1,18 @@
+//! TF_GEMMA4_CHUNK_SOLO: whole grid chunks a solo prompt pass takes, on lanes.solo_chunk's rule.
+const std = @import("std");
+const solo_chunk = @import("lanes").solo_chunk;
+
+/// The grid chunk (a pass beside a decode), and the widest solo pass: the rings and the prompt scratch grow with it.
+pub const limits: solo_chunk.Limits = .{ .busy = grid, .most = 4096, .grid = grid };
+const grid = @import("cuda_forward.zig").prompt_rows;
+
+pub fn rows() !usize {
+    return solo_chunk.rows("TF_GEMMA4_CHUNK_SOLO", limits);
+}
+
+test "TF_GEMMA4_CHUNK_SOLO takes whole grid chunks up to 4096" {
+    try std.testing.expectEqual(@as(usize, 512), try solo_chunk.parse(null, limits));
+    try std.testing.expectEqual(@as(usize, 2048), try solo_chunk.parse("2048", limits));
+    try std.testing.expectError(error.BadChunkSolo, solo_chunk.parse("1000", limits));
+    try std.testing.expectError(error.BadChunkSolo, solo_chunk.parse("8192", limits));
+}
diff --git a/zig/src/families/glm/cuda_prompt_solo.zig b/zig/src/families/glm/cuda_prompt_solo.zig
index b280f4559c..2652b2ff07 100644
--- a/zig/src/families/glm/cuda_prompt_solo.zig
+++ b/zig/src/families/glm/cuda_prompt_solo.zig
@@ -6,9 +6,7 @@ const Engine = engine.Engine;
 
 /// Whether a stream besides `s` is past its prompt pass (a lane with no fill): a solo pass would hold up its rounds.
 pub fn othersDecode(lanes: anytype, s: anytype) bool {
-    var it = lanes.iterator();
-    while (it.next()) |kv| if (kv.key_ptr.* != s and kv.value_ptr.fill == null) return true;
-    return false;
+    return @import("lanes").solo_chunk.othersDecode(lanes, s, "fill");
 }
 
 /// The rows an engine sizes its prompt buffers for: at start, as they hold memory all the time.

From ee94c0ab8dd469c55a569448077f759dba1a684a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=BCrgen=20Schmied?=
 <600316+jschmied@users.noreply.github.com>
Date: Fri, 6 Nov 2026 21:54:12 +0100
Subject: [PATCH 2/2] docs: gemma4 solo chunks in the ledger, and one constant
 for a pass's image slots

---
 CHANGELOG.md                             | 1 +
 docs/optimization-ledger.md              | 4 +++-
 zig/src/families/gemma4/cuda_backend.zig | 5 +++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 994b4f423e..203fc96bdf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
 - Qwen3.8 Flash Next on CUDA: BF16 prompt projections take a 128 × 128 tile on sm_12x wherever it fills the GPU, with the same output bytes, so prompts are 4 to 6% faster on a GB10 (NVFP4 checkpoint) and replies are unchanged. `TF_FLASHNEXT_PROMPT_BF16=0` turns the tile off, and `TF_FLASHNEXT_PROMPT_ROWS` is gone (#1846, refs #1754). Thanks to @laudney.
 - Internal: `compile-cuda-tests` builds again: `contract_controls.cu` declares the prompt-row fragment's page settings (`P256`, `IDS`, `PN`), which its control kernel predated (#2463).
 - Gemma 4 on CUDA: `TF_GEMMA4_CHUNK_SOLO=2048` lets a prompt pass take 2,048 rows while no other stream decodes, and the usual 512 once one does. On a GB10 with the 31B 4-bit release, a cold prompt's first token comes 4-5% sooner at 4K, 16K and 29K tokens with the same replies, and decode beside a filling prompt keeps its pace; each stream's sliding rings take 1.2 GiB more. Unset, nothing changes. The rule now lives in `lanes.solo_chunk`, which Flash Next's `TF_FLASHNEXT_CHUNK_SOLO` and GLM's `TF_GLM_CHUNK_SOLO` also call.
+- Gemma 4 on CUDA: `TF_GEMMA4_CHUNK_SOLO=2048` lets a prompt pass take 2,048 rows while no other stream decodes, and the usual 512 once one does. On a GB10 with the 31B 4-bit release, a cold prompt's first token comes 4-5% sooner at 4K, 16K and 29K tokens with the same replies, and decode beside a filling prompt keeps its pace; each stream's sliding rings take 1.2 GiB more. Unset, nothing changes. The rule now lives in `lanes.solo_chunk`, which Flash Next's `TF_FLASHNEXT_CHUNK_SOLO` and GLM's `TF_GLM_CHUNK_SOLO` also call (#2391).
 
 - Qwen3.5 and Qwen3.6 on CUDA: a stream's rollback copy shares its verify log's bytes, which are never in use together, so each pooled stream takes 147 MiB less on Qwen3.8-27B and 61 MiB less on the 35B-A3B; tokens and speed unchanged (#2447, part of #1943). Thanks to @jschmied.
 - LightOnOCR-3 on HIP: the native server reads `lightonai/LightOnOCR-3-0.8B`, `-1B` and `-4B` (BF16) and serves their
diff --git a/docs/optimization-ledger.md b/docs/optimization-ledger.md
index be594034bd..569117ee84 100644
--- a/docs/optimization-ledger.md
+++ b/docs/optimization-ledger.md
@@ -64,7 +64,7 @@ last two hold the families with only a Metal, HIP or scoring backend.
 | 17. DSpark | yes | yes | yes | yes | yes |
 | 18. Copy drafts | done [#1477] | done [#468] | done [#2146] | done [#2022] | done [#1974] |
 | 19. Shared drafts | done [#1562] | no (null [#1762]) | done [#1779] | done [#1779] | done [#768] |
-| 20. Solo chunks | yes | done C[#1733] M[#2167] | done [#1823] | yes | yes |
+| 20. Solo chunks | yes | done C[#1733] M[#2167] | done [#1823] | done C[#2391] | yes |
 | 21. Read-ahead | done [#1928] | no (row 22) | done [#1616] | done [#1928] | done [#2033] |
 | 22. Direct load | done [#2282] | done [#1901] | yes | done [#2282] | done [#458] |
 | 23. Page drop | done [#1884] | done [#1880] | done [#600] | done [#1884] | done [#1884] |
@@ -374,6 +374,7 @@ below; these are estimates for planning a port, not results.
     | flashnext (M) | 1,024-token reply beside a 64K prompt 48 → 8 s, prompt's first token 90 → 54 s | [#2167] |
     | flashnext (C), one GPU | prompts 15–23 % faster at `--parallel 1`, up to 8,192 rows a pass | [#2246] |
     | glm (C), TP=2 | prompts about 2 % sooner at 8K–128K, two chunks a step, replies unchanged | [#1423] |
+    | gemma4 (C) | first token 31B −4..−5 %, E4B −9..−10 %, opt-in with `TF_GEMMA4_CHUNK_SOLO` | [#2391] |
 
     `TF_GLM_CHUNK_SOLO=4096` is slower with GLM's NVFP4 checkpoint on two GB10s ([#2288]).
 
@@ -551,3 +552,4 @@ below; these are estimates for planning a port, not results.
 [#2283]: https://github.com/ashhart/TensorFold/pull/2283
 [#2288]: https://github.com/ashhart/TensorFold/pull/2288
 [#2306]: https://github.com/ashhart/TensorFold/pull/2306
+[#2391]: https://github.com/ashhart/TensorFold/pull/2391
diff --git a/zig/src/families/gemma4/cuda_backend.zig b/zig/src/families/gemma4/cuda_backend.zig
index 1781b440db..c654f77d8a 100644
--- a/zig/src/families/gemma4/cuda_backend.zig
+++ b/zig/src/families/gemma4/cuda_backend.zig
@@ -22,6 +22,7 @@ pub const max_window = 16;
 pub const pagesFor = shared.pagesFor;
 /// Drawn tokens a handle names (a ring; read before the round loop could wrap it).
 const ring = 1024;
+const max_injects = 8; // images one prompt pass can take; a solo pass stays under it
 
 const Lane = struct {
     slot: usize,
@@ -344,7 +345,7 @@ pub const Gemma = struct {
         if (s.isCancelled()) return error.Cancelled;
         // solo passes (TF_GEMMA4_CHUNK_SOLO) only without a drafter's taps, at most `injects` images a pass
         const solo = lanes.solo_chunk.passRows(&x.lanes, s, "fill", x.m.prompt_most, prompt_rows);
-        const rows = if (x.tapped or s.images.len > 8) prompt_rows else solo;
+        const rows = if (x.tapped or s.images.len > max_injects) prompt_rows else solo;
         var end = @min(p.len, at / prompt_rows * prompt_rows + rows);
         for (if (l.shared) l.marks else s.reuse.marks) |mk| if (mk > at and mk < end and !insideImage(s, mk)) {
             end = mk;
@@ -353,7 +354,7 @@ pub const Gemma = struct {
         for (s.images) |im| if (im.at < end and end < im.at + im.tokens) {
             end = if (im.at > at) im.at else im.at + im.tokens;
         };
-        var injects: [8]fwd.Inject = undefined;
+        var injects: [max_injects]fwd.Inject = undefined;
         var n: usize = 0;
         var off: usize = 0;
         for (s.images) |im| {