From a055d6ff3c22abdec78e3140b8bc88b683a79b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Schmied?= <600316+jschmied@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:02:22 +0100 Subject: [PATCH 1/5] kv pool: a Metal lane adopts a kept prompt's pages; the engines' core names kv_pool and kv_prompts --- zig/build/drama3_metal.zig | 4 ++-- zig/src/core/metal_kv_pages.zig | 30 ++++++++++++++++++++++++++++++ zig/src/tensorfold.zig | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/zig/build/drama3_metal.zig b/zig/build/drama3_metal.zig index a74e027016..733e145447 100644 --- a/zig/build/drama3_metal.zig +++ b/zig/build/drama3_metal.zig @@ -4,7 +4,7 @@ const std = @import("std"); const dir = "zig/kernels/metal/drama3"; const sources = [_][]const u8{ "dense.metal", "gdn.metal", "attn.metal", "moe.metal", "nvfp4.metal", "guidance.metal", "sample.metal", "codec.metal" }; -/// Drama in the Metal engines: its `core` is the engine's safetensors, the server's tokenizer and frame_rounds. +/// Drama in the Metal engines: its `core` is the engine's safetensors and KV pool, the tokenizer and frame_rounds. pub fn addNative(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode, metal: *std.Build.Module, native: *std.Build.Module, metal_std: []const u8) void { const tokenizer = native.import_table.get("tokenizer") orelse tok: { const t = b.createModule(.{ .root_source_file = b.path("zig/src/core/tokenizer/tokenizer.zig"), .target = target, .optimize = .ReleaseSafe, .link_libc = true }); @@ -12,7 +12,7 @@ pub fn addNative(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std. break :tok t; }; const rounds = b.createModule(.{ .root_source_file = b.path("zig/src/core/frame_rounds.zig"), .target = target, .optimize = optimize }); - const shim = b.addWriteFiles().add("drama3_core.zig", "pub const safetensors = @import(\"tensorfold\").safetensors;\npub const tokenizer = @import(\"tokenizer\");\npub const frame_rounds = @import(\"frame_rounds\");\n"); + const shim = b.addWriteFiles().add("drama3_core.zig", "pub const safetensors = @import(\"tensorfold\").safetensors;\npub const kv_pool = @import(\"tensorfold\").kv_pool;\npub const kv_prompts = @import(\"tensorfold\").kv_prompts;\npub const tokenizer = @import(\"tokenizer\");\npub const frame_rounds = @import(\"frame_rounds\");\n"); const core = b.createModule(.{ .root_source_file = shim, .target = target, .optimize = optimize, .imports = &.{ .{ .name = "tensorfold", .module = native.import_table.get("tensorfold").? }, .{ .name = "tokenizer", .module = tokenizer }, diff --git a/zig/src/core/metal_kv_pages.zig b/zig/src/core/metal_kv_pages.zig index 9a28bb1729..bbbc5ffa1c 100644 --- a/zig/src/core/metal_kv_pages.zig +++ b/zig/src/core/metal_kv_pages.zig @@ -179,6 +179,17 @@ pub const Pages = struct { p.tables.set(lane, i, id); } + /// Lane `lane`'s table starts with a kept prompt's pages, held once more (TF_KV_SHARE); a failure releases them. + pub fn adopt(p: *Pages, lane: usize, held: []const u32) !void { + const old = p.leases[lane].ids.items.len; + if (old + held.len > p.tables.most) { + for (held) |id| p.pool.release(id); + return error.PastTable; + } + try p.pool.adopt(&p.leases[lane], held); + for (held, old..) |id, i| p.tables.set(lane, i, id); + } + /// Lane `lane` keeps the pages of its first `tokens` rows; those past them go back to its promise. pub fn trim(p: *Pages, lane: usize, tokens: u64) void { p.pool.trim(&p.leases[lane], p.layout.pages(tokens)); @@ -314,3 +325,22 @@ test "the kernels' address helper pages by TF_PAGE_TOKENS" { test { _ = @import("metal_kv_pages_test.zig"); } + +test "a kept prompt's pages head a new lane's table, held by both, and stay the store's when the stream ends" { + const p = try Pages.init(std.testing.allocator, null, testLayout(), 2, 6, 4 * page_tokens, 0); + defer p.deinit(); + var kept: [2]u32 = undefined; + try std.testing.expect(p.pool.hold(&kept)); // the store's two pages + for (kept) |id| p.pool.retain(id); // a resume holds them once more for the stream + try p.admit(1, 4 - kept.len); + try p.adopt(1, &kept); + try p.cover(1, 3 * page_tokens); // the stream's own pages past them, from its promise + try std.testing.expectEqualSlices(u32, &kept, p.tables.entries(1, 2)); + try std.testing.expectEqualSlices(u32, p.ids(1), p.tables.entries(1, 3)); + try std.testing.expectEqual(@as(u32, 2), p.pool.holders(kept[0])); + p.finish(1); + try std.testing.expect(p.idle()); // kept pages are the store's, not a lane's + try std.testing.expectEqual(@as(usize, 2), p.pool.census().kept); + for (kept) |id| try p.pool.drop(id); + try std.testing.expectEqual(@as(usize, 6), p.pool.census().free); +} diff --git a/zig/src/tensorfold.zig b/zig/src/tensorfold.zig index b4929097f9..a137d951d9 100644 --- a/zig/src/tensorfold.zig +++ b/zig/src/tensorfold.zig @@ -114,6 +114,7 @@ pub const recurrent_forks = @import("core/recurrent_forks.zig"); pub const shared_kv = @import("core/shared_kv.zig"); pub const kv_pool = @import("core/kv_pool.zig"); +pub const kv_prompts = @import("core/kv_prompts.zig"); pub const rope_freq = @import("core/rope_freq.zig"); From 3d8c0a98b91a38f99c997926690490a482f5e61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Schmied?= <600316+jschmied@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:02:22 +0100 Subject: [PATCH 2/5] nemotron: TF_KV_SHARE on Metal, prompt states kept as core pool pages (#1293) The core prompt_radix store over the Nemotron pool, opened through the CUDA adapter. A kept state is its Mamba states (Snap.pages); its rows stay in pages a later stream adopts. Marks lie on the prompt chunk grid, so a resume's chunk seams are a cold pass's. Unset or 0: the host prompt cache as before. The store's marks end chunks in prompt_flight.rows, where the prompt loop cuts them. metal.zig reads the model's window with cuda_window's modelContext instead of its own copy. --- zig/src/families/nemotron/backend.zig | 32 +++----- zig/src/families/nemotron/kv_pages.zig | 6 +- zig/src/families/nemotron/learned_prompts.zig | 4 +- zig/src/families/nemotron/prompt_flight.zig | 7 +- .../families/nemotron/prompt_flight_test.zig | 24 +++++- zig/src/families/nemotron/prompt_pages.zig | 81 +++++++++++++++++++ zig/src/families/nemotron/residency.zig | 10 +-- zig/src/families/nemotron/slide.zig | 16 ++++ zig/src/families/nemotron/snapshot.zig | 64 +++++++++++---- zig/src/native/cuda_window.zig | 2 +- zig/src/native/metal.zig | 31 ++++--- zig/src/native/nemotron_disk.zig | 2 +- 12 files changed, 217 insertions(+), 62 deletions(-) create mode 100644 zig/src/families/nemotron/prompt_pages.zig diff --git a/zig/src/families/nemotron/backend.zig b/zig/src/families/nemotron/backend.zig index f026411d51..0efe348ca4 100644 --- a/zig/src/families/nemotron/backend.zig +++ b/zig/src/families/nemotron/backend.zig @@ -9,7 +9,7 @@ const st = @import("state.zig"); const kv_pages = @import("kv_pages.zig"); const kvp = @import("../../core/metal_kv_pages.zig"); const lesson_route = @import("living").lesson_route; -const layers = @import("layers.zig"); +const prompt_pages = @import("prompt_pages.zig"); const mtp = @import("mtp.zig"); const timing = @import("timing.zig"); const tree = @import("tree.zig"); @@ -82,6 +82,8 @@ pub const Metal = struct { kv_tables: ?mtl.Buffer = null, // and every lane's page table kv_pages: ?*kvp.Pages = null, // the core pool over them: a stream leases a lane's pages when it opens kv_pipes: ?*kv_pages.Pipelines = null, // the paged twins (the model's kernels point at them) + prompts: ?@import("../../core/kv_prompts.zig").Prompts = null, // TF_KV_SHARE: the store keeping prompts as pages + pending: ?*const snapshot.Snap = null, // the store's resume, restored once the stream's lane adopts its pages pub fn init(gpa: std.mem.Allocator, m: *Model, o: Options) !*Metal { const b = try gpa.create(Metal); @@ -240,7 +242,10 @@ pub const Metal = struct { self.gpa.destroy(old); _ = self.caches.remove(s); } - const c = try @import("residency.zig").openCache(self, s, rows_needed); + var pass = try prompt_pages.Pass.begin(self, s); + var ok = false; + defer pass.end(self, s, ok); + const c = try @import("residency.zig").openCache(self, s, rows_needed, pass.held()); @memcpy(self.prompt.slice(u32, ids.len), ids); const Chunks = struct { n: usize, @@ -280,7 +285,7 @@ pub const Metal = struct { } }; s.cached = 0; - var at: usize = 0; + var at = try pass.resume_(self, s, c); // TF_KV_SHARE: the store's kept pages and state if (s.reuse.saved) |saved| { // a kept state of this prompt's prefix: the pass starts there const snap: *snapshot.Snap = @ptrCast(@alignCast(saved)); if (snap.at < ids.len) if (snapshot.restore(self, c, snap)) |_| { @@ -294,7 +299,7 @@ pub const Metal = struct { var k: usize = 0; while (at < ids.len) { const decision: ?usize = if (c.decide == lesson_route.none) null else @intCast(c.decide); - const rows = @import("prompt_flight.zig").rows(s, at, self.o.chunk, decision, &k); + const rows = @import("prompt_flight.zig").rows(s, at, self.o.chunk, decision, pass.marks, &k); // TF_KV_SHARE: and its marks if (s.isCancelled()) return error.Cancelled; // release() waits for the chunk still in flight try kv_pages.grow(c, at + rows); try @import("prompt_flight.zig").submit(self, s, Chunks{ .n = ids.len, .c = c, .at = at, .rows = rows }); @@ -302,23 +307,10 @@ pub const Metal = struct { // landed tokens only: the newest chunk is still queued on the GPU s.fill_at.store(@intCast(at - rows), .monotonic); if (s.reuse.hook) |hk| if (std.mem.indexOfScalar(u32, s.reuse.marks, @intCast(at)) != null) hk.at(hk.ptr, s, @intCast(at)); + pass.mark(self, s, c, at); } - if (self.trace and c.decide != lesson_route.none) try self.traceLessons(c); - } - - /// Dev trace (TF_SLIDE_TRACE): the stream's router scores at its decision row and the lessons acting on its reply. - fn traceLessons(self: *Metal, c: *const st.Cache) !void { - const ad = layers.firstAdapter(&self.m.weights) orelse return; - try self.drain(); - const n = lesson_route.max_blocks; - const at: usize = @intCast(@max(c.rid, 0)); - const score = ad.score.slice(f32, lesson_route.max_slots * n)[at * n ..][0..n]; - const flags = ad.latch.slice(u32, lesson_route.max_slots * n)[at * n ..][0..n]; - const need = ad.need.slice(f32, n); - var line: [2048]u8 = undefined; - var w: std.Io.Writer = .fixed(&line); - for (0..ad.rank / 16) |b| w.print(" {d:.2}/{d:.2}{s}", .{ score[b], need[b], if (flags[b] != 0) "*" else "" }) catch break; - std.log.info("slide trace: decision row {d}: scores{s} (* acting)", .{ c.decide, w.buffered() }); + ok = true; + if (self.trace and c.decide != lesson_route.none) try @import("slide.zig").traceLessons(self, c); } fn firstFn(ptr: *anyopaque, s: *lanes.Stream, position: u64) anyerror!u64 { diff --git a/zig/src/families/nemotron/kv_pages.zig b/zig/src/families/nemotron/kv_pages.zig index 6ff67e01f7..085891220d 100644 --- a/zig/src/families/nemotron/kv_pages.zig +++ b/zig/src/families/nemotron/kv_pages.zig @@ -64,12 +64,12 @@ pub inline fn holds(kv: st.Kv, end: usize) void { std.debug.assert(end <= p.ids().len * kvp.page_tokens); } -/// With no stream open every page must be free: the census says so, or warns. +/// With no stream open every page is free or the store's (TF_KV_SHARE): the census says so, or warns. pub fn census(b: anytype) void { const p = b.kv_pages orelse return; const c = p.pool.census(); - if (p.idle() and c.free == c.total) { - std.log.info("nemotron: KV pages idle, {d} of {d} free", .{ c.free, c.total }); + if (p.idle() and (b.prompts != null or c.free == c.total)) { + std.log.info("nemotron: KV pages idle, {d} of {d} free, {d} kept", .{ c.free, c.total, c.kept }); } else std.log.warn("nemotron: no stream open, yet {d} of {d} KV pages are not free", .{ c.total - c.free, c.total }); } diff --git a/zig/src/families/nemotron/learned_prompts.zig b/zig/src/families/nemotron/learned_prompts.zig index a2808e2b6b..64fbd6b242 100644 --- a/zig/src/families/nemotron/learned_prompts.zig +++ b/zig/src/families/nemotron/learned_prompts.zig @@ -77,7 +77,7 @@ pub fn probe(b: *Metal) !u64 { p.failed = e; return; }; - p.snap = snapshot.save(p.b, c, mark) catch |e| { + p.snap = snapshot.save(p.b, c, mark, false) catch |e| { p.failed = e; return; }; @@ -116,7 +116,7 @@ pub fn snapRead(ptr: *anyopaque, dir: [:0]const u8, key: u64, at: u32) anyerror! var buf: [1100]u8 = undefined; const file = try path(&buf, dir, key); const head = b.head != null; - const n = snapshot.bytes(b.m.config, at, head); + const n = snapshot.bytes(b.m.config, at, head, false); const snap = try b.gpa.create(Snap); errdefer b.gpa.destroy(snap); snap.* = .{ .at = at, .head = head, .buf = try b.m.device.buffer(n, opts), .bytes = n }; diff --git a/zig/src/families/nemotron/prompt_flight.zig b/zig/src/families/nemotron/prompt_flight.zig index 06bbc26613..7d9f04851f 100644 --- a/zig/src/families/nemotron/prompt_flight.zig +++ b/zig/src/families/nemotron/prompt_flight.zig @@ -3,13 +3,16 @@ const lanes = @import("lanes"); pub const Prompt = struct { stream: *lanes.Stream, rows: usize }; -/// The original prompt cuts: requested boundaries, the decision row, and the configured row cap. -pub inline fn rows(s: *const lanes.Stream, at: usize, cap: usize, decision: ?usize, next: *usize) usize { +/// The prompt cuts: requested boundaries, the decision row, TF_KV_SHARE's store marks, and the configured row cap. +pub inline fn rows(s: *const lanes.Stream, at: usize, cap: usize, decision: ?usize, marks: []const u32, next: *usize) usize { while (next.* < s.chunks.len and s.chunks[next.*] <= at) next.* += 1; var end = if (next.* < s.chunks.len) @min(s.chunks[next.*], s.prompt_len) else s.prompt_len; if (decision) |d| if (d >= at and d + 1 < end) { end = d + 1; }; + for (marks) |m| if (m > at) { + end = @min(end, m); // the store keeps a state only where a chunk ends + }; return @min(cap, end - at); } diff --git a/zig/src/families/nemotron/prompt_flight_test.zig b/zig/src/families/nemotron/prompt_flight_test.zig index b4676f6375..5224ca2327 100644 --- a/zig/src/families/nemotron/prompt_flight_test.zig +++ b/zig/src/families/nemotron/prompt_flight_test.zig @@ -50,10 +50,14 @@ const Fake = struct { }; fn fill(f: *Fake, s: *lanes.Stream, cap: usize, decision: ?usize) !void { + try fillAt(f, s, cap, decision, &.{}); +} + +fn fillAt(f: *Fake, s: *lanes.Stream, cap: usize, decision: ?usize, marks: []const u32) !void { var at: usize = 0; var next: usize = 0; while (at < s.prompt_len) { - const n = workflow.rows(s, at, cap, decision, &next); + const n = workflow.rows(s, at, cap, decision, marks, &next); try workflow.submit(f, s, .{ .rows = n }); at += n; } @@ -79,6 +83,24 @@ test "Nemotron prefill widths: actual requested cuts, decision row and cap land try t.expectEqual(@as(f64, 12), f.gpu_ms); } +test "Nemotron prefill widths: a chunk ends at every TF_KV_SHARE store mark, in any order, beside the other cuts" { + const ids: [29]u32 = @splat(7); + var s = try lanes.Stream.init(a, .{ .id = "marks", .prompt = &ids, .max_new = 1, .chunks = &.{ 3, 20 } }); + defer s.deinit(a); + var f: Fake = .{}; + defer f.deinit(); + try fillAt(&f, &s, 8, 12, &.{ 24, 16, 29, 40 }); + try f.drain(); + try t.expectEqualSlices(u32, &.{ 3, 8, 2, 3, 4, 4, 5 }, s.prefill_widths.items); + var ends: [8]u32 = undefined; + var end: u32 = 0; + for (s.prefill_widths.items, 0..) |w, i| { + end += w; + ends[i] = end; + } + for ([_]u32{ 16, 24, 29 }) |m| try t.expect(std.mem.indexOfScalar(u32, ends[0..s.prefill_widths.items.len], m) != null); +} + test "Nemotron prefill widths: one pass waits once; verify and draft buffers never count as prompt passes" { var s = try lanes.Stream.init(a, .{ .id = "one", .prompt = &.{ 1, 2, 3, 4, 5, 6, 7 }, .max_new = 1 }); defer s.deinit(a); diff --git a/zig/src/families/nemotron/prompt_pages.zig b/zig/src/families/nemotron/prompt_pages.zig new file mode 100644 index 0000000000..dc67bde606 --- /dev/null +++ b/zig/src/families/nemotron/prompt_pages.zig @@ -0,0 +1,81 @@ +//! TF_KV_SHARE on Metal: a prompt pass resumes on the store's kept pages and keeps its own pages at the store's marks. +const std = @import("std"); +const lanes = @import("lanes"); +const st = @import("state.zig"); +const snapshot = @import("snapshot.zig"); +const kvp = @import("../../core/metal_kv_pages.zig"); +const Metal = @import("backend.zig").Metal; + +/// One prompt pass's plan from the store: the pages it adopts, where they end, and the marks it keeps at. +pub const Pass = struct { + on: bool = false, + adopt: std.ArrayList(u32) = .empty, // held once more for the stream until its lane adopts them + from: u32 = 0, + marks: []const u32 = &.{}, // the store's allocation, freed by `end` + kept: u64 = 0, + + /// The store's deepest kept state of `s`'s prompt (none without TF_KV_SHARE): its pages held, its state pending. + pub fn begin(b: *Metal, s: *lanes.Stream) !Pass { + const pr = b.prompts orelse return .{}; + b.pending = null; + var p: Pass = .{ .on = true }; + errdefer p.adopt.deinit(b.gpa); + const got = try pr.begin(s.prompt(), s.history_len, s.shared_prefixes, @ptrCast(s), &p.adopt); + p.from = got.from; + p.marks = got.marks; + return p; + } + + /// Pages the lane holds from the store: its promise is that many fewer. + pub fn held(p: *const Pass) usize { + return p.adopt.items.len; + } + + /// The lane's table starts with the adopted pages and the pending Mamba states go in; the pass resumes there. + pub fn resume_(p: *Pass, b: *Metal, s: *lanes.Stream, c: *st.Cache) !usize { + if (!p.on) return 0; + if (p.adopt.items.len > 0) { + const pg = c.kv[0].paged orelse return error.NotPaged; + const ids = p.adopt.items; + p.adopt.items.len = 0; // the lane's now: adopt releases them if it fails + try pg.lease.pages.adopt(pg.lease.lane, ids); + } + if (p.from == 0) return 0; + const k = b.pending orelse return error.KeptStateMismatch; + b.pending = null; + if (k.at != p.from or !k.pages) return error.KeptStateMismatch; + try snapshot.restore(b, c, k); + s.cached = p.from; + return p.from; + } + + /// At one of the store's marks the store takes the pages it lacks and the state; pages it held replace the lane's. + pub fn mark(p: *Pass, b: *Metal, s: *lanes.Stream, c: *st.Cache, at: usize) void { + if (!p.on or std.mem.indexOfScalar(u32, p.marks, @intCast(at)) == null) return; + p.keep(b, s, c, at) catch |e| std.log.warn("nemotron: prompt pages not kept: {s}", .{@errorName(e)}); + } + + fn keep(p: *Pass, b: *Metal, s: *lanes.Stream, c: *st.Cache, at: usize) !void { + const pg = c.kv[0].paged orelse return error.NotPaged; + const n = at / kvp.page_tokens; + const mine = try b.gpa.dupe(u32, pg.ids()[0..n]); + defer b.gpa.free(mine); + const path = try b.gpa.alloc(u32, n); + defer b.gpa.free(path); + const shared = b.prompts.?.keep(s.prompt(), @intCast(at), @ptrCast(s), mine, path); + if (shared == n) p.kept += 1; + if (std.mem.eql(u32, mine[0..shared], path[0..shared])) return; + try b.drain(); // the chunk in flight writes the lane's pages: none goes back while it may + for (path[0..shared], mine[0..shared], 0..) |id, own, i| if (id != own) pg.lease.pages.share(pg.lease.lane, i, id); + } + + /// The pass's line, and the plan's memory back. + pub fn end(p: *Pass, b: *Metal, s: *lanes.Stream, ok: bool) void { + if (!p.on) return; + if (ok) b.prompts.?.report(s.prompt().len, s.cached, p.kept); + for (p.adopt.items) |id| b.kv_pages.?.pool.release(id); // pages a failed pass never adopted + p.adopt.deinit(b.gpa); + b.prompts.?.free(p.marks); + p.* = .{}; + } +}; diff --git a/zig/src/families/nemotron/residency.zig b/zig/src/families/nemotron/residency.zig index e9705bd68a..e8fca8e29a 100644 --- a/zig/src/families/nemotron/residency.zig +++ b/zig/src/families/nemotron/residency.zig @@ -3,11 +3,11 @@ const std = @import("std"); const mtl = @import("metal"); const fields = mtl.resident.fields; -pub fn openCache(b: anytype, stream: *@import("lanes").Stream, capacity: usize) !*@import("state.zig").Cache { +pub fn openCache(b: anytype, stream: *@import("lanes").Stream, capacity: usize, held: usize) !*@import("state.zig").Cache { const state = @import("state.zig"); const cache = try b.gpa.create(state.Cache); errdefer b.gpa.destroy(cache); - const lease = try leasePages(b, capacity); + const lease = try leasePages(b, capacity, held); errdefer if (lease) |l| l.pages.finish(l.lane); cache.* = try state.Cache.initPaged(b.m.device, b.m.config, capacity, b.head != null, lease); cache.setResident(b.residency); @@ -22,13 +22,13 @@ pub fn openCache(b: anytype, stream: *@import("lanes").Stream, capacity: usize) return cache; } -/// TF_NEMOTRON_METAL_KV_PAGES: a free lane promised the pages of `capacity` rows; LaneMemory: none now. -fn leasePages(b: anytype, capacity: usize) !?@import("state.zig").Lease { +/// TF_NEMOTRON_METAL_KV_PAGES: a free lane promised the pages of `capacity` rows less `held` adopted; LaneMemory: none. +fn leasePages(b: anytype, capacity: usize, held: usize) !?@import("state.zig").Lease { const pages = b.kv_pages orelse return null; const lane = for (pages.leases, 0..) |l, i| { if (l.promised == 0 and l.ids.items.len == 0) break i; } else return error.LaneMemory; - try pages.admit(lane, pages.layout.pages(capacity)); // its pages are taken as its rows grow (kv_pages.grow) + try pages.admit(lane, pages.layout.pages(capacity) -| held); // its pages are taken as its rows grow (kv_pages.grow) return .{ .pages = pages, .lane = lane }; } diff --git a/zig/src/families/nemotron/slide.zig b/zig/src/families/nemotron/slide.zig index b6be98fa95..990d33b995 100644 --- a/zig/src/families/nemotron/slide.zig +++ b/zig/src/families/nemotron/slide.zig @@ -168,3 +168,19 @@ pub fn loadLiving(gpa: std.mem.Allocator, io: std.Io, b: *backend.Metal) !void { b.living = sites; std.log.info("slide: {d} saved lessons act on the replies their routers choose", .{blocks}); } + +/// Dev trace (TF_SLIDE_TRACE): the stream's router scores at its decision row and the lessons acting on its reply. +pub fn traceLessons(b: *backend.Metal, c: *const @import("state.zig").Cache) !void { + const lesson_route = @import("living").lesson_route; + const ad = @import("layers.zig").firstAdapter(&b.m.weights) orelse return; + try b.drain(); + const n = lesson_route.max_blocks; + const at: usize = @intCast(@max(c.rid, 0)); + const score = ad.score.slice(f32, lesson_route.max_slots * n)[at * n ..][0..n]; + const flags = ad.latch.slice(u32, lesson_route.max_slots * n)[at * n ..][0..n]; + const need = ad.need.slice(f32, n); + var line: [2048]u8 = undefined; + var w: std.Io.Writer = .fixed(&line); + for (0..ad.rank / 16) |k| w.print(" {d:.2}/{d:.2}{s}", .{ score[k], need[k], if (flags[k] != 0) "*" else "" }) catch break; + std.log.info("slide trace: decision row {d}: scores{s} (* acting)", .{ c.decide, w.buffered() }); +} diff --git a/zig/src/families/nemotron/snapshot.zig b/zig/src/families/nemotron/snapshot.zig index a8cf5e9a28..51fa29a7c8 100644 --- a/zig/src/families/nemotron/snapshot.zig +++ b/zig/src/families/nemotron/snapshot.zig @@ -10,7 +10,7 @@ const kvp = @import("../../core/metal_kv_pages.zig"); const opts = mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked; /// One kept state: `at` prompt tokens of one stream, in one buffer the GPU copies on the engine's queue. -pub const Snap = struct { at: u32, head: bool, buf: mtl.Buffer, bytes: usize, residency: ?mtl.ResidencySet.Follow = null }; +pub const Snap = struct { at: u32, head: bool, buf: mtl.Buffer, bytes: usize, residency: ?mtl.ResidencySet.Follow = null, pages: bool = false }; fn convBytes(c: cfg.Config) usize { return (c.conv_kernel - 1) * c.convDim() * 2; @@ -30,10 +30,10 @@ pub fn diskShape(c: cfg.Config, head: bool) struct { mamba: usize, planes: usize return .{ .mamba = c.count(.mamba) * (convBytes(c) + ssmBytes(c)), .planes = (c.count(.attention) + @intFromBool(head)) * 2 * c.kv_heads, .row = @intCast(rowBytes(c, 1)) }; } -/// The bytes a state at `at` tokens takes (with the head's cache when `head`). -pub fn bytes(c: cfg.Config, at: usize, head: bool) usize { +/// The bytes a state at `at` tokens takes (with the head's cache when `head`; `pages`: its rows stay in pool pages). +pub fn bytes(c: cfg.Config, at: usize, head: bool, pages: bool) usize { const kvs = c.count(.attention) + @intFromBool(head); - return c.count(.mamba) * (convBytes(c) + ssmBytes(c)) + kvs * 2 * c.kv_heads * rowBytes(c, at); + return c.count(.mamba) * (convBytes(c) + ssmBytes(c)) + if (pages) 0 else kvs * 2 * c.kv_heads * rowBytes(c, at); } /// Copies between a stream's live rows (Mamba states in `slot`) and `snap`, a dispatch each (`into`: live to snap). @@ -65,7 +65,8 @@ const Copies = struct { self.part(b, e, &off, b.pool.ssm[m], self.slot * ssmBytes(c), ssmBytes(c)); } const n = rowBytes(c, self.snap.at); - for (0..self.c.attentions + @intFromBool(self.c.mtp != null)) |i| { + const kvs = if (self.snap.pages) 0 else self.c.attentions + @intFromBool(self.c.mtp != null); // TF_KV_SHARE: shared + for (0..kvs) |i| { const kv = if (i < self.c.attentions) self.c.kv[i] else self.c.mtp.?; if (!self.into) @import("kv_pages.zig").holds(kv, self.snap.at); if (kv.paged) |p| { @@ -87,13 +88,13 @@ const Copies = struct { }; /// The stream's state after `at` prompt tokens (its pass stands there) into a new buffer, copied on the queue. -pub fn save(b: *Metal, cache: *st.Cache, at: u32) !*Snap { +pub fn save(b: *Metal, cache: *st.Cache, at: u32, pages: bool) !*Snap { const head = cache.mtp != null; if (at == 0 or cache.len != at or cache.replay != 0 or cache.pending != null or (head and cache.mtp_len != at)) return error.SnapshotOutOfStep; - const n = bytes(b.m.config, at, head); + const n = bytes(b.m.config, at, head, pages); const snap = try b.gpa.create(Snap); errdefer b.gpa.destroy(snap); - snap.* = .{ .at = at, .head = head, .buf = try b.m.device.buffer(n, opts), .bytes = n }; + snap.* = .{ .at = at, .head = head, .buf = try b.m.device.buffer(n, opts), .bytes = n, .pages = pages }; errdefer snap.buf.deinit(); try b.submit(.prefill, b.next, Copies{ .c = cache, .slot = cache.slot, .snap = snap, .into = true }); snap.residency = b.residency; @@ -105,7 +106,7 @@ pub fn save(b: *Metal, cache: *st.Cache, at: u32) !*Snap { pub fn restore(b: *Metal, cache: *st.Cache, snap: *const Snap) !void { if (cache.len != 0 or snap.head != (cache.mtp != null)) return error.SnapshotLayout; if (snap.at + 1 > cache.kv[0].capacity) return error.ContextFull; - try @import("kv_pages.zig").grow(cache, snap.at); + if (!snap.pages) try @import("kv_pages.zig").grow(cache, snap.at); // shared: the pages are the adopted ones const slot = try b.pool.take(); errdefer b.pool.give(slot); try b.submit(.prefill, b.next, Copies{ .c = cache, .slot = slot, .snap = snap, .into = false }); @@ -123,19 +124,19 @@ pub fn drop(b: *Metal, snap: *Snap) void { /// The prompt cache's Snapshots functions over the Metal backend (`ptr`: the *Metal, `owner`: the stream). pub const Cached = struct { - fn metal(ptr: *anyopaque) *Metal { + pub fn metal(ptr: *anyopaque) *Metal { return @ptrCast(@alignCast(ptr)); } pub fn snapBytes(ptr: *anyopaque, at: u32) u64 { const b = metal(ptr); - return bytes(b.m.config, at, b.head != null); + return bytes(b.m.config, at, b.head != null, false); } pub fn snapSave(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { const b = metal(ptr); const c = try b.cacheOf(@ptrCast(@alignCast(owner orelse return error.NoStream))); - return @ptrCast(try save(b, c, at)); + return @ptrCast(try save(b, c, at, false)); } pub fn snapRestore(_: *anyopaque, _: ?*anyopaque, _: *anyopaque) anyerror!void { @@ -147,6 +148,38 @@ pub const Cached = struct { } }; +/// TF_KV_SHARE: the prompt store's functions over the Metal backend; a state is its Mamba states, its rows the pages. +pub const Shared = struct { + const metal = Cached.metal; + pub const snap_lookahead: u32 = 1; // the head's row for a token waits for the next one + + pub fn share_pool(ptr: *anyopaque) ?*@import("../../core/kv_pool.zig").Pool { + return if (metal(ptr).kv_pages) |p| &p.pool else null; + } + + pub fn share_attach(ptr: *anyopaque, prompts: ?@import("../../core/kv_prompts.zig").Prompts) void { + metal(ptr).prompts = prompts; + } + + pub fn snap_bytes(ptr: *anyopaque, at: u32) u64 { + const b = metal(ptr); + return bytes(b.m.config, at, b.head != null, true); + } + + pub fn snap_save(ptr: *anyopaque, owner: ?*anyopaque, at: u32) anyerror!*anyopaque { + const b = metal(ptr); + const c = try b.cacheOf(@ptrCast(@alignCast(owner orelse return error.NoStream))); + return @ptrCast(try save(b, c, at, true)); + } + + /// The store's resume: the stream's pass restores it once its lane holds the adopted pages. + pub fn snap_restore(ptr: *anyopaque, _: ?*anyopaque, saved: *anyopaque) anyerror!void { + metal(ptr).pending = @ptrCast(@alignCast(saved)); + } + + pub const snap_drop = Cached.snapDrop; +}; + test "a state's bytes: fixed Mamba states, then a key and value row per KV head and attention layer a token" { var c: cfg.Config = undefined; c.conv_kernel = 4; @@ -161,7 +194,8 @@ test "a state's bytes: fixed Mamba states, then a key and value row per KV head c.kinds[1] = .moe; c.kinds[2] = .attention; const fixed = 3 * 6144 * 2 + 64 * 64 * 128 * 4; - try std.testing.expectEqual(@as(usize, fixed), bytes(c, 0, true)); - try std.testing.expectEqual(@as(usize, 2 * 2 * 2 * 128 * 2), bytes(c, 1, true) - bytes(c, 0, true)); - try std.testing.expectEqual(@as(usize, 2 * 2 * 128 * 2), bytes(c, 1, false) - bytes(c, 0, false)); + try std.testing.expectEqual(@as(usize, fixed), bytes(c, 0, true, false)); + try std.testing.expectEqual(@as(usize, 2 * 2 * 2 * 128 * 2), bytes(c, 1, true, false) - bytes(c, 0, true, false)); + try std.testing.expectEqual(@as(usize, 2 * 2 * 128 * 2), bytes(c, 1, false, false) - bytes(c, 0, false, false)); + try std.testing.expectEqual(@as(usize, fixed), bytes(c, 4096, true, true)); // TF_KV_SHARE: the rows stay in pages } diff --git a/zig/src/native/cuda_window.zig b/zig/src/native/cuda_window.zig index 355dc4d1b4..144043cede 100644 --- a/zig/src/native/cuda_window.zig +++ b/zig/src/native/cuda_window.zig @@ -1,4 +1,4 @@ -//! A CUDA family's context window: the model's own (config.json), and what --context makes of it. +//! A family's context window: the model's own (config.json, read for Metal too), and what --context makes of it. const std = @import("std"); const Allocator = std.mem.Allocator; diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index e7f69f03a2..8ae3e7c654 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -17,8 +17,10 @@ const cache_fit = @import("cache_fit.zig"); const metal_admit = @import("metal_admit.zig"); const Resident = @import("resident.zig").Resident; const nemo_disk = @import("nemotron_disk.zig"); +const prompt_pages = @import("cuda_prompt_pages.zig"); // the family-side store adapter, over the core pool as on CUDA const fsr_options = @import("fsr_options.zig"); const linkProblem = @import("metal_link.zig").problem; +const modelContext = @import("cuda_window.zig").modelContext; // the window from config.json, as CUDA reads it pub const backends: []const []const u8 = &.{"metal"}; pub const serves: []const []const u8 = &.{ "--checkpoint-slots", "--spill-gib" }; @@ -96,17 +98,6 @@ fn prefillStep(device: mtl.Device) u32 { return if (gen >= 5 and device.maxWorkingSet() >= 48 << 30) 8192 else 2048; } -/// The model's window (config.json's max_position_embeddings, text_config's first), 0 when it names none. -fn modelContext(a: Allocator, io: std.Io, dir: []const u8) i64 { - const path = std.fs.path.join(a, &.{ dir, "config.json" }) catch return 0; - const bytes = std.Io.Dir.cwd().readFileAlloc(io, path, a, .limited(16 << 20)) catch return 0; - const doc = std.json.parseFromSliceLeaky(std.json.Value, a, bytes, .{}) catch return 0; - if (doc != .object) return 0; - const text = if (doc.object.get("text_config")) |t| (if (t == .object) t else doc) else doc; - const limit = text.object.get("max_position_embeddings") orelse doc.object.get("max_position_embeddings") orelse return 0; - return if (limit == .integer and limit.integer > 0) limit.integer else 0; -} - /// The host's memory counts: `cache_fit.hostMemory` from the process footprint (no configured load limit). fn readMemory(_: ?*anyopaque, reset_peak: bool) ?api.Memory { return cache_fit.hostMemory(0, reset_peak); @@ -125,6 +116,7 @@ const Host = struct { host: api.LaneHost, cache: ?api.prompt_cache.Store = null, // kept prompt states (engine thread only); null: a zero budget learned: ?api.prompt_imprint.Imprint = null, // --learn: shared states on disk, read back by later servers + prompts: ?*prompt_pages.Adapter = null, // TF_KV_SHARE: prompts kept as the pool's pages (no `cache` then) round: nemotron.gpu_round.Options = .{ .depth = 8 }, // a lone greedy stream's GPU-side rounds, as the CLI runs them slide: ?metal_slide.Registration = null, // Sliding Weights' learner, with --slide @@ -150,6 +142,7 @@ const Host = struct { if (h.cache) |*store| store.deinit(); if (rows) |r| r.deinit(); // after the store: its entries leave the write queue as they go if (h.learned) |*m| m.deinit(); + if (h.prompts) |x| x.close(); // the store's pages back before the pool goes h.core.deinit(); h.cfg.deinit(h.gpa); nemotron.residency.drop(h); @@ -267,7 +260,20 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c return e; }; h.host.info_.prompt_cache_plan = plan; - if (plan.budget_bytes > 0) { + const share = paged != null and try tf.kv_prompts.sharing() > 0; // TF_KV_SHARE: prompts kept as pool pages + if (share and o.learn_conversations) { + problem.* = "--learn-conversations writes the host prompt cache's states, and TF_KV_SHARE keeps prompts as pool pages instead: serve with one of the two"; + return error.KvShareLearns; + } + h.prompts = null; + if (share and plan.budget_bytes > 0) { + if (h.metal.o.chunk % paged.?.layout.page_tokens != 0) return error.ChunkOffPages; + const x = (try prompt_pages.Adapter.open(nemotron.snapshot.Shared, gpa, h.metal, @intCast(paged.?.layout.page_tokens), try tf.kv_prompts.sharing())).?; + h.prompts = x; + x.store.rules.planned = true; // kept on the prompt chunks' grid: a resume's seams are a cold pass's, its tokens too + x.store.rules.grid = @intCast(h.metal.o.chunk); + h.host.info_.prompt_cache = true; + } else if (plan.budget_bytes > 0) { const S = nemotron.snapshot.Cached; const L = nemotron.learned_prompts; h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = h.metal, .vtable = &.{ .bytes = S.snapBytes, .save = S.snapSave, .restore = S.snapRestore, .drop = S.snapDrop, .write = L.snapWrite, .read = L.snapRead, .forget = L.snapForget, .forget_checked = api.prompt_cache.singleForgetChecked, .reclaim = api.prompt_cache.singleReclaim, .rows = &nemo_disk.image } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); @@ -275,6 +281,7 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c h.host.info_.prompt_cache = true; } errdefer if (h.cache) |*store| store.deinit(); + errdefer if (h.prompts) |x| x.close(); h.learned = null; if (o.learn) |root| { const id = @import("metal_identity.zig").learned(a, io, o.dir, h.metal) catch |e| { diff --git a/zig/src/native/nemotron_disk.zig b/zig/src/native/nemotron_disk.zig index 02fed61bf8..bf823a5050 100644 --- a/zig/src/native/nemotron_disk.zig +++ b/zig/src/native/nemotron_disk.zig @@ -38,7 +38,7 @@ fn gather(ptr: *anyopaque, saved: api.prompt_cache.Saved, ranges: []const rows.i fn blank(ptr: *anyopaque, at: u32) anyerror!api.prompt_cache.Saved { const b = metal(ptr); const head = b.head != null; - const n = nemotron.snapshot.bytes(b.m.config, at, head); + const n = nemotron.snapshot.bytes(b.m.config, at, head, false); const snap = try b.gpa.create(Snap); errdefer b.gpa.destroy(snap); snap.* = .{ .at = at, .head = head, .buf = try b.m.device.buffer(n, mtl.ResourceOptions.shared | mtl.ResourceOptions.untracked), .bytes = n }; From ecbda69369388f036bd4290d83d1dbe171b91108 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:38:57 +0000 Subject: [PATCH 3/5] changelog: #1850 under Unreleased --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a18c171ad9..644cae5879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel ## Unreleased +- Nemotron on Metal: with `TF_NEMOTRON_METAL_KV_PAGES=1`, `TF_KV_SHARE=n` keeps prompt states in the core `prompt_radix` store over the KV pool, as CUDA does; `TF_KV_SHARE` acts only on the paged pool, and unset or 0 keeps the host prompt cache (#1850). Thanks to @jschmied. + - GLM CUDA at TP=2 and up: each rank names its startup phases (NCCL init, peer warm-up, weights, calibration, ready), prints NCCL's init device, version and log path, and warns once a minute while NCCL init or the warm-up is unfinished, with the transport seen so far. It only reports; transfers are unchanged (#2354, part of #2347). - Docs: the serve flags match the CLI: `--metrics-open` opens `/v1/metrics` too, `--learn-conversations` on CUDA serves the Qwen3.5 hybrids and Flash Next as well as Nemotron, Flash Next shares sessions under an explicit `--parallel` above one and refuses `--recall` only on a Mac, and the Metal Qwen3.5-2B keeps prompt states (#2367). From b5470ea297114bfd79bda33c702b5a0e81d3fb5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Schmied?= <600316+jschmied@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:19:50 +0100 Subject: [PATCH 4/5] nemotron: a TF_KV_SHARE chunk off the pool's pages says why it refuses, as the --learn-conversations refusal does --- zig/src/native/metal.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 8ae3e7c654..3ab97b7338 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -267,7 +267,10 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c } h.prompts = null; if (share and plan.budget_bytes > 0) { - if (h.metal.o.chunk % paged.?.layout.page_tokens != 0) return error.ChunkOffPages; + if (h.metal.o.chunk % paged.?.layout.page_tokens != 0) { + problem.* = try std.fmt.allocPrint(a, "TF_KV_SHARE keeps prompts as whole pool pages on the prompt chunks' grid, and this Mac's chunk of {d} rows is not a multiple of the pool's {d}-token pages: serve without TF_KV_SHARE", .{ h.metal.o.chunk, paged.?.layout.page_tokens }); + return error.ChunkOffPages; + } const x = (try prompt_pages.Adapter.open(nemotron.snapshot.Shared, gpa, h.metal, @intCast(paged.?.layout.page_tokens), try tf.kv_prompts.sharing())).?; h.prompts = x; x.store.rules.planned = true; // kept on the prompt chunks' grid: a resume's seams are a cold pass's, its tokens too From 3dca2b1314eb116dcdf2192cc6e5eb9f35f93de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Schmied?= <600316+jschmied@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:01:29 +0100 Subject: [PATCH 5/5] kv share: the store's open on the prompt chunks' grid lives beside the core adapter, so metal.zig stays under 600 lines --- zig/src/native/cuda_prompt_pages.zig | 12 ++++++++++++ zig/src/native/metal.zig | 12 ++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/zig/src/native/cuda_prompt_pages.zig b/zig/src/native/cuda_prompt_pages.zig index b04549473d..91f3da0d6a 100644 --- a/zig/src/native/cuda_prompt_pages.zig +++ b/zig/src/native/cuda_prompt_pages.zig @@ -53,6 +53,18 @@ pub const Adapter = struct { return x; } + /// TF_KV_SHARE's store kept on the prompt chunks' grid, or why it refuses: a chunk off the pool's pages. + pub fn openOnGrid(comptime F: type, gpa: Allocator, target: anytype, page_tokens: u32, chunk: u32, a: Allocator, problem: *[]const u8) !*Adapter { + if (chunk % page_tokens != 0) { + problem.* = try std.fmt.allocPrint(a, "TF_KV_SHARE keeps prompts as whole pool pages on the prompt chunks' grid, and this Mac's chunk of {d} rows is not a multiple of the pool's {d}-token pages: serve without TF_KV_SHARE", .{ chunk, page_tokens }); + return error.ChunkOffPages; + } + const x = (try open(F, gpa, target, page_tokens, try hooks.sharing())).?; + x.store.rules.planned = true; // a resume's seams are a cold pass's, its tokens too + x.store.rules.grid = chunk; + return x; + } + fn Detach(comptime F: type) type { return struct { fn call(target: *anyopaque) void { diff --git a/zig/src/native/metal.zig b/zig/src/native/metal.zig index 3ab97b7338..3a4750449b 100644 --- a/zig/src/native/metal.zig +++ b/zig/src/native/metal.zig @@ -16,7 +16,6 @@ const learners = metal_slide.Registry(.{nemotron}); const cache_fit = @import("cache_fit.zig"); const metal_admit = @import("metal_admit.zig"); const Resident = @import("resident.zig").Resident; -const nemo_disk = @import("nemotron_disk.zig"); const prompt_pages = @import("cuda_prompt_pages.zig"); // the family-side store adapter, over the core pool as on CUDA const fsr_options = @import("fsr_options.zig"); const linkProblem = @import("metal_link.zig").problem; @@ -267,19 +266,12 @@ pub fn open(a: Allocator, gpa: Allocator, io: std.Io, o: api.Open, problem: *[]c } h.prompts = null; if (share and plan.budget_bytes > 0) { - if (h.metal.o.chunk % paged.?.layout.page_tokens != 0) { - problem.* = try std.fmt.allocPrint(a, "TF_KV_SHARE keeps prompts as whole pool pages on the prompt chunks' grid, and this Mac's chunk of {d} rows is not a multiple of the pool's {d}-token pages: serve without TF_KV_SHARE", .{ h.metal.o.chunk, paged.?.layout.page_tokens }); - return error.ChunkOffPages; - } - const x = (try prompt_pages.Adapter.open(nemotron.snapshot.Shared, gpa, h.metal, @intCast(paged.?.layout.page_tokens), try tf.kv_prompts.sharing())).?; - h.prompts = x; - x.store.rules.planned = true; // kept on the prompt chunks' grid: a resume's seams are a cold pass's, its tokens too - x.store.rules.grid = @intCast(h.metal.o.chunk); + h.prompts = try prompt_pages.Adapter.openOnGrid(nemotron.snapshot.Shared, gpa, h.metal, @intCast(paged.?.layout.page_tokens), @intCast(h.metal.o.chunk), a, problem); h.host.info_.prompt_cache = true; } else if (plan.budget_bytes > 0) { const S = nemotron.snapshot.Cached; const L = nemotron.learned_prompts; - h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = h.metal, .vtable = &.{ .bytes = S.snapBytes, .save = S.snapSave, .restore = S.snapRestore, .drop = S.snapDrop, .write = L.snapWrite, .read = L.snapRead, .forget = L.snapForget, .forget_checked = api.prompt_cache.singleForgetChecked, .reclaim = api.prompt_cache.singleReclaim, .rows = &nemo_disk.image } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); + h.cache = api.prompt_cache.Store.init(gpa, .{ .ptr = h.metal, .vtable = &.{ .bytes = S.snapBytes, .save = S.snapSave, .restore = S.snapRestore, .drop = S.snapDrop, .write = L.snapWrite, .read = L.snapRead, .forget = L.snapForget, .forget_checked = api.prompt_cache.singleForgetChecked, .reclaim = api.prompt_cache.singleReclaim, .rows = &@import("nemotron_disk.zig").image } }, .{ .lookahead = 1, .planned = true }, plan.budget_bytes); h.host.cache = &h.cache.?; // the lane host's store: a Sliding Weights step that moves weights clears it h.host.info_.prompt_cache = true; }