From 44be7efe648016938645b4753bdb01a204ca287b Mon Sep 17 00:00:00 2001
From: Elevatormusic <22101396+Elevatormusic@users.noreply.github.com>
Date: Wed, 21 Oct 2026 23:13:08 -0700
Subject: [PATCH 1/4] dsv41: special-token text in request strings is plain
 text
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An added-token spelling that a client writes in a message, a tool
result, tool call arguments or a tool definition was encoded as its
token, so "</think>" or "<｜User｜>" in a message forged a thinking end
or a turn. The Python lane keeps such text plain (server/plain_text.py);
the Zig server now does the same for a family that sets
plain_assistant_keep (dsv41):

- server/plain_text.zig: shield puts a unique marker in place of each
  spelling in the normalized messages and tools before the template
  runs. The grounding tokens stay in an assistant's content and
  reasoning_content; a lane image's placeholder stays in content when
  the server wrote it.
- Tokenizer.encodeCut: each added token that the rendered text spells
  exactly is its token; the text between, with the markers spelled
  again, is encoded with no added token matched (memo kept for text
  that spells no added token).
- Info.plain_message_text / plain_assistant_keep, set from the family.
---
 zig/src/core/engine_api.zig          |   4 +
 zig/src/core/tokenizer/steps.zig     |  10 +
 zig/src/core/tokenizer/tokenizer.zig |  89 +++++++++
 zig/src/families/dsv41/native.zig    |   2 +
 zig/src/families/dsv41/text.zig      |   4 +
 zig/src/native/cuda.zig              |   2 +
 zig/src/server/hf_text.zig           |  13 ++
 zig/src/server/model_text.zig        |  22 +++
 zig/src/server/plain_text.zig        | 278 +++++++++++++++++++++++++++
 zig/src/server/prompt.zig            |  28 ++-
 zig/src/server/root.zig              |   1 +
 zig/src/server/server.zig            |   4 +
 12 files changed, 453 insertions(+), 4 deletions(-)
 create mode 100644 zig/src/server/plain_text.zig

diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig
index 1e98911bfd..9bc64012d4 100644
--- a/zig/src/core/engine_api.zig
+++ b/zig/src/core/engine_api.zig
@@ -261,6 +261,10 @@ pub const Info = struct {
     /// A lane-vision family's image input (LaneVision: each image part its placeholder, the family's prepare its span);
     /// null: none (upstream's image input is `vision`).
     lane_vision: ?LaneVision = null,
+    /// An added-token spelling in request text is plain text, not its token (server/plain_text.zig; DeepSeek's encoding).
+    plain_message_text: bool = false,
+    /// With plain_message_text: the spellings that stay tokens in an assistant's content and reasoning.
+    plain_assistant_keep: []const []const u8 = &.{},
     /// A prompt-only request (max_tokens 0) keeps its end state: the server prefills each reply for the next turn.
     warm_turns: bool = false,
     /// The engine decodes plain whatever a request asks: no drafter loaded, or this chip is faster plain.
diff --git a/zig/src/core/tokenizer/steps.zig b/zig/src/core/tokenizer/steps.zig
index d78e9cec29..ee3f04e3d3 100644
--- a/zig/src/core/tokenizer/steps.zig
+++ b/zig/src/core/tokenizer/steps.zig
@@ -121,6 +121,16 @@ pub const Normalizer = union(enum) {
         return error.UnsupportedNormalizer;
     }
 
+    /// Whether the normalizer changes no text: a sequence of no steps (DeepSeek's).
+    pub fn isIdentity(n: *const Normalizer) bool {
+        return switch (n.*) {
+            .sequence => |steps| for (steps) |*step| {
+                if (!step.isIdentity()) break false;
+            } else true,
+            else => false,
+        };
+    }
+
     pub fn hasForms(n: *const Normalizer) bool {
         return switch (n.*) {
             .sequence => |steps| for (steps) |*step| {
diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig
index 785ed87f66..a335c91cb7 100644
--- a/zig/src/core/tokenizer/tokenizer.zig
+++ b/zig/src/core/tokenizer/tokenizer.zig
@@ -161,6 +161,63 @@ pub const Tokenizer = struct {
         return a.dupe(u32, ids.items);
     }
 
+    /// What encodeCut does to the text between two cuts before it encodes it: a shielded request's markers spelled again.
+    pub const Restore = struct {
+        ctx: *const anyopaque,
+        spelled: *const fn (ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8,
+    };
+
+    /// Ids of a rendered prompt as Python's PlainText.encode makes them: each added token that `text` spells exactly
+    /// is a cut and its token; the text between two cuts, after `restore`, is encoded with no added token matched.
+    pub fn encodeCut(t: *const Tokenizer, a: Allocator, text: []const u8, restore: Restore) Error![]u32 {
+        var arena = std.heap.ArenaAllocator.init(a);
+        defer arena.deinit();
+        var ctx = steps.Context{ .a = arena.allocator(), .matcher = .{ .a = arena.allocator() } };
+        var valid = text;
+        if (!std.unicode.utf8ValidateSlice(text)) {
+            var fixed: std.ArrayList(u8) = .empty;
+            try unicode.appendLossy(&fixed, ctx.a, text);
+            valid = fixed.items;
+        }
+        var ids: std.ArrayList(u32) = .empty;
+        var at: usize = 0;
+        // the raw trie's tokens, then the normalized trie's in the text between them, both on the text as written
+        for (try t.splitAdded(&ctx, &t.raw, valid)) |seg| {
+            if (seg.id) |id| {
+                if (!t.spells(valid[seg.start..seg.end], id)) continue;
+                try t.encodeRegion(&ctx, valid[at..seg.start], restore, &ids);
+                try ids.append(ctx.a, id);
+                at = seg.end;
+                continue;
+            }
+            for (try t.splitAdded(&ctx, &t.normalized, valid[seg.start..seg.end])) |sub| {
+                const id = sub.id orelse continue;
+                if (!t.spells(valid[seg.start + sub.start .. seg.start + sub.end], id)) continue; // found through a strip: plain text
+                try t.encodeRegion(&ctx, valid[at .. seg.start + sub.start], restore, &ids);
+                try ids.append(ctx.a, id);
+                at = seg.start + sub.end;
+            }
+        }
+        try t.encodeRegion(&ctx, valid[at..], restore, &ids);
+        return a.dupe(u32, ids.items);
+    }
+
+    /// Whether `span` is the added token `id`'s own text.
+    fn spells(t: *const Tokenizer, span: []const u8, id: u32) bool {
+        return std.mem.eql(u8, span, t.id_to_token.get(id) orelse return false);
+    }
+
+    /// One region of encodeCut: restored, then encoded as its own text (Python's plain tokenizer on each region).
+    fn encodeRegion(t: *const Tokenizer, ctx: *steps.Context, region: []const u8, restore: Restore, ids: *std.ArrayList(u32)) Error!void {
+        if (region.len == 0) return;
+        const text = try restore.spelled(restore.ctx, ctx.a, region);
+        if (text.len == 0) return;
+        // text that spells no added token encodes the same with or without them: the memo's ids are good
+        const plain = t.raw.next(text, 0) == null and t.normalized.next(text, 0) == null and (t.normalizer == null or t.normalizer.?.isIdentity());
+        if (plain and t.memo != null and text.len >= @import("memo.zig").min_bytes) return t.encodeSegment(ctx, text, true, ids);
+        return t.encodeTextWith(ctx, text, true, ids, false);
+    }
+
     fn encodeText(t: *const Tokenizer, ctx: *steps.Context, text: []const u8, first: bool, ids: *std.ArrayList(u32)) Error!void {
         return t.encodeTextWith(ctx, text, first, ids, true);
     }
@@ -386,6 +443,38 @@ test "byte-level BPE with merges, added tokens and decode" {
     try std.testing.expectEqualStrings("", unknown);
 }
 
+test "a cut encode: the text's own added tokens are tokens, the restored text between them is plain text" {
+    const json =
+        \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
+        \\  {"id":8,"content":"<t>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}],
+        \\ "normalizer":{"type":"Sequence","normalizers":[]},
+        \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
+        \\ "decoder":{"type":"ByteLevel"},
+        \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
+        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":9,"t":10,">":11},"merges":[["Ġ","a"],"a b","Ġa b"]}}
+    ;
+    const a = std.testing.allocator;
+    var t = try parse(a, json);
+    defer t.deinit();
+    const Marks = struct {
+        fn spelled(_: *const anyopaque, al: Allocator, text: []const u8) Allocator.Error![]const u8 {
+            return std.mem.replaceOwned(u8, al, text, "@0@", "<t>");
+        }
+    };
+    const restore: Tokenizer.Restore = .{ .ctx = &t, .spelled = Marks.spelled };
+    const ids = try t.encodeCut(a, "ab<|end|>a@0@b<t>c", restore);
+    defer a.free(ids);
+    // "<t>" from the marker is three plain tokens; the text's own "<t>" is token 8
+    try std.testing.expectEqualSlices(u32, &.{ 3, 7, 0, 9, 10, 11, 1, 8, 6 }, ids);
+    // no marker and no added token: the same ids as encode, through the memo too
+    t.memo = try Memo.create(a, 1 << 20);
+    const plain = try t.encode(a, "ab abc");
+    defer a.free(plain);
+    const cut = try t.encodeCut(a, "ab abc", restore);
+    defer a.free(cut);
+    try std.testing.expectEqualSlices(u32, plain, cut);
+}
+
 test "with the memo, long segments encode as without it, and the second encode takes their kept ids" {
     const json =
         \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],
diff --git a/zig/src/families/dsv41/native.zig b/zig/src/families/dsv41/native.zig
index 07074e2fa4..194d43d13e 100644
--- a/zig/src/families/dsv41/native.zig
+++ b/zig/src/families/dsv41/native.zig
@@ -52,6 +52,8 @@ pub const max_segments: u32 = 1;
 pub const prompt_rows: u32 = 2048;
 /// The served lane's reasoning effort numbers (text.zig): the server passes each name and the template hears its number.
 pub const efforts = @import("text.zig").efforts;
+/// An added-token spelling in request text is plain text; these grounding tokens stay tokens in an assistant's text.
+pub const plain_assistant_keep = @import("text.zig").grounding;
 
 pub const Options = struct { context: usize, drafts: bool, segments: usize = 1, slide: bool = false }; // slide: no learner here
 
diff --git a/zig/src/families/dsv41/text.zig b/zig/src/families/dsv41/text.zig
index 8fe1e86c68..2217fc8223 100644
--- a/zig/src/families/dsv41/text.zig
+++ b/zig/src/families/dsv41/text.zig
@@ -6,6 +6,10 @@
 //! without one hears the template's default ("high": 75), the number the lane sends; "none" turns thinking off.
 //! Reasoning split ("</think>" closes the block the generation prompt opens) and DSML tool calls are the server's own.
 
+/// The grounding tokens. DeepSeek's encoding keeps them as tokens in an assistant's content and reasoning (the model
+/// wrote them there); every other added-token spelling in request text is plain text (app.py GROUNDING).
+pub const grounding: []const []const u8 = &.{ "<｜ref｜>", "<｜/ref｜>", "<｜box｜>", "<｜/box｜>", "<｜point｜>", "<｜/point｜>", "<｜polygon｜>", "<｜/polygon｜>" };
+
 /// A name and its number, as engine_api.Effort takes them.
 pub const Effort = struct { name: []const u8, value: i64 };
 
diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig
index e3580135e7..df94ce6b88 100644
--- a/zig/src/native/cuda.zig
+++ b/zig/src/native/cuda.zig
@@ -477,6 +477,8 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O
         .vision = if (@hasField(@TypeOf(loaded), "vision")) vision_offer.offer(loaded.vision) else null,
         .lane_vision = if (@hasField(@TypeOf(loaded), "lane_vision")) loaded.lane_vision else null,
         .efforts = familyEfforts(F),
+        .plain_message_text = @hasDecl(F, "plain_assistant_keep"),
+        .plain_assistant_keep = if (@hasDecl(F, "plain_assistant_keep")) F.plain_assistant_keep else &.{},
     }, .{ .ctx = loaded.ctx, .text = F.explain });
     opened = true;
     return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h };
diff --git a/zig/src/server/hf_text.zig b/zig/src/server/hf_text.zig
index e4617f5c7c..99c9227c1a 100644
--- a/zig/src/server/hf_text.zig
+++ b/zig/src/server/hf_text.zig
@@ -212,6 +212,8 @@ pub const HfText = struct {
             .special = specialFn,
             .token_bytes = tokenBytesFn,
             .encode_plain = encodePlainFn,
+            .added_spellings = addedFn,
+            .encode_cut = encodeCutFn,
         } };
     }
 
@@ -236,6 +238,17 @@ pub const HfText = struct {
         return self(ctx).tok.encodePlain(a, s) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else error.Template;
     }
 
+    fn addedFn(ctx: *anyopaque, a: Allocator) Allocator.Error![]const []const u8 {
+        const added = self(ctx).tok.added;
+        const out = try a.alloc([]const u8, added.len);
+        for (added, out) |t, *o| o.* = t.content;
+        return out;
+    }
+
+    fn encodeCutFn(ctx: *anyopaque, a: Allocator, s: []const u8, restore: model_text.Text.Restore) model_text.Error![]u32 {
+        return self(ctx).tok.encodeCut(a, s, .{ .ctx = restore.ctx, .spelled = restore.spelled }) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else error.Template;
+    }
+
     fn decodeFn(ctx: *anyopaque, a: Allocator, ids: []const u32) Allocator.Error![]u8 {
         return self(ctx).tok.decode(a, ids, false) catch |e| if (e == error.OutOfMemory) error.OutOfMemory else a.dupe(u8, "\u{fffd}");
     }
diff --git a/zig/src/server/model_text.zig b/zig/src/server/model_text.zig
index cd7ae359a6..32d17ae085 100644
--- a/zig/src/server/model_text.zig
+++ b/zig/src/server/model_text.zig
@@ -42,6 +42,16 @@ pub const Text = struct {
         encode_plain: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8) Error![]u32 = null,
         /// A token's own bytes (a partial UTF-8 sequence kept whole) for logprob rows; null: the tokenizer can't say.
         token_bytes: ?*const fn (ctx: *anyopaque, a: Allocator, id: u32) Allocator.Error!?[]u8 = null,
+        /// Every added token's text (plain_text.zig shields these in request text); null: none are known.
+        added_spellings: ?*const fn (ctx: *anyopaque, a: Allocator) Allocator.Error![]const []const u8 = null,
+        /// Ids of a shielded prompt: each added token spelled exactly is its token, the text between restored and plain.
+        encode_cut: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8, restore: Restore) Error![]u32 = null,
+    };
+
+    /// What encode_cut does to the text between two cuts: a shielded request's markers spelled again.
+    pub const Restore = struct {
+        ctx: *const anyopaque,
+        spelled: *const fn (ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8,
     };
 
     pub fn encode(t: Text, a: Allocator, text: []const u8, add_special: bool) Error![]u32 {
@@ -53,6 +63,18 @@ pub const Text = struct {
         return f(t.ctx, a, text);
     }
 
+    /// Every added token's text, or none when this text cannot list them.
+    pub fn addedSpellings(t: Text, a: Allocator) Allocator.Error![]const []const u8 {
+        const f = t.vtable.added_spellings orelse return &.{};
+        return f(t.ctx, a);
+    }
+
+    /// The ids of a shielded prompt (plain_text.zig); null when this text has no cut encode.
+    pub fn encodeCut(t: Text, a: Allocator, text: []const u8, restore: Restore) ?Error![]u32 {
+        const f = t.vtable.encode_cut orelse return null;
+        return f(t.ctx, a, text, restore);
+    }
+
     pub fn decode(t: Text, a: Allocator, ids: []const u32) Allocator.Error![]u8 {
         return t.vtable.decode(t.ctx, a, ids);
     }
diff --git a/zig/src/server/plain_text.zig b/zig/src/server/plain_text.zig
new file mode 100644
index 0000000000..c87cc9b4e9
--- /dev/null
+++ b/zig/src/server/plain_text.zig
@@ -0,0 +1,278 @@
+//! Request text stays plain text (Python's server/plain_text.py): an added-token spelling that a client writes in a
+//! message, a tool result or a tool definition is ordinary text, not that token. The chat template makes one string and
+//! the tokenizer would turn each spelling in it into its token, so a client that writes `</think>` or `<｜User｜>` could
+//! forge a thinking end or a turn. `shield` puts a marker in place of each spelling in the request's strings before
+//! the template runs; `encode` then makes the spellings that remain (the template's own) tokens, and encodes the text
+//! between them, with the markers spelled again, with no added token matched.
+const std = @import("std");
+const json = @import("json");
+const model_text = @import("model_text.zig");
+const Value = json.Value;
+const Allocator = std.mem.Allocator;
+
+/// Leftmost-longest matcher over the added-token spellings, as Python's prefix-tree pattern finds them.
+pub const Spellings = struct {
+    first: [256]u32 = @splat(0),
+    edges: std.AutoHashMapUnmanaged(u64, u32) = .empty,
+    /// For each node: the spelling that ends there, else null.
+    ends: std.ArrayList(?[]const u8) = .empty,
+
+    pub fn init(a: Allocator, words: []const []const u8) Allocator.Error!Spellings {
+        var m: Spellings = .{};
+        try m.ends.append(a, null);
+        for (words) |w| {
+            if (w.len == 0) continue;
+            var node: u32 = 0;
+            for (w, 0..) |b, i| {
+                const slot = if (i == 0) &m.first[b] else (try m.edges.getOrPutValue(a, @as(u64, node) << 8 | b, 0)).value_ptr;
+                if (slot.* == 0) {
+                    try m.ends.append(a, null);
+                    slot.* = @intCast(m.ends.items.len - 1);
+                }
+                node = slot.*;
+            }
+            m.ends.items[node] = w;
+        }
+        return m;
+    }
+
+    const Hit = struct { start: usize, end: usize, word: []const u8 };
+
+    fn next(m: *const Spellings, s: []const u8, from: usize) ?Hit {
+        for (from..s.len) |i| {
+            var node = m.first[s[i]];
+            if (node == 0) continue;
+            var best: ?Hit = null;
+            var j = i + 1;
+            while (true) {
+                if (m.ends.items[node]) |w| best = .{ .start = i, .end = j, .word = w };
+                if (j == s.len) break;
+                node = m.edges.get(@as(u64, node) << 8 | s[j]) orelse break;
+                j += 1;
+            }
+            if (best) |b| return b;
+        }
+        return null;
+    }
+};
+
+/// A request's messages and tools after `shield`: `table[n]` is the spelling that marker `prefix` n `@` replaced.
+pub const Shielded = struct {
+    messages: Value,
+    tools: []const Value,
+    table: []const []const u8 = &.{},
+    prefix: []const u8 = "",
+
+    /// `text` with each marker turned back into its spelling.
+    pub fn spelled(s: *const Shielded, a: Allocator, text: []const u8) Allocator.Error![]const u8 {
+        if (s.table.len == 0 or std.mem.indexOf(u8, text, s.prefix) == null) return text;
+        var out: std.ArrayList(u8) = .empty;
+        var rest = text;
+        while (std.mem.indexOf(u8, rest, s.prefix)) |at| {
+            const after = rest[at + s.prefix.len ..];
+            var k: usize = 0;
+            while (k < after.len and std.ascii.isDigit(after[k])) k += 1;
+            const n = if (k > 0 and k < after.len and after[k] == '@') std.fmt.parseInt(usize, after[0..k], 10) catch null else null;
+            if (n == null or n.? >= s.table.len) { // not one of ours: ordinary text
+                try out.appendSlice(a, rest[0 .. at + s.prefix.len]);
+                rest = after;
+                continue;
+            }
+            try out.appendSlice(a, rest[0..at]);
+            try out.appendSlice(a, s.table[n.?]);
+            rest = after[k + 1 ..];
+        }
+        try out.appendSlice(a, rest);
+        return out.items;
+    }
+
+    fn spelledFn(ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8 {
+        const s: *const Shielded = @ptrCast(@alignCast(ctx));
+        return s.spelled(a, text);
+    }
+};
+
+/// The shield for one tokenizer: its added-token spellings, and those an assistant's content and reasoning keep.
+pub const PlainText = struct {
+    spellings: Spellings,
+    assistant_keep: []const []const u8,
+
+    /// Null when the text cannot list its added tokens or cannot encode a shielded prompt.
+    pub fn init(a: Allocator, text: model_text.Text, assistant_keep: []const []const u8) Allocator.Error!?PlainText {
+        if (text.vtable.encode_cut == null) return null;
+        const words = try text.addedSpellings(a);
+        if (words.len == 0) return null;
+        return .{ .spellings = try Spellings.init(a, words), .assistant_keep = assistant_keep };
+    }
+
+    /// `messages` and `tools` with each added-token spelling as a marker; the same values when none is there.
+    /// `content_keep`: spellings that the server itself wrote into this request's message content.
+    pub fn shield(p: *const PlainText, a: Allocator, messages: Value, tools: []const Value, content_keep: []const []const u8) Allocator.Error!Shielded {
+        var attempt: u64 = 0;
+        while (true) : (attempt += 1) {
+            var w: Walk = .{ .p = p, .a = a, .prefix = try std.fmt.allocPrint(a, "@TF{x:0>16}:", .{std.hash.Wyhash.hash(attempt, "tensorfold plain text")}) };
+            const shielded_messages = try w.messages(messages, content_keep);
+            const shielded_tools = try a.alloc(Value, tools.len);
+            for (tools, shielded_tools) |t, *o| o.* = try w.value(t, &.{});
+            if (w.table.items.len == 0) return .{ .messages = messages, .tools = tools };
+            // a marker is unique: no string of the request holds the prefix
+            if (!holds(messages, w.prefix) and !holds(.{ .array = @constCast(tools) }, w.prefix))
+                return .{ .messages = shielded_messages, .tools = shielded_tools, .table = w.table.items, .prefix = w.prefix };
+        }
+    }
+
+    /// The ids of rendered `text`: the template's spellings as tokens, the rest (markers spelled again) without them.
+    /// DeepSeek's normalizer changes no text, so a prompt without a marker is the tokenizer's own encode.
+    pub fn encode(p: *const PlainText, text: model_text.Text, a: Allocator, rendered: []const u8, s: *const Shielded) model_text.Error![]u32 {
+        _ = p;
+        if (s.table.len == 0 or std.mem.indexOf(u8, rendered, s.prefix) == null) return text.encode(a, rendered, false);
+        return text.encodeCut(a, rendered, .{ .ctx = s, .spelled = Shielded.spelledFn }) orelse text.encode(a, rendered, false);
+    }
+};
+
+const Walk = struct {
+    p: *const PlainText,
+    a: Allocator,
+    prefix: []const u8,
+    table: std.ArrayList([]const u8) = .empty,
+    index: std.StringHashMapUnmanaged(usize) = .empty,
+
+    fn mark(w: *Walk, out: *std.ArrayList(u8), spelling: []const u8) Allocator.Error!void {
+        const slot = try w.index.getOrPut(w.a, spelling);
+        if (!slot.found_existing) {
+            slot.value_ptr.* = w.table.items.len;
+            try w.table.append(w.a, spelling);
+        }
+        try out.print(w.a, "{s}{d}@", .{ w.prefix, slot.value_ptr.* });
+    }
+
+    fn string(w: *Walk, s: []const u8, keep: []const []const u8) Allocator.Error![]const u8 {
+        var out: std.ArrayList(u8) = .empty;
+        var at: usize = 0;
+        var changed = false;
+        while (w.p.spellings.next(s, at)) |hit| {
+            if (contains(keep, hit.word)) {
+                try out.appendSlice(w.a, s[at..hit.end]);
+            } else {
+                try out.appendSlice(w.a, s[at..hit.start]);
+                try w.mark(&out, hit.word);
+                changed = true;
+            }
+            at = hit.end;
+        }
+        if (!changed) return s;
+        try out.appendSlice(w.a, s[at..]);
+        return out.items;
+    }
+
+    /// Python's _walk: strings, object keys and values, list items, all with the same `keep`.
+    fn value(w: *Walk, v: Value, keep: []const []const u8) Allocator.Error!Value {
+        return switch (v) {
+            .string => |s| .{ .string = try w.string(s, keep) },
+            .array => |items| blk: {
+                const out = try w.a.alloc(Value, items.len);
+                for (items, out) |x, *o| o.* = try w.value(x, keep);
+                break :blk .{ .array = out };
+            },
+            .object => |o| blk: {
+                const out = try json.newObject(w.a);
+                var it = o.iterator();
+                while (it.next()) |kv| try out.put(w.a, try w.string(kv.key_ptr.*, keep), try w.value(kv.value_ptr.*, keep));
+                break :blk .{ .object = out };
+            },
+            else => v,
+        };
+    }
+
+    /// Python's _messages: each message's text parts joined first (a spelling can span two parts), an assistant's
+    /// content and reasoning keeping the assistant spellings, the server's content spellings kept in any content.
+    fn messages(w: *Walk, v: Value, content_keep: []const []const u8) Allocator.Error!Value {
+        if (v != .array) return w.value(v, &.{});
+        const out = try w.a.alloc(Value, v.array.len);
+        for (v.array, out) |m, *o| {
+            if (m != .object) {
+                o.* = try w.value(m, &.{});
+                continue;
+            }
+            const role = m.get("role");
+            const assistant = role != null and role.? == .string and std.mem.eql(u8, role.?.string, "assistant");
+            const both = try std.mem.concat(w.a, []const u8, &.{ content_keep, if (assistant) w.p.assistant_keep else &.{} });
+            const item = try json.newObject(w.a);
+            var it = m.object.iterator();
+            while (it.next()) |kv| {
+                const key = kv.key_ptr.*;
+                var x = kv.value_ptr.*;
+                if (std.mem.eql(u8, key, "content")) x = try joined(w.a, x);
+                const keep: []const []const u8 = if (std.mem.eql(u8, key, "content")) both else if (std.mem.eql(u8, key, "reasoning_content") and assistant) w.p.assistant_keep else &.{};
+                try item.put(w.a, try w.string(key, &.{}), try w.value(x, keep));
+            }
+            o.* = .{ .object = item };
+        }
+        return .{ .array = out };
+    }
+};
+
+/// Text parts as one string (every part a "text" object with a string): Python's _joined.
+fn joined(a: Allocator, content: Value) Allocator.Error!Value {
+    if (content != .array or content.array.len == 0) return content;
+    for (content.array) |part| {
+        if (part != .object) return content;
+        const t = part.get("type") orelse return content;
+        if (t != .string or !std.mem.eql(u8, t.string, "text")) return content;
+        const text = part.get("text") orelse return content;
+        if (text != .string) return content;
+    }
+    var out: std.ArrayList(u8) = .empty;
+    for (content.array) |part| try out.appendSlice(a, part.get("text").?.string);
+    return .{ .string = out.items };
+}
+
+fn contains(list: []const []const u8, word: []const u8) bool {
+    for (list) |x| if (std.mem.eql(u8, x, word)) return true;
+    return false;
+}
+
+/// Whether a string of `v` (an object key included) holds `needle`.
+fn holds(v: Value, needle: []const u8) bool {
+    return switch (v) {
+        .string => |s| std.mem.indexOf(u8, s, needle) != null,
+        .array => |items| for (items) |x| {
+            if (holds(x, needle)) break true;
+        } else false,
+        .object => |o| blk: {
+            var it = o.iterator();
+            while (it.next()) |kv| if (std.mem.indexOf(u8, kv.key_ptr.*, needle) != null or holds(kv.value_ptr.*, needle)) break :blk true;
+            break :blk false;
+        },
+        else => false,
+    };
+}
+
+test "a spelling in request text becomes a marker, the kept ones stay, and the markers spell back" {
+    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+    defer arena.deinit();
+    const a = arena.allocator();
+    const words = [_][]const u8{ "<think>", "</think>", "<｜User｜>", "<｜ref｜>", "<｜deepseek_image｜>" };
+    const p: PlainText = .{ .spellings = try Spellings.init(a, &words), .assistant_keep = &.{"<｜ref｜>"} };
+    const messages = (try json.parse(a,
+        \\[{"role":"user","content":[{"type":"text","text":"a<thi"},{"type":"text","text":"nk>b <｜ref｜> <｜deepseek_image｜>"}]},
+        \\ {"role":"assistant","content":"<｜ref｜>x</think>","reasoning_content":"<｜ref｜><｜User｜>"}]
+    )).ok;
+    const s = try p.shield(a, messages, &.{}, &.{"<｜deepseek_image｜>"});
+    try std.testing.expect(s.table.len == 4);
+    const user = s.messages.array[0].strField("content").?;
+    const assistant = s.messages.array[1].strField("content").?;
+    const reasoning = s.messages.array[1].strField("reasoning_content").?;
+    // a spelling across two text parts is found; grounding in a user turn is plain; the server's placeholder stays
+    try std.testing.expect(std.mem.indexOf(u8, user, "<think>") == null and std.mem.indexOf(u8, user, "<｜ref｜>") == null);
+    try std.testing.expect(std.mem.indexOf(u8, user, "<｜deepseek_image｜>") != null);
+    try std.testing.expect(std.mem.startsWith(u8, assistant, "<｜ref｜>x@TF") and std.mem.startsWith(u8, reasoning, "<｜ref｜>@TF"));
+    try std.testing.expectEqualStrings("a<think>b <｜ref｜> <｜deepseek_image｜>", try s.spelled(a, user));
+    try std.testing.expectEqualStrings("<｜ref｜><｜User｜>", try s.spelled(a, reasoning));
+    // no spelling: the request's own values, unjoined
+    const plain = (try json.parse(a,
+        \\[{"role":"user","content":[{"type":"text","text":"hi"}]}]
+    )).ok;
+    const none = try p.shield(a, plain, &.{}, &.{});
+    try std.testing.expect(none.table.len == 0 and none.messages.array[0].get("content").? == .array);
+}
diff --git a/zig/src/server/prompt.zig b/zig/src/server/prompt.zig
index d77c4332f7..35069d08c0 100644
--- a/zig/src/server/prompt.zig
+++ b/zig/src/server/prompt.zig
@@ -31,6 +31,11 @@ pub const isTitle = messages_mod.isTitleRequest;
 
 /// ``render_prompt_ids``: messages normalized for this template, rendered, and a dangling ``<think>`` closed.
 pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thinking: bool, effort: ?[]const u8, generation: bool) errors.Refused![]const u32 {
+    return renderKept(srv, cx, messages, tools, thinking, effort, generation, &.{});
+}
+
+/// renderIds; `kept`: spellings that the server itself wrote into the messages' content (a lane image's placeholder).
+fn renderKept(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thinking: bool, effort: ?[]const u8, generation: bool, kept: []const []const u8) errors.Refused![]const u32 {
     const normalized = try messages_mod.toolArguments(cx, try messages_mod.normalizeWith(cx, messages, srv.late_system, srv.needs_user_after_tool, .of(srv.info.vision)));
     var problem: []const u8 = "";
     const options: model_text.RenderOptions = .{
@@ -40,7 +45,7 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t
         .reasoning_effort = if (thinking) effort else null,
         .reasoning_effort_value = if (thinking) srv.effortValue(effort) else null,
     };
-    var ids = render(srv, cx.a, normalized, options, &problem) catch |e| switch (e) {
+    var ids = render(srv, cx.a, normalized, options, kept, &problem) catch |e| switch (e) {
         error.OutOfMemory => return error.OutOfMemory,
         error.Template => return cx.fail(.server, "{s}", .{problem}),
     };
@@ -57,8 +62,20 @@ pub fn renderIds(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, t
     return ids;
 }
 
-/// The template's ids for `messages`; with image input, a marker the text spells is ordinary text (images.zig).
-fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, problem: *[]const u8) model_text.Error![]u32 {
+/// The template's ids for `messages`; with image input, a marker the text spells is ordinary text (images.zig), and
+/// with Info.plain_message_text every added-token spelling the request writes is (plain_text.zig).
+fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, kept: []const []const u8, problem: *[]const u8) model_text.Error![]u32 {
+    if (srv.plain) |*p| {
+        const tools: []const Value = if (options.tools) |t| t.array else &.{};
+        const s = try p.shield(a, messages, tools, kept);
+        var shielded = options;
+        if (options.tools != null) shielded.tools = .{ .array = @constCast(s.tools) };
+        const text = srv.text.render(a, s.messages, shielded, problem) catch |e| {
+            if (e == error.Template) problem.* = try s.spelled(a, problem.*);
+            return e;
+        };
+        return p.encode(srv.text, a, text, &s);
+    }
     const v = srv.info.vision orelse return srv.text.renderIds(a, messages, options, problem);
     const markers = try images.markerText(srv.text, a, v);
     const text = try srv.text.render(a, try images.shield(a, messages, markers), options, problem);
@@ -109,7 +126,10 @@ pub fn prepare(srv: *Server, cx: *Cx, input: chat.Input, thinking: bool, effort:
         } },
         .ids => |ids| return .{ .ids = ids },
     };
-    const prompt = try renderIds(srv, cx, input.messages, input.tools, thinking, effort, true);
+    // a lane image's placeholder in content is the server's own: normalizeImages refused each one the client wrote
+    const placeholder = [1][]const u8{if (srv.info.lane_vision) |v| v.placeholder else ""};
+    const kept: []const []const u8 = if (input.lane_images.len > 0 and srv.info.lane_vision != null) &placeholder else &.{};
+    const prompt = try renderKept(srv, cx, input.messages, input.tools, thinking, effort, true, kept);
     if (input.lane_images.len > 0) return laneSpans(srv, cx, prompt, input.lane_images); // (no kept history: the cache keys tokens)
     const history = try renderIds(srv, cx, input.messages, input.tools, thinking, effort, false);
     var history_len: usize = 0;
diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig
index 06be1ffc75..5ba0a3346a 100644
--- a/zig/src/server/root.zig
+++ b/zig/src/server/root.zig
@@ -43,6 +43,7 @@ test {
     _ = @import("request_log.zig");
     _ = @import("messages.zig");
     _ = @import("images.zig");
+    _ = @import("plain_text.zig");
     _ = @import("image_key.zig");
     _ = @import("media_fetch.zig");
     _ = @import("videos.zig");
diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig
index d048ab0c17..fbd58ae794 100644
--- a/zig/src/server/server.zig
+++ b/zig/src/server/server.zig
@@ -7,6 +7,7 @@ const fields = @import("fields.zig");
 const auth = @import("auth.zig");
 const metrics = @import("metrics.zig");
 const model_text = @import("model_text.zig");
+const plain_text = @import("plain_text.zig");
 const reply_text = @import("reply_text.zig");
 const http_conn = @import("http_conn.zig");
 const listener_mod = @import("listener.zig");
@@ -88,6 +89,8 @@ pub const Server = struct {
     config: Config,
     keys: ?*auth.Store,
     late_system: []const u8 = "system",
+    /// Info.plain_message_text's shield; null: request text encodes as the template renders it.
+    plain: ?plain_text.PlainText = null,
     needs_user_after_tool: bool = false,
     markers: reply_text.Markers = reply_text.think_markers,
     chunks: chunk_plan.Plan = .{},
@@ -147,6 +150,7 @@ pub const Server = struct {
         srv.thinking_switch = std.mem.indexOf(u8, source, "enable_thinking") != null or
             std.mem.indexOf(u8, source, "thinking_mode") != null;
         srv.late_system = try lateSystem(a, text);
+        if (srv.info.plain_message_text) srv.plain = try plain_text.PlainText.init(a, text, srv.info.plain_assistant_keep);
         srv.needs_user_after_tool = needsUserAfterTool(source);
         if (text.tokenId("</think>")) |end| {
             const lead = try text.encode(a, "\n", false);

From 17b1dd4788554eac10abda00d15782c35f744fdd Mon Sep 17 00:00:00 2001
From: Elevatormusic <22101396+Elevatormusic@users.noreply.github.com>
Date: Thu, 22 Oct 2026 00:13:02 -0700
Subject: [PATCH 2/4] tokenizer: the cut-encode test's added tokens take ids
 past the model vocabulary

tokenizers gives an added token that the model vocabulary lacks the next id past the model, not its listed id, so the test vocabulary's ids 9-11 clashed with the two added tokens.
---
 zig/src/core/tokenizer/tokenizer.zig | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig
index a335c91cb7..69883ccf5a 100644
--- a/zig/src/core/tokenizer/tokenizer.zig
+++ b/zig/src/core/tokenizer/tokenizer.zig
@@ -445,13 +445,13 @@ test "byte-level BPE with merges, added tokens and decode" {
 
 test "a cut encode: the text's own added tokens are tokens, the restored text between them is plain text" {
     const json =
-        \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
-        \\  {"id":8,"content":"<t>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}],
+        \\{"added_tokens":[{"id":10,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
+        \\  {"id":11,"content":"<t>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}],
         \\ "normalizer":{"type":"Sequence","normalizers":[]},
         \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
         \\ "decoder":{"type":"ByteLevel"},
         \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
-        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":9,"t":10,">":11},"merges":[["Ġ","a"],"a b","Ġa b"]}}
+        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":7,"t":8,">":9},"merges":[["Ġ","a"],"a b","Ġa b"]}}
     ;
     const a = std.testing.allocator;
     var t = try parse(a, json);
@@ -464,8 +464,8 @@ test "a cut encode: the text's own added tokens are tokens, the restored text be
     const restore: Tokenizer.Restore = .{ .ctx = &t, .spelled = Marks.spelled };
     const ids = try t.encodeCut(a, "ab<|end|>a@0@b<t>c", restore);
     defer a.free(ids);
-    // "<t>" from the marker is three plain tokens; the text's own "<t>" is token 8
-    try std.testing.expectEqualSlices(u32, &.{ 3, 7, 0, 9, 10, 11, 1, 8, 6 }, ids);
+    // added tokens take ids past the model's 10 (tokenizers' rule): "<|end|>" 10, "<t>" 11; the marker's "<t>" is plain
+    try std.testing.expectEqualSlices(u32, &.{ 3, 10, 0, 7, 8, 9, 1, 11, 6 }, ids);
     // no marker and no added token: the same ids as encode, through the memo too
     t.memo = try Memo.create(a, 1 << 20);
     const plain = try t.encode(a, "ab abc");

From 5ea721f071c2d1a1191ac21234c412f8a4b4ba82 Mon Sep 17 00:00:00 2001
From: Elevatormusic <22101396+Elevatormusic@users.noreply.github.com>
Date: Sat, 24 Oct 2026 11:17:09 -0700
Subject: [PATCH 3/4] zig: zig fmt on five dsv41-zig files (the Zig format
 check fails on every pull request)

zig fmt 0.17.0 changes only layout and @intFromEnum to @backingInt.
The Zig format check fails on these five base-branch files in every
open dsv41-zig pull request, also where the pull request does not touch them.
---
 zig/src/families/dsv41/exl3_linear.zig      | 16 ++++++++--------
 zig/src/families/dsv41/round.zig            |  5 ++---
 zig/src/families/dsv41/split_bench_main.zig | 19 ++++++++++++++++---
 zig/src/families/dsv41/tri_basic.zig        |  6 +++---
 zig/src/server/effort_values_test.zig       |  2 +-
 5 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/zig/src/families/dsv41/exl3_linear.zig b/zig/src/families/dsv41/exl3_linear.zig
index 25a9241925..dcce543ad1 100644
--- a/zig/src/families/dsv41/exl3_linear.zig
+++ b/zig/src/families/dsv41/exl3_linear.zig
@@ -164,7 +164,7 @@ pub fn rotMany(k: *const Kernels, stream: cuda.Stream, calls: []const Call, m: u
     var a: RArgs = .{ .n = @intCast(calls.len), .M = @intCast(m) };
     var warps: usize = 0;
     for (calls, 0..) |c, i| {
-        a.l[i] = .{ .x = c.x, .suh = c.layer.suh, .xh = c.xh, .ldx = c.ldx, .x_dtype = @intFromEnum(c.x_dtype), .K = @intCast(c.layer.k), .first = @intCast(warps) };
+        a.l[i] = .{ .x = c.x, .suh = c.layer.suh, .xh = c.xh, .ldx = c.ldx, .x_dtype = @backingInt(c.x_dtype), .K = @intCast(c.layer.k), .first = @intCast(warps) };
         warps += m * (c.layer.k / 128);
     }
     var args: cuda.Args = .{};
@@ -189,7 +189,7 @@ pub fn glinear(k: *const Kernels, stream: cuda.Stream, calls: []const Call, m: u
             done[j] = true;
             const s = strides(c.layer);
             const n: usize = @intCast(a.n);
-            a.l[n] = .{ .xh = c.xh, .ldx = @intCast(c.layer.k), .T = c.layer.words, .stride_k = s[0], .stride_nb = s[1], .svh = c.layer.svh, .y = c.y, .y_dtype = @intFromEnum(c.y_dtype), .ldy = c.ldy, .K = @intCast(c.layer.k), .N = @intCast(c.layer.n), .SK = @intCast(pj.sk), .counters = c.counters, .first = @intCast(blocks) };
+            a.l[n] = .{ .xh = c.xh, .ldx = @intCast(c.layer.k), .T = c.layer.words, .stride_k = s[0], .stride_nb = s[1], .svh = c.layer.svh, .y = c.y, .y_dtype = @backingInt(c.y_dtype), .ldy = c.ldy, .K = @intCast(c.layer.k), .N = @intCast(c.layer.n), .SK = @intCast(pj.sk), .counters = c.counters, .first = @intCast(blocks) };
             if (c.rot) |r| {
                 a.l[n].rsuh = r.suh;
                 a.l[n].rxh = r.xh;
@@ -243,12 +243,12 @@ test "plans as linear.py makes them" {
 test "plans and strides equal every recorded glinear layer of the served build" {
     // (K, N, SK, WK, K2, stride_k, stride_nb) of each distinct layer the recording's glinear calls passed
     const seen = [_][7]u32{
-        .{ 512, 128, 1, 4, 16, 512, 16384 },       .{ 1280, 4096, 2, 4, 10, 320, 25600 },     .{ 1280, 16384, 1, 8, 8, 256, 20480 },
-        .{ 1280, 16384, 1, 8, 10, 320, 25600 },    .{ 3072, 25600, 1, 8, 8, 256, 49152 },     .{ 3072, 25600, 1, 8, 10, 320, 61440 },
-        .{ 4096, 1024, 8, 4, 8, 256, 65536 },      .{ 4096, 1024, 8, 4, 10, 320, 81920 },     .{ 4096, 5120, 8, 4, 8, 256, 65536 },
-        .{ 4096, 5120, 8, 4, 10, 320, 81920 },     .{ 5120, 512, 8, 4, 8, 256, 81920 },       .{ 5120, 512, 8, 4, 10, 320, 102400 },
-        .{ 5120, 512, 8, 4, 12, 384, 122880 },     .{ 5120, 1280, 8, 4, 8, 256, 81920 },      .{ 5120, 1280, 8, 4, 10, 320, 102400 },
-        .{ 5120, 1280, 8, 4, 12, 384, 122880 },    .{ 5120, 64640, 1, 8, 12, 384, 122880 },   .{ 15360, 5120, 8, 4, 8, 256, 245760 },
+        .{ 512, 128, 1, 4, 16, 512, 16384 },    .{ 1280, 4096, 2, 4, 10, 320, 25600 },   .{ 1280, 16384, 1, 8, 8, 256, 20480 },
+        .{ 1280, 16384, 1, 8, 10, 320, 25600 }, .{ 3072, 25600, 1, 8, 8, 256, 49152 },   .{ 3072, 25600, 1, 8, 10, 320, 61440 },
+        .{ 4096, 1024, 8, 4, 8, 256, 65536 },   .{ 4096, 1024, 8, 4, 10, 320, 81920 },   .{ 4096, 5120, 8, 4, 8, 256, 65536 },
+        .{ 4096, 5120, 8, 4, 10, 320, 81920 },  .{ 5120, 512, 8, 4, 8, 256, 81920 },     .{ 5120, 512, 8, 4, 10, 320, 102400 },
+        .{ 5120, 512, 8, 4, 12, 384, 122880 },  .{ 5120, 1280, 8, 4, 8, 256, 81920 },    .{ 5120, 1280, 8, 4, 10, 320, 102400 },
+        .{ 5120, 1280, 8, 4, 12, 384, 122880 }, .{ 5120, 64640, 1, 8, 12, 384, 122880 }, .{ 15360, 5120, 8, 4, 8, 256, 245760 },
     };
     for (seen) |r| {
         const p = plan(r[0], r[1]);
diff --git a/zig/src/families/dsv41/round.zig b/zig/src/families/dsv41/round.zig
index c674aa0c3f..59dac0c8de 100644
--- a/zig/src/families/dsv41/round.zig
+++ b/zig/src/families/dsv41/round.zig
@@ -446,9 +446,9 @@ pub const Round = struct {
     fn mark(rd: *Round, e: *const Engine, p: PhaseTimer.Phase) !void {
         if (rd.clk != 0 and rd.clk_marks and (rd.clk_ratio == null or rd.clk_ratio.? == rd.cur_ratio)) {
             const now = cuda.launch.count;
-            rd.launches[@intFromEnum(p)] += now -% rd.launch_mark;
+            rd.launches[@backingInt(p)] += now -% rd.launch_mark;
             rd.launch_mark = now;
-            try rd.clock(e, @intFromEnum(p));
+            try rd.clock(e, @backingInt(p));
         }
         if (rd.timer) |t| try t.mark(e.s, p);
     }
@@ -801,7 +801,6 @@ fn linearBytes(l: weights.Linear) [2]u64 {
     return .{ l.words, @as(u64, l.k) * l.n * l.k2 / 16 };
 }
 
-
 /// A measurement's L2 prefetches left out (TF_DS_L2_SKIP, read at Model.open: bit 1 the "attn" fork after q, 2 the "moe"
 /// fork after wo_b, 4 the "next" fork after the experts); 0 in service.
 pub var l2_skip: u8 = 0;
diff --git a/zig/src/families/dsv41/split_bench_main.zig b/zig/src/families/dsv41/split_bench_main.zig
index fc8952dd2a..7980a7e5ef 100644
--- a/zig/src/families/dsv41/split_bench_main.zig
+++ b/zig/src/families/dsv41/split_bench_main.zig
@@ -320,10 +320,23 @@ pub fn main(init: std.process.Init) !u8 {
             try scales.upload(0, hs);
         }
         const run: Run = .{
-            .t = t, .o = &o, .s = stream, .top = top, .n = n, .rb = rb,
+            .t = t,
+            .o = &o,
+            .s = stream,
+            .top = top,
+            .n = n,
+            .rb = rb,
             .kx = .{ .fp4 = .{ .codes = codes.ptr, .scales = scales.ptr } },
-            .q = q.ptr, .w = w.ptr, .vis = vis.ptr, .nvis = nvis.ptr, .keys = keys.ptr, .tmax = tmax.ptr, .bufs = bufs,
-            .parts = parts.ptr, .merged = merged.ptr, .vblk = vblk.ptr,
+            .q = q.ptr,
+            .w = w.ptr,
+            .vis = vis.ptr,
+            .nvis = nvis.ptr,
+            .keys = keys.ptr,
+            .tmax = tmax.ptr,
+            .bufs = bufs,
+            .parts = parts.ptr,
+            .merged = merged.ptr,
+            .vblk = vblk.ptr,
         };
 
         // every output byte, both splits, both patterns; the served keys past vis
diff --git a/zig/src/families/dsv41/tri_basic.zig b/zig/src/families/dsv41/tri_basic.zig
index 66cb3470a5..34acadee40 100644
--- a/zig/src/families/dsv41/tri_basic.zig
+++ b/zig/src/families/dsv41/tri_basic.zig
@@ -19,9 +19,9 @@ fn ci(name: []const u8, v: usize) aot.Const {
 pub fn hcPre(t: Tri, h: u64, fn_w: u64, scale: u64, base: u64, pre_in: u64, norm_w: u64, eps: f32, hc_eps: f32, iters: usize, out: u64, pre_out: u64, post: u64, comb: u64, part: u64, rows: usize, d: usize) !void {
     try t.run("_hc_partial", .{ u(rows), hc_blocks, 1 }, &.{ p("X", "*bf16", h), p("FN", "*fp32", fn_w), p("PART", "*fp32", part) }, &.{ ci("WIDE", 4 * d), ci("NB", hc_blocks), ci("SUB", 128) });
     try t.run("_hc_finish", .{ u(rows), 1, 1 }, &.{
-        p("X", "*bf16", h),          p("PART", "*fp32", part),       p("BASE", "*fp32", base), p("SCALE", "*fp32", scale),
-        p("PRE_IN", "*fp32", pre_in), p("NW", "*bf16", norm_w),      p("OUT", "*bf16", out),   p("PRE_OUT", "*fp32", pre_out),
-        p("POST", "*fp32", post),    p("COMB", "*fp32", comb),       aot.float("eps", eps),    aot.float("hc_eps", hc_eps),
+        p("X", "*bf16", h),           p("PART", "*fp32", part), p("BASE", "*fp32", base), p("SCALE", "*fp32", scale),
+        p("PRE_IN", "*fp32", pre_in), p("NW", "*bf16", norm_w), p("OUT", "*bf16", out),   p("PRE_OUT", "*fp32", pre_out),
+        p("POST", "*fp32", post),     p("COMB", "*fp32", comb), aot.float("eps", eps),    aot.float("hc_eps", hc_eps),
     }, &.{ ci("D", d), ci("NB", hc_blocks), ci("ITERS", iters), ci("BLOCK", 1024) });
 }
 
diff --git a/zig/src/server/effort_values_test.zig b/zig/src/server/effort_values_test.zig
index f8b01f8ae9..e3ec47b9ef 100644
--- a/zig/src/server/effort_values_test.zig
+++ b/zig/src/server/effort_values_test.zig
@@ -10,7 +10,7 @@ const Conn = @import("http_conn.zig").Conn;
 
 /// DeepSeek-V4.1's numbers (families/dsv41/text.zig; the served lane's EFFORT_VALUES).
 const numbers = [_]api.Effort{
-    .{ .name = "minimal", .value = 25 }, .{ .name = "low", .value = 50 },  .{ .name = "medium", .value = 62 },
+    .{ .name = "minimal", .value = 25 }, .{ .name = "low", .value = 50 },   .{ .name = "medium", .value = 62 },
     .{ .name = "high", .value = 75 },    .{ .name = "xhigh", .value = 88 }, .{ .name = "max", .value = 100 },
 };
 

From ffa7aa930cbd24613874ddf445a4a122d8bc4698 Mon Sep 17 00:00:00 2001
From: Elevatormusic <22101396+Elevatormusic@users.noreply.github.com>
Date: Sat, 24 Oct 2026 11:31:18 -0700
Subject: [PATCH 4/4] tokenizer: cut-encode and memo tests move to
 tokenizer_test.zig; plain-text comments are one line (lean check)

---
 zig/src/core/engine_api.zig               |   2 +-
 zig/src/core/tokenizer/tokenizer.zig      | 121 ++--------------------
 zig/src/core/tokenizer/tokenizer_test.zig | 120 +++++++++++++++++++++
 zig/src/families/dsv41/text.zig           |   3 +-
 zig/src/server/model_text.zig             |   2 +-
 zig/src/server/plain_text.zig             |  15 +--
 zig/src/server/prompt.zig                 |   3 +-
 7 files changed, 134 insertions(+), 132 deletions(-)
 create mode 100644 zig/src/core/tokenizer/tokenizer_test.zig

diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig
index 9bc64012d4..8055b59e4a 100644
--- a/zig/src/core/engine_api.zig
+++ b/zig/src/core/engine_api.zig
@@ -261,7 +261,7 @@ pub const Info = struct {
     /// A lane-vision family's image input (LaneVision: each image part its placeholder, the family's prepare its span);
     /// null: none (upstream's image input is `vision`).
     lane_vision: ?LaneVision = null,
-    /// An added-token spelling in request text is plain text, not its token (server/plain_text.zig; DeepSeek's encoding).
+    /// An added-token spelling in request text is plain text, not its token (server/plain_text.zig).
     plain_message_text: bool = false,
     /// With plain_message_text: the spellings that stay tokens in an assistant's content and reasoning.
     plain_assistant_keep: []const []const u8 = &.{},
diff --git a/zig/src/core/tokenizer/tokenizer.zig b/zig/src/core/tokenizer/tokenizer.zig
index 69883ccf5a..fb317e4229 100644
--- a/zig/src/core/tokenizer/tokenizer.zig
+++ b/zig/src/core/tokenizer/tokenizer.zig
@@ -161,14 +161,13 @@ pub const Tokenizer = struct {
         return a.dupe(u32, ids.items);
     }
 
-    /// What encodeCut does to the text between two cuts before it encodes it: a shielded request's markers spelled again.
+    /// What encodeCut does to the text between two cuts before it encodes it (a shielded request's markers spelled).
     pub const Restore = struct {
         ctx: *const anyopaque,
         spelled: *const fn (ctx: *const anyopaque, a: Allocator, text: []const u8) Allocator.Error![]const u8,
     };
 
-    /// Ids of a rendered prompt as Python's PlainText.encode makes them: each added token that `text` spells exactly
-    /// is a cut and its token; the text between two cuts, after `restore`, is encoded with no added token matched.
+    /// Python's PlainText.encode: exact added-token spellings are cuts; the restored text between has no added token.
     pub fn encodeCut(t: *const Tokenizer, a: Allocator, text: []const u8, restore: Restore) Error![]u32 {
         var arena = std.heap.ArenaAllocator.init(a);
         defer arena.deinit();
@@ -443,118 +442,6 @@ test "byte-level BPE with merges, added tokens and decode" {
     try std.testing.expectEqualStrings("", unknown);
 }
 
-test "a cut encode: the text's own added tokens are tokens, the restored text between them is plain text" {
-    const json =
-        \\{"added_tokens":[{"id":10,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
-        \\  {"id":11,"content":"<t>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}],
-        \\ "normalizer":{"type":"Sequence","normalizers":[]},
-        \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
-        \\ "decoder":{"type":"ByteLevel"},
-        \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
-        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":7,"t":8,">":9},"merges":[["Ġ","a"],"a b","Ġa b"]}}
-    ;
-    const a = std.testing.allocator;
-    var t = try parse(a, json);
-    defer t.deinit();
-    const Marks = struct {
-        fn spelled(_: *const anyopaque, al: Allocator, text: []const u8) Allocator.Error![]const u8 {
-            return std.mem.replaceOwned(u8, al, text, "@0@", "<t>");
-        }
-    };
-    const restore: Tokenizer.Restore = .{ .ctx = &t, .spelled = Marks.spelled };
-    const ids = try t.encodeCut(a, "ab<|end|>a@0@b<t>c", restore);
-    defer a.free(ids);
-    // added tokens take ids past the model's 10 (tokenizers' rule): "<|end|>" 10, "<t>" 11; the marker's "<t>" is plain
-    try std.testing.expectEqualSlices(u32, &.{ 3, 10, 0, 7, 8, 9, 1, 11, 6 }, ids);
-    // no marker and no added token: the same ids as encode, through the memo too
-    t.memo = try Memo.create(a, 1 << 20);
-    const plain = try t.encode(a, "ab abc");
-    defer a.free(plain);
-    const cut = try t.encodeCut(a, "ab abc", restore);
-    defer a.free(cut);
-    try std.testing.expectEqualSlices(u32, plain, cut);
-}
-
-test "with the memo, long segments encode as without it, and the second encode takes their kept ids" {
-    const json =
-        \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],
-        \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
-        \\ "decoder":{"type":"ByteLevel"},
-        \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
-        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6},"merges":[["Ġ","a"],"a b","Ġa b"]}}
-    ;
-    const a = std.testing.allocator;
-    var t = try parse(a, json);
-    defer t.deinit();
-    var text: std.ArrayList(u8) = .empty;
-    defer text.deinit(a);
-    for (0..300) |_| try text.appendSlice(a, "ab abc ");
-    try text.appendSlice(a, "<|end|>");
-    for (0..300) |_| try text.appendSlice(a, " ab cab");
-    const plain = try t.encode(a, text.items);
-    defer a.free(plain);
-    t.memo = try Memo.create(a, 1 << 20);
-    for (0..2) |_| {
-        const kept = try t.encode(a, text.items);
-        defer a.free(kept);
-        try std.testing.expectEqualSlices(u32, plain, kept);
-    }
-    try std.testing.expectEqual(@as(u64, 2), t.memo.?.hits); // both segments, the second time
-}
-
-test "a kept segment's ids equal a fresh encode's wherever the segment sits" {
-    const added =
-        \\{"added_tokens":[{"id":20,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
-        \\ {"id":21,"content":"<|l|>","single_word":false,"lstrip":true,"rstrip":false,"normalized":false,"special":true},
-        \\ {"id":22,"content":"<|r|>","single_word":false,"lstrip":false,"rstrip":true,"normalized":false,"special":true}],
-    ;
-    const kinds = [_][]const u8{
-        added ++
-            \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
-            \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
-            \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"x":7,"ï":8,"¿":9,"½":10},"merges":[["Ġ","a"],"a b","Ġa b"]}}
-        ,
-        added ++
-            \\ "pre_tokenizer":{"type":"Metaspace","replacement":"▁","prepend_scheme":"first","split":true},
-            \\ "model":{"type":"BPE","unk_token":"<unk>","fuse_unk":true,"byte_fallback":true,
-            \\  "vocab":{"<unk>":0,"a":1,"b":2,"▁":3,"ab":4,"▁a":5,"▁ab":6,"c":7,"x":8,"<0xEF>":9,"<0xBF>":10,"<0xBD>":11},"merges":["a b","▁ a","▁a b"]}}
-        ,
-    };
-    const a = std.testing.allocator;
-    var body: std.ArrayList(u8) = .empty;
-    defer body.deinit(a);
-    for (0..200) |i| try body.appendSlice(a, if (i % 3 == 0) "ab abc " else "cab ab ");
-    const b = body.items;
-    // first or not, after lstrip and rstrip tokens, with a leading space, around a lossy byte, repeated in one prompt
-    const texts = [_][]const []const u8{
-        &.{b},               &.{ "<|end|>", b }, &.{ b, "<|end|>", b }, &.{ "x  <|l|>", b },
-        &.{ "<|r|>   ", b }, &.{ " ", b },       &.{ b, "\xff", b },    &.{ "<|end|> ", b, "<|r|>", b },
-    };
-    for (kinds, 0..) |json, kind| {
-        var t = try parse(a, json);
-        defer t.deinit();
-        var fresh: [texts.len][]u32 = undefined;
-        for (texts, &fresh) |parts, *ids| {
-            const s = try std.mem.concat(a, u8, parts);
-            defer a.free(s);
-            ids.* = try t.encode(a, s);
-        }
-        defer for (fresh) |ids| a.free(ids);
-        // the Metaspace tokenizer prepends to the first segment alone, so a memo blind to `first` fails below
-        if (kind == 1) try std.testing.expect(!std.mem.eql(u32, fresh[0], fresh[1][1..]));
-        t.memo = try Memo.create(a, 1 << 20);
-        for (0..2) |round| for (0..texts.len) |k| {
-            const i = if (round == 0) k else texts.len - 1 - k;
-            const s = try std.mem.concat(a, u8, texts[i]);
-            defer a.free(s);
-            const kept = try t.encode(a, s);
-            defer a.free(kept);
-            try std.testing.expectEqualSlices(u32, fresh[i], kept);
-        };
-        try std.testing.expect(t.memo.?.hits >= 10); // every long segment of the second round, at least
-    }
-}
-
 test "sentencepiece-style BPE with byte fallback" {
     const json =
         \\{"added_tokens":[],"normalizer":{"type":"Replace","pattern":{"String":" "},"content":"▁"},
@@ -681,3 +568,7 @@ test {
 test {
     _ = @import("identity.zig");
 }
+
+test {
+    _ = @import("tokenizer_test.zig");
+}
diff --git a/zig/src/core/tokenizer/tokenizer_test.zig b/zig/src/core/tokenizer/tokenizer_test.zig
new file mode 100644
index 0000000000..8834e2b161
--- /dev/null
+++ b/zig/src/core/tokenizer/tokenizer_test.zig
@@ -0,0 +1,120 @@
+//! The tokenizer's cut encode and memo tests.
+
+const std = @import("std");
+const tokenizer = @import("tokenizer.zig");
+const Allocator = std.mem.Allocator;
+const Tokenizer = tokenizer.Tokenizer;
+const Memo = tokenizer.Memo;
+const parse = tokenizer.parse;
+
+test "a cut encode: the text's own added tokens are tokens, the restored text between them is plain text" {
+    const json =
+        \\{"added_tokens":[{"id":10,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
+        \\  {"id":11,"content":"<t>","single_word":false,"lstrip":false,"rstrip":false,"normalized":true,"special":false}],
+        \\ "normalizer":{"type":"Sequence","normalizers":[]},
+        \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
+        \\ "decoder":{"type":"ByteLevel"},
+        \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
+        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"<":7,"t":8,">":9},"merges":[["Ġ","a"],"a b","Ġa b"]}}
+    ;
+    const a = std.testing.allocator;
+    var t = try parse(a, json);
+    defer t.deinit();
+    const Marks = struct {
+        fn spelled(_: *const anyopaque, al: Allocator, text: []const u8) Allocator.Error![]const u8 {
+            return std.mem.replaceOwned(u8, al, text, "@0@", "<t>");
+        }
+    };
+    const restore: Tokenizer.Restore = .{ .ctx = &t, .spelled = Marks.spelled };
+    const ids = try t.encodeCut(a, "ab<|end|>a@0@b<t>c", restore);
+    defer a.free(ids);
+    // added tokens take ids past the model's 10 (tokenizers' rule): "<|end|>" 10, "<t>" 11; the marker's "<t>" is plain
+    try std.testing.expectEqualSlices(u32, &.{ 3, 10, 0, 7, 8, 9, 1, 11, 6 }, ids);
+    // no marker and no added token: the same ids as encode, through the memo too
+    t.memo = try Memo.create(a, 1 << 20);
+    const plain = try t.encode(a, "ab abc");
+    defer a.free(plain);
+    const cut = try t.encodeCut(a, "ab abc", restore);
+    defer a.free(cut);
+    try std.testing.expectEqualSlices(u32, plain, cut);
+}
+
+test "with the memo, long segments encode as without it, and the second encode takes their kept ids" {
+    const json =
+        \\{"added_tokens":[{"id":7,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],
+        \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
+        \\ "decoder":{"type":"ByteLevel"},
+        \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
+        \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6},"merges":[["Ġ","a"],"a b","Ġa b"]}}
+    ;
+    const a = std.testing.allocator;
+    var t = try parse(a, json);
+    defer t.deinit();
+    var text: std.ArrayList(u8) = .empty;
+    defer text.deinit(a);
+    for (0..300) |_| try text.appendSlice(a, "ab abc ");
+    try text.appendSlice(a, "<|end|>");
+    for (0..300) |_| try text.appendSlice(a, " ab cab");
+    const plain = try t.encode(a, text.items);
+    defer a.free(plain);
+    t.memo = try Memo.create(a, 1 << 20);
+    for (0..2) |_| {
+        const kept = try t.encode(a, text.items);
+        defer a.free(kept);
+        try std.testing.expectEqualSlices(u32, plain, kept);
+    }
+    try std.testing.expectEqual(@as(u64, 2), t.memo.?.hits); // both segments, the second time
+}
+
+test "a kept segment's ids equal a fresh encode's wherever the segment sits" {
+    const added =
+        \\{"added_tokens":[{"id":20,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},
+        \\ {"id":21,"content":"<|l|>","single_word":false,"lstrip":true,"rstrip":false,"normalized":false,"special":true},
+        \\ {"id":22,"content":"<|r|>","single_word":false,"lstrip":false,"rstrip":true,"normalized":false,"special":true}],
+    ;
+    const kinds = [_][]const u8{
+        added ++
+            \\ "pre_tokenizer":{"type":"Sequence","pretokenizers":[{"type":"Split","pattern":{"Regex":" ?\\S+|\\s+"},"behavior":"Isolated","invert":false},{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":false,"use_regex":false}]},
+            \\ "model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":"","end_of_word_suffix":"","fuse_unk":false,"byte_fallback":false,"ignore_merges":false,
+            \\  "vocab":{"a":0,"b":1,"Ġ":2,"ab":3,"Ġa":4,"Ġab":5,"c":6,"x":7,"ï":8,"¿":9,"½":10},"merges":[["Ġ","a"],"a b","Ġa b"]}}
+        ,
+        added ++
+            \\ "pre_tokenizer":{"type":"Metaspace","replacement":"▁","prepend_scheme":"first","split":true},
+            \\ "model":{"type":"BPE","unk_token":"<unk>","fuse_unk":true,"byte_fallback":true,
+            \\  "vocab":{"<unk>":0,"a":1,"b":2,"▁":3,"ab":4,"▁a":5,"▁ab":6,"c":7,"x":8,"<0xEF>":9,"<0xBF>":10,"<0xBD>":11},"merges":["a b","▁ a","▁a b"]}}
+        ,
+    };
+    const a = std.testing.allocator;
+    var body: std.ArrayList(u8) = .empty;
+    defer body.deinit(a);
+    for (0..200) |i| try body.appendSlice(a, if (i % 3 == 0) "ab abc " else "cab ab ");
+    const b = body.items;
+    // first or not, after lstrip and rstrip tokens, with a leading space, around a lossy byte, repeated in one prompt
+    const texts = [_][]const []const u8{
+        &.{b},               &.{ "<|end|>", b }, &.{ b, "<|end|>", b }, &.{ "x  <|l|>", b },
+        &.{ "<|r|>   ", b }, &.{ " ", b },       &.{ b, "\xff", b },    &.{ "<|end|> ", b, "<|r|>", b },
+    };
+    for (kinds, 0..) |json, kind| {
+        var t = try parse(a, json);
+        defer t.deinit();
+        var fresh: [texts.len][]u32 = undefined;
+        for (texts, &fresh) |parts, *ids| {
+            const s = try std.mem.concat(a, u8, parts);
+            defer a.free(s);
+            ids.* = try t.encode(a, s);
+        }
+        defer for (fresh) |ids| a.free(ids);
+        // the Metaspace tokenizer prepends to the first segment alone, so a memo blind to `first` fails below
+        if (kind == 1) try std.testing.expect(!std.mem.eql(u32, fresh[0], fresh[1][1..]));
+        t.memo = try Memo.create(a, 1 << 20);
+        for (0..2) |round| for (0..texts.len) |k| {
+            const i = if (round == 0) k else texts.len - 1 - k;
+            const s = try std.mem.concat(a, u8, texts[i]);
+            defer a.free(s);
+            const kept = try t.encode(a, s);
+            defer a.free(kept);
+            try std.testing.expectEqualSlices(u32, fresh[i], kept);
+        };
+        try std.testing.expect(t.memo.?.hits >= 10); // every long segment of the second round, at least
+    }
+}
diff --git a/zig/src/families/dsv41/text.zig b/zig/src/families/dsv41/text.zig
index 2217fc8223..1012e7e38a 100644
--- a/zig/src/families/dsv41/text.zig
+++ b/zig/src/families/dsv41/text.zig
@@ -6,8 +6,7 @@
 //! without one hears the template's default ("high": 75), the number the lane sends; "none" turns thinking off.
 //! Reasoning split ("</think>" closes the block the generation prompt opens) and DSML tool calls are the server's own.
 
-/// The grounding tokens. DeepSeek's encoding keeps them as tokens in an assistant's content and reasoning (the model
-/// wrote them there); every other added-token spelling in request text is plain text (app.py GROUNDING).
+/// Grounding tokens (app.py GROUNDING) stay tokens in an assistant's content and reasoning; other spellings are text.
 pub const grounding: []const []const u8 = &.{ "<｜ref｜>", "<｜/ref｜>", "<｜box｜>", "<｜/box｜>", "<｜point｜>", "<｜/point｜>", "<｜polygon｜>", "<｜/polygon｜>" };
 
 /// A name and its number, as engine_api.Effort takes them.
diff --git a/zig/src/server/model_text.zig b/zig/src/server/model_text.zig
index 32d17ae085..683e3d09d4 100644
--- a/zig/src/server/model_text.zig
+++ b/zig/src/server/model_text.zig
@@ -44,7 +44,7 @@ pub const Text = struct {
         token_bytes: ?*const fn (ctx: *anyopaque, a: Allocator, id: u32) Allocator.Error!?[]u8 = null,
         /// Every added token's text (plain_text.zig shields these in request text); null: none are known.
         added_spellings: ?*const fn (ctx: *anyopaque, a: Allocator) Allocator.Error![]const []const u8 = null,
-        /// Ids of a shielded prompt: each added token spelled exactly is its token, the text between restored and plain.
+        /// Ids of a shielded prompt: exact added-token spellings are tokens; the restored text between is plain.
         encode_cut: ?*const fn (ctx: *anyopaque, a: Allocator, text: []const u8, restore: Restore) Error![]u32 = null,
     };
 
diff --git a/zig/src/server/plain_text.zig b/zig/src/server/plain_text.zig
index c87cc9b4e9..e32da10d3b 100644
--- a/zig/src/server/plain_text.zig
+++ b/zig/src/server/plain_text.zig
@@ -1,9 +1,4 @@
-//! Request text stays plain text (Python's server/plain_text.py): an added-token spelling that a client writes in a
-//! message, a tool result or a tool definition is ordinary text, not that token. The chat template makes one string and
-//! the tokenizer would turn each spelling in it into its token, so a client that writes `</think>` or `<｜User｜>` could
-//! forge a thinking end or a turn. `shield` puts a marker in place of each spelling in the request's strings before
-//! the template runs; `encode` then makes the spellings that remain (the template's own) tokens, and encodes the text
-//! between them, with the markers spelled again, with no added token matched.
+//! Request text stays plain text (server/plain_text.py): a client's `</think>` or `<｜User｜>` cannot forge a token.
 const std = @import("std");
 const json = @import("json");
 const model_text = @import("model_text.zig");
@@ -105,8 +100,7 @@ pub const PlainText = struct {
         return .{ .spellings = try Spellings.init(a, words), .assistant_keep = assistant_keep };
     }
 
-    /// `messages` and `tools` with each added-token spelling as a marker; the same values when none is there.
-    /// `content_keep`: spellings that the server itself wrote into this request's message content.
+    /// `messages` and `tools` with each spelling as a marker; `content_keep`: spellings the server wrote in content.
     pub fn shield(p: *const PlainText, a: Allocator, messages: Value, tools: []const Value, content_keep: []const []const u8) Allocator.Error!Shielded {
         var attempt: u64 = 0;
         while (true) : (attempt += 1) {
@@ -122,9 +116,9 @@ pub const PlainText = struct {
     }
 
     /// The ids of rendered `text`: the template's spellings as tokens, the rest (markers spelled again) without them.
-    /// DeepSeek's normalizer changes no text, so a prompt without a marker is the tokenizer's own encode.
     pub fn encode(p: *const PlainText, text: model_text.Text, a: Allocator, rendered: []const u8, s: *const Shielded) model_text.Error![]u32 {
         _ = p;
+        // DeepSeek's normalizer changes no text, so a prompt without a marker is the tokenizer's own encode
         if (s.table.len == 0 or std.mem.indexOf(u8, rendered, s.prefix) == null) return text.encode(a, rendered, false);
         return text.encodeCut(a, rendered, .{ .ctx = s, .spelled = Shielded.spelledFn }) orelse text.encode(a, rendered, false);
     }
@@ -184,8 +178,7 @@ const Walk = struct {
         };
     }
 
-    /// Python's _messages: each message's text parts joined first (a spelling can span two parts), an assistant's
-    /// content and reasoning keeping the assistant spellings, the server's content spellings kept in any content.
+    /// Python's _messages: text parts joined first (a spelling can span two); assistant and server spellings kept.
     fn messages(w: *Walk, v: Value, content_keep: []const []const u8) Allocator.Error!Value {
         if (v != .array) return w.value(v, &.{});
         const out = try w.a.alloc(Value, v.array.len);
diff --git a/zig/src/server/prompt.zig b/zig/src/server/prompt.zig
index 35069d08c0..0cdbeaaa48 100644
--- a/zig/src/server/prompt.zig
+++ b/zig/src/server/prompt.zig
@@ -62,8 +62,7 @@ fn renderKept(srv: *Server, cx: *Cx, messages: Value, tools: []const Value, thin
     return ids;
 }
 
-/// The template's ids for `messages`; with image input, a marker the text spells is ordinary text (images.zig), and
-/// with Info.plain_message_text every added-token spelling the request writes is (plain_text.zig).
+/// The template's ids for `messages`; image markers (images.zig) and plain_message_text spellings are plain text.
 fn render(srv: *Server, a: std.mem.Allocator, messages: Value, options: model_text.RenderOptions, kept: []const []const u8, problem: *[]const u8) model_text.Error![]u32 {
     if (srv.plain) |*p| {
         const tools: []const Value = if (options.tools) |t| t.array else &.{};