From e152df75b3dff01ba6e88e16785871e0238945eb Mon Sep 17 00:00:00 2001 From: Scott Woolsey Biggart <297695+swbiggart@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:08:23 -0700 Subject: [PATCH] fix(dsv41): enforce native required and named tool prefixes --- ATTRIBUTION.md | 16 + docs/receipts/dsv41-forced-tools/README.md | 112 +++++++ .../dsv41-forced-tools/benchmark.json | 277 ++++++++++++++++++ zig/src/core/engine_api.zig | 12 +- zig/src/core/lane_host.zig | 1 + zig/src/core/lanes/call_gate.zig | 162 ++++++++++ zig/src/core/lanes/call_gate_test.zig | 198 +++++++++++++ zig/src/core/lanes/engine.zig | 10 + zig/src/core/lanes/engine_test.zig | 67 ++++- zig/src/core/lanes/lanes.zig | 1 + zig/src/core/lanes/open.zig | 1 + zig/src/core/lanes/stream.zig | 5 + zig/src/native/cuda.zig | 1 + zig/src/server/call_choice_test.zig | 109 +++++++ zig/src/server/root.zig | 1 + zig/src/server/server.zig | 44 ++- 16 files changed, 1002 insertions(+), 15 deletions(-) create mode 100644 docs/receipts/dsv41-forced-tools/README.md create mode 100644 docs/receipts/dsv41-forced-tools/benchmark.json create mode 100644 zig/src/core/lanes/call_gate.zig create mode 100644 zig/src/core/lanes/call_gate_test.zig create mode 100644 zig/src/server/call_choice_test.zig diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 35dcdf010a..f3b6b7a429 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -53,3 +53,19 @@ ourselves are not listed. - "DeepSeek", "DeepSeek-V4.1-Flash" and "DeepSeek-V4.1-Flash-1017" belong to DeepSeek. "DGX Spark" and "GB10" belong to NVIDIA. We use the names only to say what this branch runs and on what hardware. - This branch is not affiliated with or endorsed by DeepSeek, NVIDIA, the TensorFold authors, Mia-AiLab or MiaAI-Lab. + +## Native DeepSeek required-tool prefix gate + +`zig/src/core/lanes/call_gate.zig` and its regression fixtures adapt the Python +call gate and DSML tests from Elevatormusic/TensorFold PR #1, commit +`003dd66273313d7d69d5675097e873e3af39ffd0`, under Apache-2.0: +https://github.com/Elevatormusic/TensorFold/pull/1 + +The Zig port adds request-local hooks, native lane integration, and explicit +errors for invalid correction state. The rendered-template probe recognizes the +separate `<` text head and `|DSML|` token. Existing TensorFold copyright and +license notices remain applicable. This is prefix/name enforcement, not JSON +argument-schema enforcement. + +Jayleaton's native DeepSeek grammar implementation was consulted as an +architectural reference; no xgrammar or backend code was copied from that fork. diff --git a/docs/receipts/dsv41-forced-tools/README.md b/docs/receipts/dsv41-forced-tools/README.md new file mode 100644 index 0000000000..be6c4d6041 --- /dev/null +++ b/docs/receipts/dsv41-forced-tools/README.md @@ -0,0 +1,112 @@ +# Native DeepSeek required and named tool calls + +Base: `bertholomus/TensorFold:deepseek-v41-zig-tp4` at +`b8b115e7ed6c86d81b7575caa6f3d4ade52b323c`. +Tested source: `253c38ca4a30fca8e0fe242d5599844e84cc187c`, Zig tree +`12ad2fa7f1d21e85fa47129c2a5d97b8dee315a0`. The receipt amendment changes no code. + +DeepSeek's tool-call opener spans a text head, a DSML token and a text suffix. +This port recognizes that split opener and enables a request-local call gate +for native CUDA `dsv41`. It constrains the first tool-call opener and an offered +function name across serial, speculative and concurrent generation. Named +choices use the narrowed tool list; thinking stays unconstrained until it closes. +`auto` and `none` do not install a gate. Upstream grammar handling is preserved. + +The gate checks tokens before forwarding or drafting, cuts speculative output +at the first invalid token, and observes only committed tokens. Quoted names +remain constrained until the closing quote. Leading whitespace before the +opener is permitted; trailing whitespace inside its text head is rejected. +It does not enforce argument JSON/schema conformance, later calls, or completion +when a token limit, stop condition or cancellation ends generation. + +## Validation (2026-10-21 UTC) + +Repository-pinned Zig 0.17.0, macOS ARM64 host checks: + +| Command | Result | +| --- | --- | +| `zig test -lc zig/src/core/lanes/lanes.zig` | 132 passed | +| `ZIG=zig bash zig/tests/server/zig_test.sh` | 375 server, five JSON and 170 engine API tests passed | +| `zig build native -Dtarget=aarch64-linux-gnu -Dcpu=generic -j4 --summary all` | All seven steps passed | +| `zig build test test-golden -Dcpu=apple_m1 -j4 --summary all` | 208/212 steps succeeded; 2,291 tests passed, three skipped; overall failure at inherited macOS `SO.BUSY_POLL` in `dsv41/link.zig:23` | +| Golden corpus from that run | 309/319 equal, ten known differences, zero unexpected | +| `python3 tools/zig/lean_check.py` | 1,615 inherited findings, unchanged from base | +| `git diff --check` | Passed | + +Coverage includes required/named preflight, auto/none, unsupported-backend refusal, +split and merged tokens, quoted-name delimiters, opener whitespace, thinking +boundaries, request isolation, committed correction history and output limits. +Astra reviewed the implementation and independently passed all seven focused +gate tests. Host compilation is separate from the GPU +checks below; the full host suite is not represented as a clean pass. + +## Matched GPU benchmark (2026-10-21 UTC) + +Four NVIDIA GB10 hosts running Ubuntu 24.04, native Zig CUDA TP4 DSpark. +EXL3 checkpoint `bertholomus/DeepSeek-V4.1-Flash-1017-EXL3-4.0bpw` at +`a2f91e2794923d0beebebe0d267fb3468eaf0b65`, Engram at +`1bae0e1088993b9b82de08f59de5be5a34e42170`, patched NCCL 2.30.7. +Both profiles use Zig 0.17.0, identical CUDA kernel assets and +`-Dtarget=aarch64-linux-gnu.2.28 -Doptimize=fast -Dsm=121`. +The same external build adapter exposes the existing production lane executable; +production transport source does not differ between profiles. + +Effective runtime: `--context 1048576 --parallel 16 --max-tokens 32768`, +window 2097152, round rows48, served depth solo, neighbor RDMA enabled, GID3. +Measurements use the head-loopback API as `deepseek-v4.1-flash`. +[Benchmark data](benchmark.json) retains source/binary hashes, per-load timings, +OS/kernel details, request settings and the benchmark client hash. + +Order: upstream, patched, upstream, patched; fresh load and identical warmup +each time. Four streams and 256 output tokens: set-a one warmup and six measured +bursts, set-a 30-second steady admission, set-b 90-second sustained admission. +Ordinary requests do not install a call gate. Results average two loads per build. + +| Workload | Upstream tok/s | Patched tok/s | Change | +| --- | ---: | ---: | ---: | +| Four-stream burst | 132.40 | 132.81 | +0.311% | +| 30-second steady | 147.97 | 147.73 | -0.162% | +| 90-second sustained | 169.84 | 169.59 | -0.150% | + +No meaningful performance reduction was observed in this benchmark. All 96 +measured burst replies match UTF-8 across builds; all 36 qualification replies +match token IDs and text across builds and serial/drafted/concurrent paths. +All workload stages report zero worker errors. Fourteen serial and four +concurrent tool-choice feature cases pass on the patched build. + +The supplementary cooperative-tool workload compares auto/required/named on the +patched build: four streams, alternating choice order, one warmup and six timed +bursts per choice. All 84 calls return the expected function and arguments; +output token IDs match across choices. Full-response aggregate throughput: + +| Choice | tok/s | Change from auto | +| --- | ---: | ---: | +| auto | 201.57 | +0.000% | +| required | 200.91 | -0.327% | +| named | 201.29 | -0.138% | + +Required/named throughput is within 0.33% of auto. These samples are not a +statistical upper bound on overhead. A forced choice that changes the generated +answer can change latency. The active-call workload is separate from the upstream +benchmark; these checks do not establish argument-schema enforcement or +long-context quality. + +Public prompts and client: [pinned `kit_bench.py`](https://github.com/bertholomus/deepseek-v4.1-tensorfold-tp4-4xgb10/blob/9af1f4567ee49ec2a31f68975982093277e7090d/tools/kit_bench.py). +Its SHA-256 matches `workloads.client_sha256` in the data. Equivalent invocations +(the measurement wrapper additionally retained raw responses): + +```sh +python3 kit_bench.py --base http://127.0.0.1:40053 --model deepseek-v4.1-flash concurrent --set a --streams 4 --tokens 256 --reps 1 +python3 kit_bench.py --base http://127.0.0.1:40053 --model deepseek-v4.1-flash concurrent --set a --streams 4 --tokens 256 --reps 6 +python3 kit_bench.py --base http://127.0.0.1:40053 --model deepseek-v4.1-flash sustained --set a --streams 4 --tokens 256 --seconds 30 +python3 kit_bench.py --base http://127.0.0.1:40053 --model deepseek-v4.1-flash sustained --set b --streams 4 --tokens 256 --seconds 90 +``` + +## Provenance + +The gate and initial fixtures were ported from an earlier native adaptation of +[Elevatormusic/TensorFold PR #1](https://github.com/Elevatormusic/TensorFold/pull/1), +commit `003dd66273313d7d69d5675097e873e3af39ffd0`, under Apache-2.0. +See [ATTRIBUTION.md](../../../ATTRIBUTION.md). The scheduler integration was +adapted to this branch's `Engine.addStream` and existing round paths. This +change contains no transport optimization or SGLang code. diff --git a/docs/receipts/dsv41-forced-tools/benchmark.json b/docs/receipts/dsv41-forced-tools/benchmark.json new file mode 100644 index 0000000000..b2f5232d71 --- /dev/null +++ b/docs/receipts/dsv41-forced-tools/benchmark.json @@ -0,0 +1,277 @@ +{ + "profiles": { + "0-base": { + "burst_tps": 132.6409170849111, + "burst_repetitions": [ + 133.65128802716237, + 132.91054583763008, + 132.59406779758848, + 132.48123450831213, + 132.1282995703259, + 132.0800667684477 + ], + "steady_tps": 148.3, + "sustained_tps": 169.88, + "steady": { + "streams": 4, + "seconds": 30, + "max_tokens": 256, + "set": "a", + "requests": 19, + "aggregate_tps": 148.3, + "per_stream_tps_p50": 47.5, + "per_stream_tps_p95": 61.543, + "per_stream_tps_p05": 22.921, + "ttft_s_p50": 0.206, + "ttft_s_p95": 0.429 + }, + "sustained": { + "streams": 4, + "seconds": 90, + "max_tokens": 256, + "set": "b", + "requests": 62, + "aggregate_tps": 169.88, + "per_stream_tps_p50": 51.641, + "per_stream_tps_p95": 83.021, + "per_stream_tps_p05": 25.997, + "ttft_s_p50": 0.156, + "ttft_s_p95": 0.273 + } + }, + "1-candidate": { + "burst_tps": 132.9428278370476, + "burst_repetitions": [ + 132.36725036958143, + 133.6823153140167, + 132.03482926114978, + 133.5584290061675, + 133.48077877508268, + 132.5333642962874 + ], + "steady_tps": 148.08, + "sustained_tps": 169.82, + "steady": { + "streams": 4, + "seconds": 30, + "max_tokens": 256, + "set": "a", + "requests": 19, + "aggregate_tps": 148.08, + "per_stream_tps_p50": 47.938, + "per_stream_tps_p95": 61.306, + "per_stream_tps_p05": 22.981, + "ttft_s_p50": 0.199, + "ttft_s_p95": 0.49 + }, + "sustained": { + "streams": 4, + "seconds": 90, + "max_tokens": 256, + "set": "b", + "requests": 62, + "aggregate_tps": 169.82, + "per_stream_tps_p50": 51.509, + "per_stream_tps_p95": 82.905, + "per_stream_tps_p05": 26.065, + "ttft_s_p50": 0.156, + "ttft_s_p95": 0.24 + } + }, + "2-base": { + "burst_tps": 132.15785629131187, + "burst_repetitions": [ + 132.2480853294622, + 132.0944595596113, + 132.3703780960966, + 131.99234697534195, + 132.14958934645915, + 132.09227844090003 + ], + "steady_tps": 147.64, + "sustained_tps": 169.8, + "steady": { + "streams": 4, + "seconds": 30, + "max_tokens": 256, + "set": "a", + "requests": 19, + "aggregate_tps": 147.64, + "per_stream_tps_p50": 47.369, + "per_stream_tps_p95": 61.097, + "per_stream_tps_p05": 22.875, + "ttft_s_p50": 0.192, + "ttft_s_p95": 0.483 + }, + "sustained": { + "streams": 4, + "seconds": 90, + "max_tokens": 256, + "set": "b", + "requests": 62, + "aggregate_tps": 169.8, + "per_stream_tps_p50": 50.894, + "per_stream_tps_p95": 82.959, + "per_stream_tps_p05": 26.059, + "ttft_s_p50": 0.156, + "ttft_s_p95": 0.21 + } + }, + "3-candidate": { + "burst_tps": 132.67876691467984, + "burst_repetitions": [ + 131.90364845944518, + 133.04801901148596, + 132.23347262613427, + 133.3805526886387, + 133.53948264666187, + 131.96742605571302 + ], + "steady_tps": 147.38, + "sustained_tps": 169.35, + "steady": { + "streams": 4, + "seconds": 30, + "max_tokens": 256, + "set": "a", + "requests": 19, + "aggregate_tps": 147.38, + "per_stream_tps_p50": 46.852, + "per_stream_tps_p95": 61.137, + "per_stream_tps_p05": 22.581, + "ttft_s_p50": 0.196, + "ttft_s_p95": 0.432 + }, + "sustained": { + "streams": 4, + "seconds": 90, + "max_tokens": 256, + "set": "b", + "requests": 60, + "aggregate_tps": 169.35, + "per_stream_tps_p50": 50.628, + "per_stream_tps_p95": 82.754, + "per_stream_tps_p05": 26.093, + "ttft_s_p50": 0.157, + "ttft_s_p95": 0.243 + } + } + }, + "pooled": { + "burst_tps": { + "base": 132.3993866881115, + "forced_tool_patch": 132.81079737586373, + "change_percent": 0.3107345872540801 + }, + "steady_tps": { + "base": 147.97, + "forced_tool_patch": 147.73000000000002, + "change_percent": -0.16219503953502912 + }, + "sustained_tps": { + "base": 169.84, + "forced_tool_patch": 169.58499999999998, + "change_percent": -0.15014130946774396 + } + }, + "cross_build_exact_utf8_replies": 96, + "cross_build_exact_token_replies": 36, + "scope": "ordinary requests: forced gate unused; identical source base and runtime; two loads per profile; practical measurement, not a statistical bound", + "active_tools": { + "pass": true, + "exact_tokens_across_choices": true, + "mean_tps": { + "auto": 201.56557242418165, + "required": 200.90695584002935, + "named": 201.28669127920435 + }, + "calls": 84, + "streams": 4, + "warmups_per_choice": 1, + "measured_bursts_per_choice": 6, + "temperature": 0, + "max_tokens": 256, + "thinking": false, + "description": "Call save_numbers with label batch0 through batch3 and every integer from 1 through 30 in order; validate complete calls and compare tokens; alternate auto/required/named order each repetition." + }, + "source": { + "base": "b8b115e7ed6c86d81b7575caa6f3d4ade52b323c", + "candidate": "253c38ca4a30fca8e0fe242d5599844e84cc187c", + "production_patch_sha256": "56bafbd0c5038961fba165d55fd99b03657cd4146619e85d262221e07c980807", + "external_build_adapter_sha256": "de740aa4db2ac6548adf35351bbc13da222774a8c2eab87a61dcd4824456d928", + "build_adapter_scope": "adds executable target for unchanged production dsv41 lanes; same adapter in both source archives", + "production_family_transport_changes_between_profiles": false, + "tested_zig_tree": "12ad2fa7f1d21e85fa47129c2a5d97b8dee315a0" + }, + "binaries": { + "base": { + "tensorfold-native": "deb87918014aabad4cc002fb680ae4236cb60c53c73f29b841f008064f4a155a", + "tf-dsv41-lanes": "3e211ab74be8470742347b8af247efe04aefc277bef14ccc60a6b6e1ab4e9ba9" + }, + "candidate": { + "tensorfold-native": "ed479ca40f0f1744e55ef9b790a0df46b4dad6356fcb3969d95f823c7cfce840", + "tf-dsv41-lanes": "2f135afdbb2fc2e68a9c12696935ad6d885a974056a7afd2a55c94e22236b0f8" + } + }, + "runtime": { + "gpu": "4 x NVIDIA GB10", + "backend": "native Zig CUDA TP4 DSpark", + "compiler": "Zig 0.17.0", + "target": "aarch64-linux-gnu.2.28", + "optimize": "fast", + "sm": 121, + "checkpoint": "bertholomus/DeepSeek-V4.1-Flash-1017-EXL3-4.0bpw", + "checkpoint_revision": "a2f91e2794923d0beebebe0d267fb3468eaf0b65", + "engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", + "context": 1048576, + "parallel": 16, + "max_tokens": 32768, + "window": 2097152, + "round_rows": 48, + "served_depth": "solo", + "neighbor_rdma": true, + "gid_index": 3, + "nccl": "2.30.7 patched", + "nccl_sha256": "1cdf5f3e2d617d5c07a87903eb8849d5bd7758f6c900455d613a2e0eb289d63e", + "os": "Ubuntu 24.04 (rank 1: 24.04.3 LTS; ranks 0, 2, 3: 24.04.4 LTS)", + "kernels_by_rank": [ + "6.17.0-1029-nvidia", + "6.17.0-1032-nvidia", + "6.17.0-1022-nvidia", + "6.17.0-1022-nvidia" + ] + }, + "workloads": { + "order": [ + "base", + "candidate", + "base", + "candidate" + ], + "burst": { + "set": "a", + "streams": 4, + "tokens": 256, + "warmups": 1, + "reps": 6 + }, + "steady": { + "set": "a", + "streams": 4, + "tokens": 256, + "seconds": 30 + }, + "sustained": { + "set": "b", + "streams": 4, + "tokens": 256, + "seconds": 90 + }, + "client_sha256": "f5f24b1e105e6d073e26b780983be3f4654219b986290e1f6a106c1955232bc5", + "client_source": "https://github.com/bertholomus/deepseek-v4.1-tensorfold-tp4-4xgb10/blob/9af1f4567ee49ec2a31f68975982093277e7090d/tools/kit_bench.py" + }, + "feature_cases": { + "serial_pass": 14, + "concurrent_pass": 4 + } +} diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index 1e98911bfd..d8e8ec5f0a 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -35,16 +35,8 @@ pub const LaneImage = lanes.stream.LaneImage; pub const LaneVision = lanes.stream.LaneVision; /// tool_choice "required" or a named function: outside a think block the answer opens a call to an offered tool. -pub const CallGate = struct { - /// The token that opens a call, the template text before the name, and the mark that ends the name. - opener: u32, - lead: []const u8 = "", - tail: []const u8 = "", - /// Offered names (empty: any); the gate completes a name its prefix starts. - names: []const []const u8 = &.{}, - think_open: ?u32 = null, - think_end: ?u32 = null, -}; +pub const CallGate = lanes.stream.CallGate.Spec; +pub const CallFilter = lanes.stream.CallGate; /// response_format and the guided_* fields: every token keeps the reply inside this grammar. pub const Structure = struct { diff --git a/zig/src/core/lane_host.zig b/zig/src/core/lane_host.zig index ccee37a8e2..885be2f173 100644 --- a/zig/src/core/lane_host.zig +++ b/zig/src/core/lane_host.zig @@ -356,6 +356,7 @@ pub const LaneHost = struct { .prompt_logprobs = r.prompt_logprobs, .loop_guard = r.loop_guard, .loop_windows = r.loop_windows, + .call_gate = if (r.call) |c| c.filter else null, .chunks = r.chunks, .history_len = r.history_len, .shared_prefixes = r.shared_prefixes, diff --git a/zig/src/core/lanes/call_gate.zig b/zig/src/core/lanes/call_gate.zig new file mode 100644 index 0000000000..211b30081a --- /dev/null +++ b/zig/src/core/lanes/call_gate.zig @@ -0,0 +1,162 @@ +//! Required tool-call prefixes, ported from Elevatormusic/TensorFold's call_gate.py (Apache-2.0; see ATTRIBUTION.md). +const std = @import("std"); +const A = std.mem.Allocator; + +pub const Cut = struct { at: usize, fix: []const u32 }; +pub const Hook = struct { + ptr: *anyopaque, + cut: *const fn (*anyopaque, A, []const u32) anyerror!?Cut, + observe: *const fn (*anyopaque, u32) anyerror!void, + + pub fn feed(h: Hook, a: A, gpa: A, force: *std.ArrayList(u32), backend: anytype, input: anytype) !@TypeOf(input) { + const token = switch (input) { + .handle => |handle| try backend.read(handle), + .value => |value| value, + }; + return .{ .value = try h.enforce(a, gpa, force, token) }; + } + + pub fn enforce(h: Hook, a: A, gpa: A, force: *std.ArrayList(u32), token: u32) !u32 { + if (try h.cut(h.ptr, a, &.{token})) |hit| { + force.clearRetainingCapacity(); + try force.appendSlice(gpa, hit.fix[1..]); + return hit.fix[0]; + } + return token; + } +}; +pub const Spec = struct { + opener: u32, + head: []const u8 = "", + lead: []const u8 = "", + tail: []const u8 = "", + names: []const []const u8 = &.{}, + think_open: ?u32 = null, + think_end: ?u32 = null, + filter: ?Hook = null, +}; +pub const Codec = struct { + ptr: *anyopaque, + encode: *const fn (*anyopaque, A, []const u8) anyerror![]const u32, + text: *const fn (*anyopaque, A, u32) anyerror![]const u8, +}; +const Phase = enum { answer, open, lead, name, done }; +const State = struct { armed: bool = true, phase: Phase = .answer, seen: []const u8 = "" }; +const Step = struct { state: State, fix: ?[]const u32 = null }; + +pub const Gate = struct { + a: A, + spec: Spec, + codec: Codec, + state: State = .{}, + + pub fn init(a: A, spec: Spec, codec: Codec, prompt: []const u32) Gate { + var g: Gate = .{ .a = a, .spec = spec, .codec = codec }; + for (prompt) |t| { + if (spec.think_open != null and t == spec.think_open.?) g.state.armed = false; + if (spec.think_end != null and t == spec.think_end.?) g.state.armed = true; + } + return g; + } + + pub fn hook(g: *Gate) Hook { + return .{ .ptr = g, .cut = cutFn, .observe = observeFn }; + } + + fn encode(g: *const Gate, a: A, text: []const u8) ![]const u32 { + return g.codec.encode(g.codec.ptr, a, text); + } + + fn opening(g: *const Gate, a: A, head: bool) ![]const u32 { + const h = if (head) try g.encode(a, g.spec.head) else &.{}; + const lead = if (g.spec.names.len > 0) try g.encode(a, g.spec.lead) else &.{}; + return std.mem.concat(a, u32, &.{ h, &.{g.spec.opener}, lead }); + } + + fn ends(g: *const Gate, c: u8) bool { + if (g.spec.tail.len == 0 and std.ascii.isWhitespace(c)) return true; + return std.mem.indexOfScalar(u8, if (g.spec.tail.len > 0) g.spec.tail else "<>{}\"'(),", c) != null; + } + + fn step(g: *const Gate, a: A, before: State, token: u32) !Step { + var s = before; + if (s.phase == .done) return .{ .state = s }; + if (s.phase == .answer) { + if (!s.armed) { + s.armed = g.spec.think_end != null and token == g.spec.think_end.?; + return .{ .state = s }; + } + if (g.spec.think_open != null and token == g.spec.think_open.?) { + s.armed = false; + return .{ .state = s }; + } + if (token == g.spec.opener and g.spec.head.len == 0) { + s.phase = .lead; + return .{ .state = s }; + } + const text = try g.codec.text(g.codec.ptr, a, token); + const trimmed = std.mem.trimStart(u8, text, " \t\r\n\x0b\x0c"); + if (trimmed.len == 0) return .{ .state = s }; + if (g.spec.head.len > 0 and std.mem.eql(u8, trimmed, g.spec.head)) { + s.phase = .open; + return .{ .state = s }; + } + return .{ .state = s, .fix = try g.opening(a, true) }; + } + if (s.phase == .open) { + if (token != g.spec.opener) return .{ .state = s, .fix = try g.opening(a, false) }; + s.phase = .lead; + return .{ .state = s }; + } + if (g.spec.names.len == 0) return .{ .state = .{ .phase = .done } }; + const piece = try g.codec.text(g.codec.ptr, a, token); + var written = try std.mem.concat(a, u8, &.{ s.seen, piece }); + var owed: []const u8 = ""; + if (s.phase == .lead) { + if (std.mem.startsWith(u8, g.spec.lead, written)) { + s.phase = if (written.len == g.spec.lead.len) .name else .lead; + s.seen = if (s.phase == .name) "" else written; + return .{ .state = s }; + } + if (!std.mem.startsWith(u8, written, g.spec.lead)) + return .{ .state = s, .fix = try g.encode(a, g.spec.lead[s.seen.len..]) }; + owed = g.spec.lead[s.seen.len..]; + s.seen = ""; + written = written[g.spec.lead.len..]; + } + var end: usize = 0; + while (end < written.len and !g.ends(written[end])) : (end += 1) {} + for (g.spec.names) |name| { + if (end == written.len and std.mem.startsWith(u8, name, written)) + return .{ .state = .{ .armed = s.armed, .phase = .name, .seen = written } }; + if (end < written.len and std.mem.eql(u8, name, written[0..end])) + return .{ .state = .{ .phase = .done } }; + } + for (g.spec.names) |name| if (std.mem.startsWith(u8, name, s.seen)) { + const rest = try std.mem.concat(a, u8, &.{ owed, name[s.seen.len..], g.spec.tail }); + return .{ .state = before, .fix = try g.encode(a, rest) }; + }; + return error.InvalidCallPrefix; + } + + fn cutFn(ptr: *anyopaque, a: A, tokens: []const u32) !?Cut { + const g: *Gate = @ptrCast(@alignCast(ptr)); + var s = g.state; + for (tokens, 0..) |t, i| { + const next = try g.step(a, s, t); + if (next.fix) |fix| { + if (fix.len == 0) return error.EmptyCallFix; + return .{ .at = i, .fix = fix }; + } + s = next.state; + } + return null; + } + + fn observeFn(ptr: *anyopaque, token: u32) !void { + const g: *Gate = @ptrCast(@alignCast(ptr)); + const next = try g.step(g.a, g.state, token); + if (next.fix != null) return error.UnenforcedCallToken; + g.state = next.state; + } +}; diff --git a/zig/src/core/lanes/call_gate_test.zig b/zig/src/core/lanes/call_gate_test.zig new file mode 100644 index 0000000000..f8f3c95a09 --- /dev/null +++ b/zig/src/core/lanes/call_gate_test.zig @@ -0,0 +1,198 @@ +const std = @import("std"); +const cg = @import("call_gate.zig"); +const expect = std.testing.expect; +const eq = std.testing.expectEqualSlices; + +pub const Codec = struct { + pub fn encode(_: *anyopaque, a: std.mem.Allocator, bytes: []const u8) ![]const u32 { + const result = try a.alloc(u32, bytes.len); + for (bytes, result) |c, *t| t.* = c; + return result; + } + pub fn text(_: *anyopaque, a: std.mem.Allocator, token: u32) ![]const u8 { + if (token == 1000) return "|DSML|"; + if (token == 1001 or token == 1002) return "think"; + if (token == 1003) return spec.lead ++ "get_x"; + if (token == 1004) return "get_time\">"; + if (token == 1005) return "get_time unexpected\""; + if (token == 1006) return "< "; + if (token == 1007) return " \t<"; + const result = try a.alloc(u8, 1); + result[0] = @intCast(token); + return result; + } + pub fn codec(ptr: *anyopaque) cg.Codec { + return .{ .ptr = ptr, .encode = encode, .text = text }; + } +}; + +pub const spec: cg.Spec = .{ + .head = "<", + .opener = 1000, + .lead = " calls>\n<|DSML| invoke name=\"", + .tail = "\"", + .names = &.{ "get_weather", "get_time" }, + .think_open = 1001, + .think_end = 1002, +}; + +fn ids(a: std.mem.Allocator, text: []const u8) ![]const u32 { + var dummy: u8 = 0; + return Codec.encode(&dummy, a, text); +} +fn opening(a: std.mem.Allocator) ![]const u32 { + return std.mem.concat(a, u32, &.{ &.{ '<', 1000 }, try ids(a, spec.lead) }); +} +fn observe(g: *cg.Gate, tokens: []const u32) !void { + const hook = g.hook(); + for (tokens) |t| try hook.observe(hook.ptr, t); +} + +test "DSML head and opener are enforced without mutating proposed state" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var g = cg.Gate.init(a, spec, Codec.codec(&dummy), &.{}); + const h = g.hook(); + const bad = try ids(a, "Sure"); + const first = (try h.cut(h.ptr, a, bad)).?; + try expect(first.at == 0); + try eq(u32, try opening(a), first.fix); + const again = (try h.cut(h.ptr, a, bad)).?; + try eq(u32, first.fix, again.fix); + try std.testing.expectError(error.UnenforcedCallToken, h.observe(h.ptr, 'S')); + const whitespace = (try h.cut(h.ptr, a, &.{ ' ', 'S' })).?; + try expect(whitespace.at == 1); + const split = (try h.cut(h.ptr, a, &.{ '<', 'x' })).?; + try expect(split.at == 1); + try eq(u32, (try opening(a))[1..], split.fix); + const bare = (try h.cut(h.ptr, a, &.{1000})).?; + try eq(u32, try opening(a), bare.fix); + try observe(&g, first.fix); + try observe(&g, try ids(a, "get_time\"")); + try expect(try h.cut(h.ptr, a, bad) == null); +} + +test "required and named choices correct only the first divergent name token" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + for ([_][]const []const u8{ &.{ "get_weather", "get_time" }, &.{"get_time"} }) |names| { + var s = spec; + s.names = names; + var g = cg.Gate.init(a, s, Codec.codec(&dummy), &.{}); + try observe(&g, try opening(a)); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, try ids(a, "get_x"))).?; + try expect(hit.at == 4); + try eq(u32, try ids(a, try std.mem.concat(a, u8, &.{ names[0][4..], "\"" })), hit.fix); + try observe(&g, try ids(a, "get_")); + try observe(&g, hit.fix); + try expect(try h.cut(h.ptr, a, try ids(a, ">{arbitrary arguments}")) == null); + } +} + +test "thinking is unconstrained until closed and each request owns its state" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var thinking = cg.Gate.init(a, spec, Codec.codec(&dummy), &.{1001}); + var answer = cg.Gate.init(a, spec, Codec.codec(&dummy), &.{ 1001, 1002 }); + const h = thinking.hook(); + const other = answer.hook(); + const prose = try ids(a, "Sure"); + try expect(try h.cut(h.ptr, a, prose) == null); + try expect(try other.cut(other.ptr, a, prose) != null); + const hit = (try h.cut(h.ptr, a, &.{ 'S', 1002, 'x' })).?; + try expect(hit.at == 2); + try observe(&thinking, &.{ 'S', 1002 }); + try observe(&thinking, hit.fix); + try observe(&thinking, try ids(a, "get_weather\"")); +} + +test "single-token call opener retains its existing behavior" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var g = cg.Gate.init(a, .{ .opener = 1000 }, Codec.codec(&dummy), &.{}); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, &.{'x'})).?; + try eq(u32, &.{1000}, hit.fix); + try observe(&g, hit.fix); + try expect(try h.cut(h.ptr, a, &.{'x'}) == null); +} + +test "a merged token spanning the lead and name corrects the whole uncommitted token" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var g = cg.Gate.init(a, spec, Codec.codec(&dummy), &.{}); + try observe(&g, &.{ '<', 1000 }); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, &.{1003})).?; + try expect(hit.at == 0); + try eq(u32, try ids(a, spec.lead ++ "get_weather\""), hit.fix); + try observe(&g, hit.fix); + try expect(try h.cut(h.ptr, a, &.{'x'}) == null); + + var named = spec; + named.names = &.{"get_time"}; + var other = cg.Gate.init(a, named, Codec.codec(&dummy), &.{}); + try observe(&other, try opening(a)); + try observe(&other, &.{1004}); + const named_hook = other.hook(); + try expect(try named_hook.cut(named_hook.ptr, a, &.{'x'}) == null); +} + +test "quoted names require their closing quote across split and merged tokens" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var named = spec; + named.names = &.{"get_time"}; + for ([_][]const []const u8{ spec.names, named.names }) |names| { + var s = spec; + s.names = names; + for ([_]u8{ ' ', '\t', '\r', '\n' }) |space| { + var g = cg.Gate.init(a, s, Codec.codec(&dummy), &.{}); + try observe(&g, try opening(a)); + try observe(&g, try ids(a, "get_time")); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, &.{space})).?; + try expect(hit.at == 0); + try eq(u32, try ids(a, "\""), hit.fix); + try std.testing.expectError(error.UnenforcedCallToken, h.observe(h.ptr, space)); + try observe(&g, hit.fix); + try expect(try h.cut(h.ptr, a, try ids(a, ">{arguments}")) == null); + } + var g = cg.Gate.init(a, s, Codec.codec(&dummy), &.{}); + try observe(&g, try opening(a)); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, &.{1005})).?; + try expect(hit.at == 0); + try eq(u32, try ids(a, try std.mem.concat(a, u8, &.{ names[0], "\"" })), hit.fix); + try observe(&g, hit.fix); + } +} + +test "DSML head accepts leading whitespace but rejects trailing whitespace" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var g = cg.Gate.init(a, spec, Codec.codec(&dummy), &.{}); + const h = g.hook(); + const hit = (try h.cut(h.ptr, a, &.{ 1006, 1000 })).?; + try expect(hit.at == 0); + try eq(u32, try opening(a), hit.fix); + try std.testing.expectError(error.UnenforcedCallToken, h.observe(h.ptr, 1006)); + try expect(try h.cut(h.ptr, a, &.{ 1007, 1000 }) == null); + try observe(&g, &.{ 1007, 1000 }); + try observe(&g, try ids(a, spec.lead ++ "get_time\"")); +} diff --git a/zig/src/core/lanes/engine.zig b/zig/src/core/lanes/engine.zig index f11d8dbec5..eb6288ed46 100644 --- a/zig/src/core/lanes/engine.zig +++ b/zig/src/core/lanes/engine.zig @@ -393,6 +393,14 @@ pub const Engine = struct { committed.shrinkRetainingCapacity(c); try committed.append(a, try s.startClose(e.gpa)); } + if (s.call_gate) |gate| if (try gate.cut(gate.ptr, a, committed.items)) |hit| { + cut = hit.at; + path = path[0 .. hit.at + 1]; + committed.shrinkRetainingCapacity(hit.at); + s.force.clearRetainingCapacity(); + try s.force.appendSlice(e.gpa, hit.fix[1..]); + try committed.append(a, hit.fix[0]); + }; s.rounds += 1; var committed_rows: []LogRow = &.{}; if (rows_lp.len > 0) { @@ -510,6 +518,7 @@ pub const Engine = struct { var current: Feed = .{ .handle = s.inflight.? }; s.inflight = null; if (try e.forcedNext(s)) |t| current = .{ .value = t }; + if (s.call_gate) |gate| current = try gate.feed(e.arena.allocator(), e.gpa, &s.force, e.backend, current); var token: u32 = undefined; if (mode == .drain) { token = try e.readFeed(current); // the last queued step: no new one @@ -535,6 +544,7 @@ pub const Engine = struct { var current: Feed = .{ .handle = s.inflight.? }; s.inflight = null; if (try e.forcedNext(s)) |t| current = .{ .value = t }; + if (s.call_gate) |gate| current = try gate.feed(e.arena.allocator(), e.gpa, &s.force, e.backend, current); const token = try e.readFeed(current); s.rounds += 1; const landed = try s.commit(e.gpa, &.{token}, &.{}); diff --git a/zig/src/core/lanes/engine_test.zig b/zig/src/core/lanes/engine_test.zig index 3b7749c04c..6c093d5f25 100644 --- a/zig/src/core/lanes/engine_test.zig +++ b/zig/src/core/lanes/engine_test.zig @@ -17,6 +17,7 @@ const Case = struct { sampling: ?Sampling = null, drafts: bool = true, think_budget: u32 = 0, + call_gate: ?sm.CallGate.Hook = null, logprobs: ?u8 = null, classes: ?Classes = null, // structured output under this test grammar eos: []const u32 = &.{96}, @@ -116,7 +117,7 @@ fn run(cases: []const Case) ![][]u32 { g.* = given; k.* = .init(g.rules(), (fake.vocab + 31) / 32, given.after); } - s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = c.prompt, .max_new = c.max_new, .eos = c.eos, .sampling = c.sampling, .drafts = c.drafts, .proposer = p.proposer(), .think_budget = c.think_budget, .think_close = &.{ 90, 91, 92 }, .think_end = 91, .logprobs = c.logprobs, .grammar = if (c.classes != null) k else null }); + s.* = try sm.Stream.init(gpa, .{ .id = "s", .prompt = c.prompt, .max_new = c.max_new, .eos = c.eos, .sampling = c.sampling, .drafts = c.drafts, .proposer = p.proposer(), .think_budget = c.think_budget, .think_close = &.{ 90, 91, 92 }, .think_end = 91, .logprobs = c.logprobs, .grammar = if (c.classes != null) k else null, .call_gate = c.call_gate }); } defer for (streams, proposers) |*s, *p| { s.deinit(gpa); @@ -475,3 +476,67 @@ test "a family that keeps copies to shared rounds proposes none for a lone strea try std.testing.expectEqual(lone_copies, (try win.copyProposal(&engine, &s)).tokens.len >= 2); } } + +test "call gates preserve target history across drafted, serial and shared rounds" { + const fixture = @import("call_gate_test.zig"); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var gates: [4]sm.CallGate.Gate = undefined; + for (&gates) |*gate| gate.* = sm.CallGate.Gate.init(a, fixture.spec, fixture.Codec.codec(&dummy), &.{}); + const serial = try run(&.{.{ .prompt = &p1, .drafts = false, .max_new = 90, .call_gate = gates[0].hook() }}); + defer free(serial); + const drafted = try run(&.{.{ .prompt = &p1, .max_new = 90, .call_gate = gates[1].hook() }}); + defer free(drafted); + const shared = try run(&.{ + .{ .prompt = &p1, .max_new = 90, .call_gate = gates[2].hook() }, + .{ .prompt = &p2, .max_new = 90, .call_gate = gates[3].hook() }, + }); + defer free(shared); + try std.testing.expectEqualSlices(u32, serial[0], drafted[0]); + try std.testing.expectEqualSlices(u32, serial[0], shared[0]); + var solo_gate = sm.CallGate.Gate.init(a, fixture.spec, fixture.Codec.codec(&dummy), &.{}); + const solo = try run(&.{.{ .prompt = &p2, .max_new = 90, .call_gate = solo_gate.hook() }}); + defer free(solo); + try std.testing.expectEqualSlices(u32, solo[0], shared[1]); + const lead = try fixture.Codec.encode(&dummy, a, fixture.spec.lead); + const prefix = try std.mem.concat(a, u32, &.{ &.{ '<', 1000 }, lead }); + try std.testing.expectEqualSlices(u32, prefix, serial[0][0..prefix.len]); + // Once the function name ends, every subsequent sample must use exactly the committed history. + var history: std.ArrayList(u32) = .empty; + defer history.deinit(gpa); + try history.appendSlice(gpa, &p1); + var name_ended = false; + var checked: usize = 0; + for (serial[0], 0..) |t, i| { + if (name_ended) { + try std.testing.expectEqual(fake.next(history.items, null, history.items.len), t); + checked += 1; + } + if (i >= prefix.len and t == '"') name_ended = true; + try history.append(gpa, t); + } + try std.testing.expect(checked > 3); +} + +test "call gate survives thinking budget cuts and respects the output limit" { + const fixture = @import("call_gate_test.zig"); + var arena = std.heap.ArenaAllocator.init(gpa); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + var spec = fixture.spec; + spec.think_open = 90; + spec.think_end = 91; + var gates: [3]sm.CallGate.Gate = undefined; + for (&gates) |*gate| gate.* = sm.CallGate.Gate.init(a, spec, fixture.Codec.codec(&dummy), &.{90}); + const drafted = try run(&.{.{ .prompt = &p2, .max_new = 90, .think_budget = 3, .call_gate = gates[0].hook() }}); + defer free(drafted); + const serial = try run(&.{.{ .prompt = &p2, .max_new = 90, .think_budget = 3, .drafts = false, .call_gate = gates[1].hook() }}); + defer free(serial); + try std.testing.expectEqualSlices(u32, serial[0], drafted[0]); + const short = try run(&.{.{ .prompt = &p2, .max_new = 2, .call_gate = gates[2].hook() }}); + defer free(short); + try std.testing.expectEqual(@as(usize, 2), short[0].len); +} diff --git a/zig/src/core/lanes/lanes.zig b/zig/src/core/lanes/lanes.zig index e943bf9d46..c323ba108b 100644 --- a/zig/src/core/lanes/lanes.zig +++ b/zig/src/core/lanes/lanes.zig @@ -42,6 +42,7 @@ test { std.testing.refAllDecls(@This()); _ = @import("engine_test.zig"); _ = @import("recorder_test.zig"); + _ = @import("call_gate_test.zig"); _ = @import("gpu_full_test.zig"); _ = @import("drafted_test.zig"); _ = @import("pricing_test.zig"); diff --git a/zig/src/core/lanes/open.zig b/zig/src/core/lanes/open.zig index 7cddf9a064..00fd04dbc3 100644 --- a/zig/src/core/lanes/open.zig +++ b/zig/src/core/lanes/open.zig @@ -37,6 +37,7 @@ pub fn opened(e: *Engine, s: *Stream) !void { const drawn = try e.drawFirst(s, position); var feed: Feed = .{ .handle = drawn }; if (try e.forcedNext(s)) |t| feed = .{ .value = t }; + if (s.call_gate) |gate| feed = try gate.feed(e.arena.allocator(), e.gpa, &s.force, e.backend, feed); var asked: ?u32 = null; if (e.cfg.family_mtp and s.drafts) { // the head reads the prompt's last row and the first token, and drafts the one after it diff --git a/zig/src/core/lanes/stream.zig b/zig/src/core/lanes/stream.zig index 6763759ab1..17e947d22e 100644 --- a/zig/src/core/lanes/stream.zig +++ b/zig/src/core/lanes/stream.zig @@ -1,6 +1,7 @@ //! One stream (Python LaneStream) and the round loop's state for it (Python's dicts keyed by stream id). const std = @import("std"); const grammar = @import("grammar.zig"); +pub const CallGate = @import("call_gate.zig"); const shape = @import("shape.zig"); const plan_lanes = @import("plan_lanes.zig"); const Allocator = std.mem.Allocator; @@ -145,6 +146,7 @@ pub const Spec = struct { think_open: ?bool = null, // null: open when a budget is set (the server's rule) loop_guard: bool = false, loop_windows: u32 = 0, // the guard's novelty signal: this many dry windows in a row close thinking (0: cycles only) + call_gate: ?CallGate.Hook = null, chunks: []const u32 = &.{}, // where prefill chunks start after 0 (Python's PrefillPlan); empty: the backend's step history_len: u32 = 0, // prompt prefix lengths a backend may keep for a later turn: the rendered history, shared_prefixes: []const u32 = &.{}, // then shared system blocks @@ -177,6 +179,7 @@ pub const Stream = struct { think_end: i64, think_open: bool, loop_guard: bool, + call_gate: ?CallGate.Hook, loop_period: ?u32 = null, novelty: ?Novelty = null, // the guard's novelty signal (Spec.loop_windows) loop_novelty: ?f64 = null, // the dry window's share of new grams, when that signal closed the think block @@ -251,6 +254,7 @@ pub const Stream = struct { .think_open = spec.think_open orelse (spec.think_budget > 0 or (spec.loop_guard and spec.think_end >= 0)), .loop_guard = spec.loop_guard, .novelty = if (spec.loop_guard and spec.loop_windows > 0) .{ .windows = spec.loop_windows } else null, + .call_gate = spec.call_gate, .chunks = spec.chunks, .history_len = spec.history_len, .shared_prefixes = spec.shared_prefixes, @@ -398,6 +402,7 @@ pub const Stream = struct { var ended = false; // a novelty window ended in this batch: only its first end may fire, as loopCut reads it for (tokens, 0..) |t, i| { if (s.finished) break; + if (s.call_gate) |gate| try gate.observe(gate.ptr, t); try s.context.append(gpa, t); if (rows.len > 0) try s.rows.append(gpa, rows[i]); landed += 1; diff --git a/zig/src/native/cuda.zig b/zig/src/native/cuda.zig index e3580135e7..271e5663bf 100644 --- a/zig/src/native/cuda.zig +++ b/zig/src/native/cuda.zig @@ -477,6 +477,7 @@ fn openWith(comptime F: type, a: Allocator, gpa: Allocator, io: std.Io, o: api.O .vision = if (@hasField(@TypeOf(loaded), "vision")) vision_offer.offer(loaded.vision) else null, .lane_vision = if (@hasField(@TypeOf(loaded), "lane_vision")) loaded.lane_vision else null, .efforts = familyEfforts(F), + .call_gates = F == dsv41.native, }, .{ .ctx = loaded.ctx, .text = F.explain }); opened = true; return .{ .engine = h.host.engine(), .close = Host.close, .ctx = h }; diff --git a/zig/src/server/call_choice_test.zig b/zig/src/server/call_choice_test.zig new file mode 100644 index 0000000000..c41effa2be --- /dev/null +++ b/zig/src/server/call_choice_test.zig @@ -0,0 +1,109 @@ +//! Required and named choices reach the native call gate through a split-token DSML template. +const std = @import("std"); +const api = @import("engine_api"); +const json = @import("json"); +const errors = @import("errors.zig"); +const model_text = @import("model_text.zig"); +const tool_specs = @import("tool_specs.zig"); +const Server = @import("server.zig").Server; +const A = std.mem.Allocator; + +const DsmlText = struct { + const marker = "|DSML|"; + + fn text(t: *@This()) model_text.Text { + return .{ .ctx = t, .vtable = &.{ .encode = encode, .decode = decode, .token_id = tokenId, .token_string = tokenString, .vocab_size = vocabSize, .eos_ids = eosIds, .render = render, .template_source = templateSource } }; + } + + fn encode(_: *anyopaque, a: A, input: []const u8, _: bool) model_text.Error![]u32 { + var ids: std.ArrayList(u32) = .empty; + var at: usize = 0; + while (at < input.len) { + if (std.mem.startsWith(u8, input[at..], marker)) { + try ids.append(a, 1000); + at += marker.len; + } else { + try ids.append(a, input[at]); + at += 1; + } + } + return ids.toOwnedSlice(a); + } + + fn decode(_: *anyopaque, a: A, ids: []const u32) A.Error![]u8 { + var out: std.ArrayList(u8) = .empty; + for (ids) |id| { + if (id == 1000) try out.appendSlice(a, marker) else try out.append(a, @intCast(id)); + } + return out.toOwnedSlice(a); + } + + fn tokenId(_: *anyopaque, piece: []const u8) ?u32 { + return if (std.mem.eql(u8, piece, marker)) 1000 else null; + } + + fn tokenString(ctx: *anyopaque, a: A, id: u32) A.Error![]u8 { + return decode(ctx, a, &.{id}); + } + + fn vocabSize(_: *anyopaque) u32 { + return 1001; + } + + fn eosIds(_: *anyopaque) []const u32 { + return &.{}; + } + + fn render(_: *anyopaque, a: A, _: json.Value, _: model_text.RenderOptions, _: *[]const u8) model_text.Error![]u8 { + return a.dupe(u8, "<|DSML| calls>\n<|DSML| invoke name=\"tfprobe_fn\">"); + } + + fn templateSource(_: *anyopaque) []const u8 { + return ""; + } +}; + +fn preflight(srv: *Server, cx: *errors.Cx, choice: []const u8) !api.Request { + const tools = (try json.parse(cx.a, "[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\"}},{\"type\":\"function\",\"function\":{\"name\":\"get_time\"}}]")).ok; + const selected = (try json.parse(cx.a, choice)).ok; + const active = try tool_specs.active(cx, tools, selected); + const fields = try json.newObject(cx.a); + try fields.put(cx.a, "tool_call_required", .{ .bool = try tool_specs.choiceRequiresCall(cx.a, selected) }); + var request: api.Request = .{ .prompt = &.{}, .max_tokens = 128 }; + try srv.checkFeatures(cx, .{ .object = fields }, active.len > 0, false, &.{}, active, &request); + return request; +} + +test "DSML required and named preflight accepts a split opener and the active names" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var text: DsmlText = .{}; + var srv: Server = .{ + .gpa = a, + .io = std.testing.io, + .engine = undefined, + .info = .{ .call_gates = true }, + .text = text.text(), + .config = .{ .served_name = "test", .model_ids = &.{"test"} }, + .keys = null, + .metrics = .{ .gpa = a }, + .store = .{ .gpa = a }, + .arena = .init(a), + .slide = .init(a, std.testing.io, null), + .teacher = .init(a), + }; + var cx: errors.Cx = .{ .a = a }; + const required = try preflight(&srv, &cx, "\"required\""); + try std.testing.expectEqual(@as(u32, 1000), required.call.?.opener); + try std.testing.expectEqual(@as(usize, 2), required.call.?.names.len); + const named = try preflight(&srv, &cx, "{\"type\":\"function\",\"function\":{\"name\":\"get_time\"}}"); + try std.testing.expectEqual(@as(usize, 1), named.call.?.names.len); + try std.testing.expectEqualStrings("get_time", named.call.?.names[0]); + try std.testing.expect((try preflight(&srv, &cx, "\"auto\"")).call == null); + try std.testing.expect((try preflight(&srv, &cx, "\"none\"")).call == null); + srv.info.call_gates = false; + try std.testing.expectError(error.Refused, preflight(&srv, &cx, "\"required\"")); + try std.testing.expectError(error.Refused, preflight(&srv, &cx, "{\"type\":\"function\",\"function\":{\"name\":\"get_time\"}}")); + try std.testing.expect((try preflight(&srv, &cx, "\"auto\"")).call == null); +} diff --git a/zig/src/server/root.zig b/zig/src/server/root.zig index 06be1ffc75..41fff9f27c 100644 --- a/zig/src/server/root.zig +++ b/zig/src/server/root.zig @@ -100,4 +100,5 @@ test { _ = @import("benchmark_args.zig"); _ = @import("benchmark_cli_test.zig"); _ = @import("benchmark_run_test.zig"); + _ = @import("call_choice_test.zig"); } diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig index d048ab0c17..3cbea5f3da 100644 --- a/zig/src/server/server.zig +++ b/zig/src/server/server.zig @@ -212,7 +212,6 @@ pub const Server = struct { /// The call gate and grammar a request asks for, or its refusal when the engine cannot enforce them. pub fn checkFeatures(srv: *Server, cx: *Cx, f: Value, has_tools: bool, thinking: bool, prompt_ids: []const u32, tools: []const Value, request: *api.Request) errors.Refused!void { - _ = prompt_ids; const required = if (f.get("tool_call_required")) |r| r.truthy() else false; if (has_tools and required and srv.info.structures) { const one_call = if (f.get("tool_call_single")) |x| x.truthy() else false; @@ -231,7 +230,14 @@ pub const Server = struct { }; break :blk if (ids.len > 0) ids[0] else null; } else srv.text.tokenId(""); - request.call = .{ .opener = opener.?, .lead = form.?.lead orelse "", .tail = form.?.tail orelse "", .names = names.items, .think_open = think_open, .think_end = srv.text.tokenId(srv.markers.close) }; + request.call = .{ .head = form.?.head, .opener = opener.?, .lead = form.?.lead orelse "", .tail = form.?.tail orelse "", .names = names.items, .think_open = think_open, .think_end = srv.text.tokenId(srv.markers.close) }; + const gate = try cx.a.create(api.CallFilter.Gate); + gate.* = api.CallFilter.Gate.init(cx.a, request.call.?, .{ + .ptr = srv, + .encode = callEncode, + .text = callText, + }, prompt_ids); + request.call.?.filter = gate.hook(); } if (try grammar.spec(cx, f)) |s| { if (!srv.info.structures) return cx.refuse("structured output (response_format and the guided_* fields) is not supported by this engine yet"); @@ -282,13 +288,29 @@ pub const Server = struct { return srv.grammars; } - const Form = struct { opener: []const u8, lead: ?[]const u8, tail: ?[]const u8 }; + const Form = struct { head: []const u8 = "", opener: []const u8, lead: ?[]const u8, tail: ?[]const u8 }; + + fn callEncode(ptr: *anyopaque, a: std.mem.Allocator, text: []const u8) ![]const u32 { + const srv: *Server = @ptrCast(@alignCast(ptr)); + return srv.text.encode(a, text, false); + } + + fn callText(ptr: *anyopaque, a: std.mem.Allocator, token: u32) ![]const u8 { + const srv: *Server = @ptrCast(@alignCast(ptr)); + return srv.text.decode(a, &.{token}); + } /// (opener, lead, tail) of this template's tool calls, read from a rendered probe call. fn callForm(srv: *Server, cx: *Cx) errors.Refused!?Form { const probe = (try json.parse(cx.a, "[{\"role\": \"user\", \"content\": \"x\"}, {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": [{\"id\": \"call_0\", \"type\": \"function\", \"function\": {\"name\": \"tfprobe_fn\", \"arguments\": {}}}]}]")).ok; var scratch: Cx = .{ .a = cx.a }; const text = if (prompt.renderIds(srv, &scratch, probe, &.{}, false, null, false)) |ids| try srv.text.decode(cx.a, ids) else |_| ""; + if (renderedCallForm(text, srv.text.tokenId("|DSML|") != null)) |form| return form; + for (reply_text.calls) |c| if (srv.text.tokenId(c[0]) != null) return .{ .opener = c[0], .lead = null, .tail = null }; + return null; + } + + fn renderedCallForm(text: []const u8, dsml_token: bool) ?Form { if (std.mem.lastIndexOf(u8, text, "tfprobe_fn")) |at| { var best: ?usize = null; var opener: []const u8 = ""; @@ -300,10 +322,12 @@ pub const Server = struct { }; if (best) |s| { const end = at + "tfprobe_fn".len; + const dsml = "|DSML|"; + if (std.mem.eql(u8, opener, "<|DSML| calls>") and dsml_token) + return .{ .head = "<", .opener = dsml, .lead = text[s + 1 + dsml.len .. at], .tail = if (end < text.len) text[end .. end + 1] else "" }; return .{ .opener = opener, .lead = text[s + opener.len .. at], .tail = if (end < text.len) text[end .. end + 1] else "" }; } } - for (reply_text.calls) |c| if (srv.text.tokenId(c[0]) != null) return .{ .opener = c[0], .lead = null, .tail = null }; return null; } @@ -421,3 +445,15 @@ fn lateSystem(a: Allocator, text: model_text.Text) ![]const u8 { fn needsUserAfterTool(source: []const u8) bool { return std.mem.indexOf(u8, source, "No user query found") != null; } + +test "rendered DeepSeek call uses text head then DSML token" { + const form = Server.renderedCallForm("<|DSML| calls>\n<|DSML| invoke name=\"tfprobe_fn\">", true).?; + try std.testing.expectEqualStrings("<", form.head); + try std.testing.expectEqualStrings("|DSML|", form.opener); + try std.testing.expectEqualStrings(" calls>\n<|DSML| invoke name=\"", form.lead.?); + try std.testing.expectEqualStrings("\"", form.tail.?); + const legacy = Server.renderedCallForm("{\"name\":\"tfprobe_fn\"}", false).?; + try std.testing.expectEqualStrings("", legacy.head); + try std.testing.expectEqualStrings("", legacy.opener); + try std.testing.expect(Server.renderedCallForm("no rendered call", true) == null); +}