From 4dafbd3dd80a9a7aaa712041fd05ed4ebbe9ce79 Mon Sep 17 00:00:00 2001 From: devopsfortunato <57877156+devopsfortunato@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:17:09 +0100 Subject: [PATCH 1/2] engine api: an optional image capability beside speech (#1573) --- CHANGELOG.md | 1 + zig/src/core/engine_api.zig | 27 ++-- zig/src/core/engine_api_test.zig | 1 + zig/src/core/image_api.zig | 24 ++++ zig/src/core/image_api_test.zig | 219 +++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 12 deletions(-) create mode 100644 zig/src/core/image_api.zig create mode 100644 zig/src/core/image_api_test.zig diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a40563961..b2856928bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -255,6 +255,7 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel - Flash Next on Metal reports a filling prompt's progress: the dashboard shows what's left, and `TENSORFOLD_PREFILL_ORDER=srpt` works from a server's first long prompt (#2194). - CI: `zig` and `zig-windows` keep main's build cache instead of rebuilding from nothing, every check still reruns, and the quick checks run first. +- The engine API carries an optional image capability beside speech: `Info.image` states what an engine's image host serves and a defaulted `VTable.submit_image` hands back one encoded PNG a request through the caller's own list and allocator, so an engine without the entry is unchanged (#1573). - HIP: the `tf_sum_partials` records follow ROCm 10.1's hipcc, taking 8 limit problems off the contract (#2233). Thanks to @AbrahamCete. diff --git a/zig/src/core/engine_api.zig b/zig/src/core/engine_api.zig index ad99495d00..70ad410097 100644 --- a/zig/src/core/engine_api.zig +++ b/zig/src/core/engine_api.zig @@ -27,10 +27,7 @@ pub const seedFor = lanes.sampling.seedFor; pub const LogprobRow = lanes.LogprobRow; /// The request's stop strings, checked on the engine's thread after each committed token (the server decodes the tail). -pub const Stop = struct { - ctx: *anyopaque, - check: *const fn (ctx: *anyopaque, emitted: []const u32) bool, -}; +pub const Stop = struct { ctx: *anyopaque, check: *const fn (ctx: *anyopaque, emitted: []const u32) bool }; pub const CallGate = @import("engine_request_controls.zig").CallGate; pub const Structure = @import("engine_request_controls.zig").Structure; @@ -124,6 +121,10 @@ pub const SpeechRequest = @import("speech_api.zig").SpeechRequest; pub const SpeechResult = @import("speech_api.zig").SpeechResult; pub const SpeechInfo = @import("speech_api.zig").SpeechInfo; pub const SpeechSubmitError = error{ Closed, Busy, InvalidRequest, Unsupported }; +pub const ImageRequest = @import("image_api.zig").ImageRequest; +pub const ImageResult = @import("image_api.zig").ImageResult; +pub const ImageInfo = @import("image_api.zig").ImageInfo; +pub const ImageSubmitError = error{ Closed, Busy, InvalidRequest, Unsupported }; pub const Reason = enum { stop, length, cancelled, failed, silence }; pub const speech_guard = @import("speech_guard.zig"); @@ -165,6 +166,8 @@ pub const Info = struct { embeddings: ?lanes.embedding.Info = null, readouts: ?lanes.readout.Info = null, // non-null: the engine answers readout requests and nothing else speech: ?SpeechInfo = null, + /// The engine serves one picture a request through `submit_image`; null: no image capability. + image: ?ImageInfo = null, lanes: u32 = 1, /// Server preflight prompt-plus-reply limit (0: the engine owns admission). context_window: u32 = 0, @@ -241,10 +244,7 @@ pub const Info = struct { } }; -pub const Admit = struct { - ctx: *anyopaque, - fits: *const fn (ctx: *anyopaque, prompt: u32, max_tokens: u32) bool, -}; +pub const Admit = struct { ctx: *anyopaque, fits: *const fn (ctx: *anyopaque, prompt: u32, max_tokens: u32) bool }; /// The bytes the lane host may promise at once, and what one request's caches cost at most (core/kv_ledger.zig). pub const Ledger = struct { @@ -376,10 +376,7 @@ pub const Opened = struct { pub const Memory = struct { active: u64 = 0, cache: u64 = 0, peak: u64 = 0, pinned: u64 = 0, limit: ?u64 = null }; /// A backend's words for its own refusals (a request it cannot serve); null: the error's name. -pub const Explain = struct { - ctx: ?*anyopaque = null, - text: *const fn (ctx: ?*anyopaque, err: anyerror) ?[]const u8, -}; +pub const Explain = struct { ctx: ?*anyopaque = null, text: *const fn (ctx: ?*anyopaque, err: anyerror) ?[]const u8 }; /// A backend's own memory counts for LaneHost; read from HTTP threads, so it must not touch the GPU. pub const MemorySource = struct { @@ -424,6 +421,7 @@ pub const Engine = struct { info: *const fn (ctx: *anyopaque) Info, submit: *const fn (ctx: *anyopaque, id: Id, request: *const Request, sink: Sink) SubmitError!void, submit_speech: ?*const fn (*anyopaque, Id, *const SpeechRequest, Sink) SpeechSubmitError!void = null, + submit_image: ?*const fn (*anyopaque, Id, *const ImageRequest, Sink) ImageSubmitError!void = null, /// Ends a request between rounds; its ``finished`` (cancelled) still arrives. Unknown ids are ignored. cancel: *const fn (ctx: *anyopaque, id: Id) void, status: *const fn (ctx: *anyopaque, out: *Status, stream_tokens: []u32) void, @@ -471,6 +469,11 @@ pub const Engine = struct { const f = e.vtable.submit_speech orelse return error.Unsupported; return f(e.ctx, id, request, sink); } + /// The image hook, or Unsupported when this engine serves no picture. + pub fn submitImage(e: Engine, id: Id, request: *const ImageRequest, sink: Sink) ImageSubmitError!void { + const f = e.vtable.submit_image orelse return error.Unsupported; + return f(e.ctx, id, request, sink); + } pub fn cancel(e: Engine, id: Id) void { e.vtable.cancel(e.ctx, id); } diff --git a/zig/src/core/engine_api_test.zig b/zig/src/core/engine_api_test.zig index 030fd852a2..74a4aadc51 100644 --- a/zig/src/core/engine_api_test.zig +++ b/zig/src/core/engine_api_test.zig @@ -14,4 +14,5 @@ test { _ = @import("prompt_imprint.zig"); _ = @import("keepalive.zig"); _ = @import("originals_test.zig"); + _ = @import("image_api_test.zig"); } diff --git a/zig/src/core/image_api.zig b/zig/src/core/image_api.zig new file mode 100644 index 0000000000..e67fee21c7 --- /dev/null +++ b/zig/src/core/image_api.zig @@ -0,0 +1,24 @@ +//! The image model's own request and reply types, beside the speech ones in engine_api.zig. +const std = @import("std"); + +/// One picture: the encoded PNG's byte count and why the pass ended; the bytes are in the request's `png`. +pub const ImageResult = struct { bytes: usize = 0, reason: @import("engine_api.zig").Reason = .stop }; + +/// What the engine's image host serves, fixed once it has loaded: the host's stated limits, each one non-zero. +pub const ImageInfo = struct { max_side: u16, max_pixels: u32, max_steps: u32, max_prompt_tokens: u32 }; + +/// The one picture a request carries, caller-owned until `finished`. +pub const ImageRequest = struct { + prompt: []const u32, + negative: []const u32 = &.{}, + width: u16, + height: u16, + steps: u32, + guidance: f32, + seed: u64, + result: *ImageResult, + /// The engine appends its one PNG here with `alloc`; the caller frees it after `finished`. + png: *std.ArrayList(u8), + /// The allocator `png` lives in: the engine appends with it, the caller frees with it. + alloc: std.mem.Allocator, +}; diff --git a/zig/src/core/image_api_test.zig b/zig/src/core/image_api_test.zig new file mode 100644 index 0000000000..406299ae04 --- /dev/null +++ b/zig/src/core/image_api_test.zig @@ -0,0 +1,219 @@ +//! The engine API's image capability, against a fake host that records what it was handed and hands back one PNG. +const std = @import("std"); +const api = @import("engine_api.zig"); +const t = std.testing; +const a = t.allocator; + +/// The bytes a fake host appends: a PNG signature, so the caller can tell its own list from an untouched one. +const png = "\x89PNG\r\n\x1a\nIDAT"; + +/// What a fake host's entry does when it is reached. +const Mode = enum { ok, busy, invalid }; + +/// One fake image host: `serve` says whether it has the entry at all, `mode` what that entry answers. +const Stub = struct { + serve: bool = true, + mode: Mode = .ok, + reason: api.Reason = .stop, + limits: ?api.ImageInfo = null, + /// Every field the entry was handed, as the caller passed it. + seen: Seen = .{}, + /// The table `engine` hands out: `serve` decides whether it carries the image entry. + vtable: api.Engine.VTable = undefined, + + pub const Seen = struct { + calls: usize = 0, + id: api.Id = 0, + prompt: []const u32 = &.{}, + negative: []const u32 = &.{}, + width: u16 = 0, + height: u16 = 0, + steps: u32 = 0, + guidance: f32 = 0, + seed: u64 = 0, + }; + + fn engine(s: *Stub) api.Engine { + s.vtable = .{ + .info = info, + .submit = submit, + .cancel = cancel, + .status = status, + .memory = memory, + .submit_image = if (s.serve) submitImage else null, + }; + return .{ .ctx = s, .vtable = &s.vtable }; + } + + fn info(ctx: *anyopaque) api.Info { + const s: *Stub = @ptrCast(@alignCast(ctx)); + return .{ .text_generation = false, .image = s.limits }; + } + + fn submitImage(ctx: *anyopaque, id: api.Id, request: *const api.ImageRequest, sink: api.Sink) api.ImageSubmitError!void { + const s: *Stub = @ptrCast(@alignCast(ctx)); + s.seen = .{ + .calls = s.seen.calls + 1, + .id = id, + .prompt = request.prompt, + .negative = request.negative, + .width = request.width, + .height = request.height, + .steps = request.steps, + .guidance = request.guidance, + .seed = request.seed, + }; + switch (s.mode) { + .ok => {}, + .busy => return error.Busy, + .invalid => return error.InvalidRequest, + } + request.png.appendSlice(request.alloc, png) catch return error.Busy; // no room: the caller may come back + request.result.* = .{ .bytes = png.len, .reason = s.reason }; + sink.event(sink.ctx, id, &.{ .finished = .{ .reason = s.reason } }); + } + + fn submit(_: *anyopaque, _: api.Id, _: *const api.Request, _: api.Sink) api.SubmitError!void { + unreachable; + } + fn cancel(_: *anyopaque, _: api.Id) void {} + fn status(_: *anyopaque, _: *api.Status, _: []u32) void {} + fn memory(_: *anyopaque, _: bool) ?api.Memory { + return null; + } +}; + +/// The caller's end of the sink: one `finished` a picture, with the reason the engine stated. +const Record = struct { + events: usize = 0, + id: api.Id = 0, + reason: api.Reason = .stop, + + fn sink(r: *Record) api.Sink { + return .{ .ctx = r, .event = event }; + } + + fn event(ctx: *anyopaque, id: api.Id, e: *const api.Event) void { + const r: *Record = @ptrCast(@alignCast(ctx)); + switch (e.*) { + .finished => |f| { + r.events += 1; + r.id = id; + r.reason = f.reason; + }, + else => {}, + } + } +}; + +/// One caller-owned request: its prompt ids, its result, and the list its PNG lands in. +const Case = struct { + prompt: [3]u32 = .{ 11, 12, 13 }, + negative: [2]u32 = .{ 7, 8 }, + result: api.ImageResult = .{}, + list: std.ArrayList(u8) = .empty, + + fn request(c: *Case) api.ImageRequest { + return .{ + .prompt = &c.prompt, + .negative = &c.negative, + .width = 1024, + .height = 768, + .steps = 8, + .guidance = 3.5, + .seed = 42, + .result = &c.result, + .png = &c.list, + .alloc = a, + }; + } +}; + +test "the image entry is reached with exactly the caller's fields" { + var stub = Stub{}; + var case = Case{}; + var record = Record{}; + const engine = stub.engine(); + var request = case.request(); + try engine.submitImage(9, &request, record.sink()); + defer case.list.deinit(a); + try t.expectEqual(@as(usize, 1), stub.seen.calls); + try t.expectEqual(@as(api.Id, 9), stub.seen.id); + try t.expectEqualSlices(u32, &.{ 11, 12, 13 }, stub.seen.prompt); + try t.expectEqualSlices(u32, &.{ 7, 8 }, stub.seen.negative); + try t.expectEqual(@as(u16, 1024), stub.seen.width); + try t.expectEqual(@as(u16, 768), stub.seen.height); + try t.expectEqual(@as(u32, 8), stub.seen.steps); + try t.expectEqual(@as(f32, 3.5), stub.seen.guidance); + try t.expectEqual(@as(u64, 42), stub.seen.seed); +} + +test "the PNG the engine appends is the caller's to free with its own allocator" { + var stub = Stub{}; + var case = Case{}; + var record = Record{}; + const engine = stub.engine(); + var request = case.request(); + try engine.submitImage(1, &request, record.sink()); + defer case.list.deinit(a); // the allocator the engine appended with: a mismatch or a leak fails here + try t.expectEqualSlices(u8, png, case.list.items); + try t.expectEqual(@as(usize, png.len), case.result.bytes); + try t.expectEqual(api.Reason.stop, case.result.reason); +} + +test "an engine without the image entry is unsupported" { + var stub = Stub{ .serve = false }; + var case = Case{}; + var record = Record{}; + const engine = stub.engine(); + var request = case.request(); + try t.expectError(error.Unsupported, engine.submitImage(1, &request, record.sink())); + try t.expectEqual(@as(usize, 0), stub.seen.calls); + try t.expectEqual(@as(usize, 0), case.list.items.len); + try t.expectEqual(@as(usize, 0), record.events); + try t.expect(engine.info().image == null); +} + +test "an engine error comes back to the caller unchanged" { + inline for (.{ Mode.busy, Mode.invalid }) |mode| { + var stub = Stub{ .mode = mode }; + var case = Case{}; + var record = Record{}; + const engine = stub.engine(); + var request = case.request(); + const want: api.ImageSubmitError = switch (mode) { + .busy => error.Busy, + .invalid => error.InvalidRequest, + .ok => unreachable, + }; + try t.expectError(want, engine.submitImage(3, &request, record.sink())); + try t.expectEqual(@as(usize, 0), case.list.items.len); + try t.expectEqual(@as(usize, 0), record.events); + } +} + +test "Info carries the engine's image limits exactly, and none by default" { + const limits = api.ImageInfo{ .max_side = 2048, .max_pixels = 4_194_304, .max_steps = 50, .max_prompt_tokens = 512 }; + var stub = Stub{ .limits = limits }; + const engine = stub.engine(); + try t.expectEqual(limits, engine.info().image.?); + try t.expectEqual(@as(u16, 2048), engine.info().image.?.max_side); + try t.expectEqual(@as(u32, 4_194_304), engine.info().image.?.max_pixels); + try t.expectEqual(@as(u32, 50), engine.info().image.?.max_steps); + try t.expectEqual(@as(u32, 512), engine.info().image.?.max_prompt_tokens); + try t.expect((api.Info{}).image == null); +} + +test "the finished event the engine sends arrives at the caller's sink" { + var stub = Stub{ .reason = .length }; + var case = Case{}; + var record = Record{}; + const engine = stub.engine(); + var request = case.request(); + try engine.submitImage(12, &request, record.sink()); + defer case.list.deinit(a); + try t.expectEqual(@as(usize, 1), record.events); + try t.expectEqual(@as(api.Id, 12), record.id); + try t.expectEqual(api.Reason.length, record.reason); + try t.expectEqual(api.Reason.length, case.result.reason); +} From 87f4563910dc2d73556407ddb89dcbd8f781a322 Mon Sep 17 00:00:00 2001 From: ashhart <71890694+ashhart@users.noreply.github.com> Date: Sat, 7 Nov 2026 05:11:23 +0000 Subject: [PATCH 2/2] changelog: back under Unreleased --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2856928bf..2337429323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel ## Unreleased +- The engine API carries an optional image capability beside speech: `Info.image` states what an engine's image host serves and a defaulted `VTable.submit_image` hands back one encoded PNG a request through the caller's own list and allocator, so an engine without the entry is unchanged (#1573). + - Build: `zig build compile-dsv41-server` and `dsv41-serve` build again: the DeepSeek server module gets the `lanes` module its `/v1/systemone` route reads (#2412). Thanks to @Defilan. - Tests: four host steps that no CI job runs build and pass again: `test-gemma4-affine-host` pins `affine_mm.metal`'s current bytes, the compile-only Metal contract runners import `masked_sources`, `compile-metal-residency-host` imports `kimi_k3`, and the GLM Metal fact tests give `facts()` a host device object instead of an undefined one, which crashed them at random (#2409, part of #2031). @@ -255,7 +257,6 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel - Flash Next on Metal reports a filling prompt's progress: the dashboard shows what's left, and `TENSORFOLD_PREFILL_ORDER=srpt` works from a server's first long prompt (#2194). - CI: `zig` and `zig-windows` keep main's build cache instead of rebuilding from nothing, every check still reruns, and the quick checks run first. -- The engine API carries an optional image capability beside speech: `Info.image` states what an engine's image host serves and a defaulted `VTable.submit_image` hands back one encoded PNG a request through the caller's own list and allocator, so an engine without the entry is unchanged (#1573). - HIP: the `tf_sum_partials` records follow ROCm 10.1's hipcc, taking 8 limit problems off the contract (#2233). Thanks to @AbrahamCete.