engine api: an optional image capability beside speech (#1573)

CHANGELOG.md

@@ -4,6 +4,8 @@ Each release's page on GitHub has its notes and measurements. See [the 1.0.0 rel
 
 ## Unreleased
 
+- The engine API carries an optional image capability beside speech: `Info.image` states what an engine's image host serves and a defaulted `VTable.submit_image` hands back one encoded PNG a request through the caller's own list and allocator, so an engine without the entry is unchanged (#1573).
+
 - Build: `zig build compile-dsv41-server` and `dsv41-serve` build again: the DeepSeek server module gets the `lanes` module its `/v1/systemone` route reads (#2412). Thanks to @Defilan.
 
 - Tests: four host steps that no CI job runs build and pass again: `test-gemma4-affine-host` pins `affine_mm.metal`'s current bytes, the compile-only Metal contract runners import `masked_sources`, `compile-metal-residency-host` imports `kimi_k3`, and the GLM Metal fact tests give `facts()` a host device object instead of an undefined one, which crashed them at random (#2409, part of #2031).

zig/src/core/engine_api.zig

@@ -27,10 +27,7 @@ pub const seedFor = lanes.sampling.seedFor;
 pub const LogprobRow = lanes.LogprobRow;
 
 /// The request's stop strings, checked on the engine's thread after each committed token (the server decodes the tail).
-pub const Stop = struct {
-    ctx: *anyopaque,
-    check: *const fn (ctx: *anyopaque, emitted: []const u32) bool,
-};
+pub const Stop = struct { ctx: *anyopaque, check: *const fn (ctx: *anyopaque, emitted: []const u32) bool };
 
 pub const CallGate = @import("engine_request_controls.zig").CallGate;
 pub const Structure = @import("engine_request_controls.zig").Structure;
@@ -124,6 +121,10 @@ pub const SpeechRequest = @import("speech_api.zig").SpeechRequest;
 pub const SpeechResult = @import("speech_api.zig").SpeechResult;
 pub const SpeechInfo = @import("speech_api.zig").SpeechInfo;
 pub const SpeechSubmitError = error{ Closed, Busy, InvalidRequest, Unsupported };
+pub const ImageRequest = @import("image_api.zig").ImageRequest;
+pub const ImageResult = @import("image_api.zig").ImageResult;
+pub const ImageInfo = @import("image_api.zig").ImageInfo;
+pub const ImageSubmitError = error{ Closed, Busy, InvalidRequest, Unsupported };
 
 pub const Reason = enum { stop, length, cancelled, failed, silence };
 pub const speech_guard = @import("speech_guard.zig");
@@ -165,6 +166,8 @@ pub const Info = struct {
     embeddings: ?lanes.embedding.Info = null,
     readouts: ?lanes.readout.Info = null, // non-null: the engine answers readout requests and nothing else
     speech: ?SpeechInfo = null,
+    /// The engine serves one picture a request through `submit_image`; null: no image capability.
+    image: ?ImageInfo = null,
     lanes: u32 = 1,
     /// Server preflight prompt-plus-reply limit (0: the engine owns admission).
     context_window: u32 = 0,
@@ -241,10 +244,7 @@ pub const Info = struct {
     }
 };
 
-pub const Admit = struct {
-    ctx: *anyopaque,
-    fits: *const fn (ctx: *anyopaque, prompt: u32, max_tokens: u32) bool,
-};
+pub const Admit = struct { ctx: *anyopaque, fits: *const fn (ctx: *anyopaque, prompt: u32, max_tokens: u32) bool };
 
 /// The bytes the lane host may promise at once, and what one request's caches cost at most (core/kv_ledger.zig).
 pub const Ledger = struct {
@@ -376,10 +376,7 @@ pub const Opened = struct {
 pub const Memory = struct { active: u64 = 0, cache: u64 = 0, peak: u64 = 0, pinned: u64 = 0, limit: ?u64 = null };
 
 /// A backend's words for its own refusals (a request it cannot serve); null: the error's name.
-pub const Explain = struct {
-    ctx: ?*anyopaque = null,
-    text: *const fn (ctx: ?*anyopaque, err: anyerror) ?[]const u8,
-};
+pub const Explain = struct { ctx: ?*anyopaque = null, text: *const fn (ctx: ?*anyopaque, err: anyerror) ?[]const u8 };
 
 /// A backend's own memory counts for LaneHost; read from HTTP threads, so it must not touch the GPU.
 pub const MemorySource = struct {
@@ -424,6 +421,7 @@ pub const Engine = struct {
         info: *const fn (ctx: *anyopaque) Info,
         submit: *const fn (ctx: *anyopaque, id: Id, request: *const Request, sink: Sink) SubmitError!void,
         submit_speech: ?*const fn (*anyopaque, Id, *const SpeechRequest, Sink) SpeechSubmitError!void = null,
+        submit_image: ?*const fn (*anyopaque, Id, *const ImageRequest, Sink) ImageSubmitError!void = null,
         /// Ends a request between rounds; its ``finished`` (cancelled) still arrives. Unknown ids are ignored.
         cancel: *const fn (ctx: *anyopaque, id: Id) void,
         status: *const fn (ctx: *anyopaque, out: *Status, stream_tokens: []u32) void,
@@ -471,6 +469,11 @@ pub const Engine = struct {
         const f = e.vtable.submit_speech orelse return error.Unsupported;
         return f(e.ctx, id, request, sink);
     }
+    /// The image hook, or Unsupported when this engine serves no picture.
+    pub fn submitImage(e: Engine, id: Id, request: *const ImageRequest, sink: Sink) ImageSubmitError!void {
+        const f = e.vtable.submit_image orelse return error.Unsupported;
+        return f(e.ctx, id, request, sink);
+    }
     pub fn cancel(e: Engine, id: Id) void {
         e.vtable.cancel(e.ctx, id);
     }

zig/src/core/engine_api_test.zig

@@ -14,4 +14,5 @@ test {
     _ = @import("prompt_imprint.zig");
     _ = @import("keepalive.zig");
     _ = @import("originals_test.zig");
+    _ = @import("image_api_test.zig");
 }

zig/src/core/image_api.zig

@@ -0,0 +1,24 @@
+//! The image model's own request and reply types, beside the speech ones in engine_api.zig.
+const std = @import("std");
+
+/// One picture: the encoded PNG's byte count and why the pass ended; the bytes are in the request's `png`.
+pub const ImageResult = struct { bytes: usize = 0, reason: @import("engine_api.zig").Reason = .stop };
+
+/// What the engine's image host serves, fixed once it has loaded: the host's stated limits, each one non-zero.
+pub const ImageInfo = struct { max_side: u16, max_pixels: u32, max_steps: u32, max_prompt_tokens: u32 };
+
+/// The one picture a request carries, caller-owned until `finished`.
+pub const ImageRequest = struct {
+    prompt: []const u32,
+    negative: []const u32 = &.{},
+    width: u16,
+    height: u16,
+    steps: u32,
+    guidance: f32,
+    seed: u64,
+    result: *ImageResult,
+    /// The engine appends its one PNG here with `alloc`; the caller frees it after `finished`.
+    png: *std.ArrayList(u8),
+    /// The allocator `png` lives in: the engine appends with it, the caller frees with it.
+    alloc: std.mem.Allocator,
+};

zig/src/core/image_api_test.zig

@@ -0,0 +1,219 @@
+//! The engine API's image capability, against a fake host that records what it was handed and hands back one PNG.
+const std = @import("std");
+const api = @import("engine_api.zig");
+const t = std.testing;
+const a = t.allocator;
+
+/// The bytes a fake host appends: a PNG signature, so the caller can tell its own list from an untouched one.
+const png = "\x89PNG\r\n\x1a\nIDAT";
+
+/// What a fake host's entry does when it is reached.
+const Mode = enum { ok, busy, invalid };
+
+/// One fake image host: `serve` says whether it has the entry at all, `mode` what that entry answers.
+const Stub = struct {
+    serve: bool = true,
+    mode: Mode = .ok,
+    reason: api.Reason = .stop,
+    limits: ?api.ImageInfo = null,
+    /// Every field the entry was handed, as the caller passed it.
+    seen: Seen = .{},
+    /// The table `engine` hands out: `serve` decides whether it carries the image entry.
+    vtable: api.Engine.VTable = undefined,
+
+    pub const Seen = struct {
+        calls: usize = 0,
+        id: api.Id = 0,
+        prompt: []const u32 = &.{},
+        negative: []const u32 = &.{},
+        width: u16 = 0,
+        height: u16 = 0,
+        steps: u32 = 0,
+        guidance: f32 = 0,
+        seed: u64 = 0,
+    };
+
+    fn engine(s: *Stub) api.Engine {
+        s.vtable = .{
+            .info = info,
+            .submit = submit,
+            .cancel = cancel,
+            .status = status,
+            .memory = memory,
+            .submit_image = if (s.serve) submitImage else null,
+        };
+        return .{ .ctx = s, .vtable = &s.vtable };
+    }
+
+    fn info(ctx: *anyopaque) api.Info {
+        const s: *Stub = @ptrCast(@alignCast(ctx));
+        return .{ .text_generation = false, .image = s.limits };
+    }
+
+    fn submitImage(ctx: *anyopaque, id: api.Id, request: *const api.ImageRequest, sink: api.Sink) api.ImageSubmitError!void {
+        const s: *Stub = @ptrCast(@alignCast(ctx));
+        s.seen = .{
+            .calls = s.seen.calls + 1,
+            .id = id,
+            .prompt = request.prompt,
+            .negative = request.negative,
+            .width = request.width,
+            .height = request.height,
+            .steps = request.steps,
+            .guidance = request.guidance,
+            .seed = request.seed,
+        };
+        switch (s.mode) {
+            .ok => {},
+            .busy => return error.Busy,
+            .invalid => return error.InvalidRequest,
+        }
+        request.png.appendSlice(request.alloc, png) catch return error.Busy; // no room: the caller may come back
+        request.result.* = .{ .bytes = png.len, .reason = s.reason };
+        sink.event(sink.ctx, id, &.{ .finished = .{ .reason = s.reason } });
+    }
+
+    fn submit(_: *anyopaque, _: api.Id, _: *const api.Request, _: api.Sink) api.SubmitError!void {
+        unreachable;
+    }
+    fn cancel(_: *anyopaque, _: api.Id) void {}
+    fn status(_: *anyopaque, _: *api.Status, _: []u32) void {}
+    fn memory(_: *anyopaque, _: bool) ?api.Memory {
+        return null;
+    }
+};
+
+/// The caller's end of the sink: one `finished` a picture, with the reason the engine stated.
+const Record = struct {
+    events: usize = 0,
+    id: api.Id = 0,
+    reason: api.Reason = .stop,
+
+    fn sink(r: *Record) api.Sink {
+        return .{ .ctx = r, .event = event };
+    }
+
+    fn event(ctx: *anyopaque, id: api.Id, e: *const api.Event) void {
+        const r: *Record = @ptrCast(@alignCast(ctx));
+        switch (e.*) {
+            .finished => |f| {
+                r.events += 1;
+                r.id = id;
+                r.reason = f.reason;
+            },
+            else => {},
+        }
+    }
+};
+
+/// One caller-owned request: its prompt ids, its result, and the list its PNG lands in.
+const Case = struct {
+    prompt: [3]u32 = .{ 11, 12, 13 },
+    negative: [2]u32 = .{ 7, 8 },
+    result: api.ImageResult = .{},
+    list: std.ArrayList(u8) = .empty,
+
+    fn request(c: *Case) api.ImageRequest {
+        return .{
+            .prompt = &c.prompt,
+            .negative = &c.negative,
+            .width = 1024,
+            .height = 768,
+            .steps = 8,
+            .guidance = 3.5,
+            .seed = 42,
+            .result = &c.result,
+            .png = &c.list,
+            .alloc = a,
+        };
+    }
+};
+
+test "the image entry is reached with exactly the caller's fields" {
+    var stub = Stub{};
+    var case = Case{};
+    var record = Record{};
+    const engine = stub.engine();
+    var request = case.request();
+    try engine.submitImage(9, &request, record.sink());
+    defer case.list.deinit(a);
+    try t.expectEqual(@as(usize, 1), stub.seen.calls);
+    try t.expectEqual(@as(api.Id, 9), stub.seen.id);
+    try t.expectEqualSlices(u32, &.{ 11, 12, 13 }, stub.seen.prompt);
+    try t.expectEqualSlices(u32, &.{ 7, 8 }, stub.seen.negative);
+    try t.expectEqual(@as(u16, 1024), stub.seen.width);
+    try t.expectEqual(@as(u16, 768), stub.seen.height);
+    try t.expectEqual(@as(u32, 8), stub.seen.steps);
+    try t.expectEqual(@as(f32, 3.5), stub.seen.guidance);
+    try t.expectEqual(@as(u64, 42), stub.seen.seed);
+}
+
+test "the PNG the engine appends is the caller's to free with its own allocator" {
+    var stub = Stub{};
+    var case = Case{};
+    var record = Record{};
+    const engine = stub.engine();
+    var request = case.request();
+    try engine.submitImage(1, &request, record.sink());
+    defer case.list.deinit(a); // the allocator the engine appended with: a mismatch or a leak fails here
+    try t.expectEqualSlices(u8, png, case.list.items);
+    try t.expectEqual(@as(usize, png.len), case.result.bytes);
+    try t.expectEqual(api.Reason.stop, case.result.reason);
+}
+
+test "an engine without the image entry is unsupported" {
+    var stub = Stub{ .serve = false };
+    var case = Case{};
+    var record = Record{};
+    const engine = stub.engine();
+    var request = case.request();
+    try t.expectError(error.Unsupported, engine.submitImage(1, &request, record.sink()));
+    try t.expectEqual(@as(usize, 0), stub.seen.calls);
+    try t.expectEqual(@as(usize, 0), case.list.items.len);
+    try t.expectEqual(@as(usize, 0), record.events);
+    try t.expect(engine.info().image == null);
+}
+
+test "an engine error comes back to the caller unchanged" {
+    inline for (.{ Mode.busy, Mode.invalid }) |mode| {
+        var stub = Stub{ .mode = mode };
+        var case = Case{};
+        var record = Record{};
+        const engine = stub.engine();
+        var request = case.request();
+        const want: api.ImageSubmitError = switch (mode) {
+            .busy => error.Busy,
+            .invalid => error.InvalidRequest,
+            .ok => unreachable,
+        };
+        try t.expectError(want, engine.submitImage(3, &request, record.sink()));
+        try t.expectEqual(@as(usize, 0), case.list.items.len);
+        try t.expectEqual(@as(usize, 0), record.events);
+    }
+}
+
+test "Info carries the engine's image limits exactly, and none by default" {
+    const limits = api.ImageInfo{ .max_side = 2048, .max_pixels = 4_194_304, .max_steps = 50, .max_prompt_tokens = 512 };
+    var stub = Stub{ .limits = limits };
+    const engine = stub.engine();
+    try t.expectEqual(limits, engine.info().image.?);
+    try t.expectEqual(@as(u16, 2048), engine.info().image.?.max_side);
+    try t.expectEqual(@as(u32, 4_194_304), engine.info().image.?.max_pixels);
+    try t.expectEqual(@as(u32, 50), engine.info().image.?.max_steps);
+    try t.expectEqual(@as(u32, 512), engine.info().image.?.max_prompt_tokens);
+    try t.expect((api.Info{}).image == null);
+}
+
+test "the finished event the engine sends arrives at the caller's sink" {
+    var stub = Stub{ .reason = .length };
+    var case = Case{};
+    var record = Record{};
+    const engine = stub.engine();
+    var request = case.request();
+    try engine.submitImage(12, &request, record.sink());
+    defer case.list.deinit(a);
+    try t.expectEqual(@as(usize, 1), record.events);
+    try t.expectEqual(@as(api.Id, 12), record.id);
+    try t.expectEqual(api.Reason.length, record.reason);
+    try t.expectEqual(api.Reason.length, case.result.reason);
+}