diff --git a/zig/src/server/images.zig b/zig/src/server/images.zig index d2969d740c..8367e5e837 100644 --- a/zig/src/server/images.zig +++ b/zig/src/server/images.zig @@ -240,6 +240,8 @@ pub fn expand(vision: ?api.Vision, fetch: ?Fetch, text: model_text.Text, cx: *Cx marks += @intFromBool(t == v.image_token); video_marks += @intFromBool(video_token != null and t == video_token.?); } + // none at all: the template has no image path (it swaps media parts for text), a configuration problem + if (marks == 0 and n_images > 0) return cx.fail(.server, "the chat template wrote no image placeholder for {d} image part{s}; a template that turns media parts into text has no image path: serve with --chat-template naming one that writes the image token", .{ n_images, if (n_images == 1) "" else "s" }); if (marks != n_images) return cx.fail(.server, "the chat template wrote {d} image placeholders for {d} image parts", .{ marks, n_images }); if (video_marks != n_videos) return cx.fail(.server, "the chat template wrote {d} video placeholders for {d} video parts", .{ video_marks, n_videos }); if (ps.len == 0) return null; diff --git a/zig/src/server/server.zig b/zig/src/server/server.zig index 3633391aa1..36d96c412a 100644 --- a/zig/src/server/server.zig +++ b/zig/src/server/server.zig @@ -90,6 +90,8 @@ pub const Server = struct { keys: ?*auth.Store, late_system: []const u8 = "system", needs_user_after_tool: bool = false, + /// With image input: whether the chat template writes an image placeholder for an image part (templateImages). + template_images: bool = true, markers: reply_text.Markers = reply_text.think_markers, chunks: chunk_plan.Plan = .{}, effort_levels: []const []const u8 = &.{}, @@ -148,6 +150,10 @@ pub const Server = struct { std.mem.indexOf(u8, source, "thinking_mode") != null; srv.late_system = try lateSystem(a, text); srv.needs_user_after_tool = needsUserAfterTool(source); + if (srv.info.vision != null) { + srv.template_images = templateImages(srv); + if (!srv.template_images) log.line("warning: --vision: the chat template writes no image placeholder for an image part, so image requests will fail; serve with --chat-template naming a template that has an image path", .{}); + } if (text.tokenId("")) |end| { const lead = try text.encode(a, "\n", false); const trail = try text.encode(a, "\n\n", false); @@ -389,6 +395,19 @@ fn lateSystem(a: Allocator, text: model_text.Text) ![]const u8 { return if (std.mem.indexOf(u8, rendered, probe_text) != null) "system" else "user"; } +/// With image input: whether a user turn holding an image part renders an image placeholder (else a startup warning). +fn templateImages(srv: *Server) bool { + const v = srv.info.vision orelse return true; + var arena: std.heap.ArenaAllocator = .init(srv.gpa); + defer arena.deinit(); + const a = arena.allocator(); + var cx: errors.Cx = .{ .a = a }; + const probe = (json.parse(a, "[{\"role\": \"user\", \"content\": [{\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/png;base64,AA==\"}}, {\"type\": \"text\", \"text\": \"x\"}]}]") catch return true).ok; + const ids = @import("prompt.zig").renderIds(srv, &cx, probe, &.{}, false, null, true) catch return true; // a render failure is the request's to report + for (ids) |t| if (t == v.image_token) return true; + return false; +} + /// Templates requiring a user query gain a user turn after trailing tool results replace their last user turn. fn needsUserAfterTool(source: []const u8) bool { return std.mem.indexOf(u8, source, "No user query found") != null; diff --git a/zig/src/server/videos.zig b/zig/src/server/videos.zig index 15c51a40bf..0dffe24cec 100644 --- a/zig/src/server/videos.zig +++ b/zig/src/server/videos.zig @@ -237,3 +237,24 @@ test "a request's videos are limited by the engine's offer, not by a constant" { try std.testing.expectError(error.Refused, images.expand(v, null, images.MarkText.text(), &cx, two, &[_]u32{ 8, 8 }, 0)); try std.testing.expectEqualStrings("a request supports at most 1 videos", cx.message); } + +test "a template that writes no image placeholder is named as the cause, a miscount keeps its counts" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var dummy: u8 = 0; + const v: api.Vision = .{ .ctx = &dummy, .prepare = noPrepare, .image_token = 9, .image_tokens = 64, .max_images = 4 }; + const two = switch (try json.parseText(a, + \\[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,YQ=="}}, + \\ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YQ=="}}, {"type": "text", "text": "x"}]}] + )) { + .ok => |m| m, + .err => return error.TestUnexpectedResult, + }; + var cx: Cx = .{ .a = a }; + try std.testing.expectError(error.Refused, images.expand(v, null, images.MarkText.text(), &cx, two, &[_]u32{ 1, 2 }, 0)); + try std.testing.expectEqualStrings("the chat template wrote no image placeholder for 2 image parts; a template that turns media parts into text has no image path: serve with --chat-template naming one that writes the image token", cx.message); + cx = .{ .a = a }; + try std.testing.expectError(error.Refused, images.expand(v, null, images.MarkText.text(), &cx, two, &[_]u32{ 1, 9 }, 0)); + try std.testing.expectEqualStrings("the chat template wrote 1 image placeholders for 2 image parts", cx.message); +}