diff --git a/apps/daemon/internal/agent/claudesdk/functions.go b/apps/daemon/internal/agent/claudesdk/functions.go index 752d1347f..a223c6161 100644 --- a/apps/daemon/internal/agent/claudesdk/functions.go +++ b/apps/daemon/internal/agent/claudesdk/functions.go @@ -104,14 +104,6 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio if err := result.ValidateContent(); err != nil { return err } - for _, part := range result.Content { - if part.Type == "input_image" && !result.Success { - return fmt.Errorf("claudesdk: native error results cannot retain images") - } - } - if err := (proto.MessageInput{{Content: result.Content}}).ValidateInlineImages(); err != nil { - return err - } data, err := json.Marshal(struct { Type string `json:"type"` TurnID string `json:"turn_id"` diff --git a/apps/daemon/internal/agent/claudesdk/functions_test.go b/apps/daemon/internal/agent/claudesdk/functions_test.go index d9c358b83..7b30bade0 100644 --- a/apps/daemon/internal/agent/claudesdk/functions_test.go +++ b/apps/daemon/internal/agent/claudesdk/functions_test.go @@ -51,11 +51,6 @@ func TestFunctionFactoryNativeReceipts(t *testing.T) { if err := submitter.SubmitFunctionResult(ctx, invalid); err == nil { t.Fatal("missing content consumed call") } - image := "https://example.invalid/image" - invalid.Content = []proto.InputContent{{Type: "input_image", ImageURL: &image}} - if err := submitter.SubmitFunctionResult(ctx, invalid); err == nil { - t.Fatal("image should fail before delivery") - } first, second := "first-"+call.CallID, "second-"+call.CallID value := proto.FunctionResultPayload{DeliveryID: "delivery-" + call.CallID, CallID: call.CallID, Success: call.CallID == "b", Content: []proto.InputContent{{Type: "input_text", Text: &first}, {Type: "input_text", Text: &second}}} go func() { submissions <- submitter.SubmitFunctionResult(ctx, value) }() diff --git a/contracts/agents-api/harness-capabilities.md b/contracts/agents-api/harness-capabilities.md index 5a755516a..040ff5cab 100644 --- a/contracts/agents-api/harness-capabilities.md +++ b/contracts/agents-api/harness-capabilities.md @@ -21,8 +21,7 @@ Placements are `none` (no Environment), hosted (`openai_hosted`) and self-hosted | Text Turns, active input, cancellation, restart and continuation | Verified on all placements | Verified on all placements | Verified on all placements | | [Files and Artifacts](./environment-files.md) | Verified: hosted, self-hosted | Verified: hosted, self-hosted | Verified: hosted, self-hosted | | [Whitespace-only message text](./message-content.md) | Admitted; delivered unchanged | Rejected | Rejected | -| [Inline PNG and JPEG message images](./message-content.md) | Verified on all placements | Verified on all placements | Rejected | -| Remote image URLs | Rejected | Rejected | Rejected | +| [Message images](./message-content.md#images) | PNG and JPEG data URLs: verified on all placements; other data URLs: admitted; remote URLs: rejected | PNG and JPEG data URLs: verified on all placements; remote URLs and GIF and WebP data URLs: admitted | Rejected | | `reasoning.effort` | `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`: admitted | `low`, `medium`, `high`, `xhigh`, `max`: admitted | Rejected | | `reasoning.summary` | `concise`, `detailed`, `auto`: admitted | Rejected | Rejected | | `service_tier` other than `auto` | `default`, `flex`, `priority`: admitted | Rejected | Rejected | @@ -36,7 +35,7 @@ Native model parameters and provider protocols per Harness are in [model executi | Operation | Codex | Claude Code | MiniMax Code | | --- | --- | --- | --- | | [Public functions](./execution-tools.md#functions) with text results | Verified: `none`, hosted; admitted: self-hosted | Verified on all placements; object-root schemas only | Rejected | -| [Function results with images](./message-content.md#function-results) | Verified: `none`, hosted; admitted: self-hosted | Verified on all placements; successful inline PNG or JPEG results only | Rejected | +| [Function results with images](./message-content.md#function-results) | Verified: `none`, hosted; admitted: self-hosted; remote URLs: rejected | Successful results only; PNG and JPEG data URLs: verified on all placements; GIF and WebP: admitted | Rejected | | [Structured output](./execution-tools.md#structured-output) | Admitted on all placements | Verified on all placements | Rejected | | [Deferred function discovery](./execution-tools.md#deferred-function-discovery) | Rejected | Verified: `none`, self-hosted; admitted: hosted | Rejected | | [Disabled web search and programmatic tool calling](./execution-tools.md#web-search-and-programmatic-tool-calling) | Verified: `none`; admitted: hosted, self-hosted | Verified: `none`; admitted: hosted, self-hosted | Verified: `none`; admitted: hosted, self-hosted | diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index 098bbe718..895563f4a 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -189,7 +189,7 @@ The declaration holds: - `Capabilities`, a `proto.AgentKindCapabilities` that states the most the Harness can do on a capable host, with every field `Supported` or `Unsupported` and the zero value invalid ([Capability declarations](../../docs/runtime-protocol.md#capability-declarations)); - the supported `Placements` and public `MCPOrigins`, and whether whitespace-only text is accepted; - `Execution`, the official execution option values the adapter maps to native settings: web search modes and whether search settings are accepted, reasoning efforts and summaries, service tiers, programmatic-tool-calling values and text verbosities. Core admission and the Runtime reject an undeclared value with the same check ([tool policy](./execution-tools.md#web-search-and-programmatic-tool-calling)). Core always sends the official default text verbosity, so `TextVerbosities` must include `medium`; -- optional pure restrictions, `ValidateConfiguration`, `ValidateTools` and `ValidateFunctionResult`, that add native limits to the common checks. A nil validator means the common checks suffice. They cannot query business data, decrypt credentials or control native processes, Configuration restrictions run first. With a `ValidateConfiguration` callback, tool-decoding errors precede `ValidateTools` errors; without one, `ValidateTools` errors precede a decoding error. +- optional pure restrictions, `ValidateConfiguration`, `ValidateTools`, `ValidateMessageInput` and `ValidateFunctionResult`, that add native limits to the common checks. `ValidateMessageInput` runs only for messages that carry images, after the `MessageImages` capability check and before anything is stored, and decides which `image_url` values the Harness accepts; Core checks only that the reference is nonempty. A nil validator means the common checks suffice. They cannot query business data, decrypt credentials or control native processes, Configuration restrictions run first. With a `ValidateConfiguration` callback, tool-decoding errors precede `ValidateTools` errors; without one, `ValidateTools` errors precede a decoding error. `harnessconfig.NewRegistry` validates every declaration and panics on an invalid or duplicate one: kinds and labels must be nonempty, placements must list at least one supported placement, MCP origins must be a non-nil list, unknown or duplicate choices and origins without a corresponding placement are rejected, and bearer support requires an MCP origin, and `Execution` must declare text verbosity `medium`. Shared dispatch checks capability combinations, never engine names. diff --git a/contracts/agents-api/index.md b/contracts/agents-api/index.md index 3d49c80f0..4b332cb35 100644 --- a/contracts/agents-api/index.md +++ b/contracts/agents-api/index.md @@ -28,7 +28,7 @@ Evidence for a status comes from the pinned official SDK and raw HTTP against th | --- | --- | --- | --- | | Agents | create, retrieve, update, list, delete | Implemented. Every pinned setting is saved; Session admission runs a subset | [Agents](./wire-semantics.md#agents) | | Sessions | create (JSON or stream), retrieve, update, list, delete | Implemented. Update takes `metadata` only; deletion requires an idle or failed Session | [Sessions](./wire-semantics.md#sessions), [creation streaming](./sessions-events.md#creation-streaming) | -| Session events | create, stream | Partial: messages with text and inline images, cancellation, function results; the stream is live only | [Sessions, events and history](./sessions-events.md), [message content](./message-content.md) | +| Session events | create, stream | Partial: messages with text and images, cancellation, function results; the stream is live only | [Sessions, events and history](./sessions-events.md), [message content](./message-content.md) | | Turns | retrieve, list | Implemented; Session Turn routes hold root Turns only | [Turns and Items](./sessions-events.md#turns-and-items) | | Items | list | Partial: messages, commands, MCP calls, functions, web search, reasoning and Subagent coordination Items; other native variants are not projected | [Turns and Items](./sessions-events.md#turns-and-items) | | Artifacts | retrieve, list, delete, content | Implemented | [Environment files and Artifacts](./environment-files.md) | @@ -120,7 +120,7 @@ Each item is Core's deliberate or native behavior where the official service beh - Pinned Codex can lose command output emitted before its stream subscription. - Claude Code and MiniMax Code report no public usage. - Core gives no crash-safe or exactly-once guarantee for native side effects; claimed work fails on restart without replay. -- Images must be inline PNG or JPEG data URIs; remote URLs, `file_id` and `detail` are rejected. +- `file_id` and `detail` on images are rejected; each Harness declares which `image_url` values it accepts ([message content](./message-content.md#images)). **Environments and Templates** diff --git a/contracts/agents-api/message-content.md b/contracts/agents-api/message-content.md index f46842055..892f366bd 100644 --- a/contracts/agents-api/message-content.md +++ b/contracts/agents-api/message-content.md @@ -17,12 +17,12 @@ An empty text part beside other content, such as `["", "text"]`, is accepted and ## Images -An `input_image` part carries `image_url` as an inline data URI: `data:image/png;base64,…` or `data:image/jpeg;base64,…`. The base64 must be canonical and the decoded image must match the declared type. Core accepts no remote URL, `file_id` or `detail`. +An `input_image` part carries a nonempty `image_url`, a remote URL or a base64 data URL as the official API allows. Core accepts no `file_id` or `detail` and otherwise checks only the Harness's declared rule below; it never downloads or rewrites an image, and the model provider's own error for an image it cannot read returns unchanged. | Harness | Message images | | --- | --- | -| Codex | Accepted on every placement the Harness supports: `none`, `openai_hosted` and `self_hosted` | -| Claude Code | Accepted on `none`, `openai_hosted` and `self_hosted` | +| Codex | Any data URL, passed unchanged, on `none`, `openai_hosted` and `self_hosted`. An `http` or `https` URL returns 400, because native Codex rejects remote image URLs | +| Claude Code | A reference that is not a data URL becomes a native URL source; a data URL must be canonical base64 of `image/png`, `image/jpeg`, `image/gif` or `image/webp`, and any other returns 400. On `none`, `openai_hosted` and `self_hosted` | | MiniMax Code | Rejected | Admission checks the Harness before anything is written; an image the Harness cannot take returns 400. The Runtime must also report message image support: Core binds a Session whose input carries images only to such a Runtime, and delivery to a Runtime without it fails. Use a model that accepts images. @@ -49,8 +49,8 @@ An `agent.session.input.tool_result` event carries `success`, an optional nullab | Harness | Function results | | --- | --- | -| Codex | Text and ordered text/image output. Core checks only that each part is well formed and passes image references to the Harness unchanged | -| Claude Code | Text output. Images only in successful results and only as inline PNG or JPEG; an image in a failed result or a remote reference returns 400 before anything is stored, and the pending call stays open. The Harness may resize or re-encode images in its own history; public Items keep the submitted bytes | +| Codex | Text and ordered text/image output, passed to the Harness unchanged. An `http` or `https` image URL returns 400, because native Codex rejects remote image URLs | +| Claude Code | Text output. Images only in successful results and only as canonical base64 data URLs of `image/png`, `image/jpeg`, `image/gif` or `image/webp`, because a native tool result carries no URL; an image in a failed result or a remote reference returns 400 before anything is stored, and the pending call stays open. The Harness may resize or re-encode an image, or replace one it cannot process with a text note, in its own history; public Items keep the submitted bytes | | MiniMax Code | No public functions | ### Application receipts diff --git a/contracts/agents-api/zh/harness-capabilities.md b/contracts/agents-api/zh/harness-capabilities.md index cef752996..24e8158c4 100644 --- a/contracts/agents-api/zh/harness-capabilities.md +++ b/contracts/agents-api/zh/harness-capabilities.md @@ -1,7 +1,7 @@ --- title: "Harness 能力" source: contracts/agents-api/harness-capabilities.md -source_hash: 818fecdaf4bca14f5d37b6135af4eed6c13b48c801e4dd1034317a86ef5a247c +source_hash: c31a323311129a5d005cec014003634c0e4e30ed824798cdda3ee10f2902efd9 --- 本页列出每个 Harness 在每种部署位置支持的能力。Core 根据 `internal/harnessconfig//declaration.go` 中 Harness 的声明决定准入，运行 Session 的 Runtime 也必须声明操作。所链接契约定义各操作；[Harness 接入](harness-onboarding.md#qualify-the-adapter)说明资格验证方法。 @@ -23,8 +23,7 @@ source_hash: 818fecdaf4bca14f5d37b6135af4eed6c13b48c801e4dd1034317a86ef5a247c | 文本 Turn、活动输入、取消、重启与继续 | 所有位置已验证 | 所有位置已验证 | 所有位置已验证 | | [文件与 Artifact](environment-files.md) | 已验证：托管、自托管 | 已验证：托管、自托管 | 已验证：托管、自托管 | | [仅空白消息文本](message-content.md) | 已准入；原样交付 | 已拒绝 | 已拒绝 | -| [内联 PNG、JPEG 消息图像](message-content.md) | 所有位置已验证 | 所有位置已验证 | 已拒绝 | -| 远程图像 URL | 已拒绝 | 已拒绝 | 已拒绝 | +| [消息图像](message-content.md#images) | PNG、JPEG data URL：所有位置已验证；其他 data URL：已准入；远程 URL：已拒绝 | PNG、JPEG data URL：所有位置已验证；远程 URL 及 GIF、WebP data URL：已准入 | 已拒绝 | | `reasoning.effort` | `none`、`minimal`、`low`、`medium`、`high`、`xhigh`、`max`：已准入 | `low`、`medium`、`high`、`xhigh`、`max`：已准入 | 已拒绝 | | `reasoning.summary` | `concise`、`detailed`、`auto`：已准入 | 已拒绝 | 已拒绝 | | 非 `auto` 的 `service_tier` | `default`、`flex`、`priority`：已准入 | 已拒绝 | 已拒绝 | @@ -38,7 +37,7 @@ source_hash: 818fecdaf4bca14f5d37b6135af4eed6c13b48c801e4dd1034317a86ef5a247c | 操作 | Codex | Claude Code | MiniMax Code | | --- | --- | --- | --- | | 文本结果的[公开函数](execution-tools.md#functions) | 已验证：`none`、托管；已准入：自托管 | 所有位置已验证；仅对象根 schema | 已拒绝 | -| [含图像函数结果](message-content.md#function-results) | 已验证：`none`、托管；已准入：自托管 | 所有位置已验证；仅成功结果中的内联 PNG 或 JPEG | 已拒绝 | +| [含图像函数结果](message-content.md#function-results) | 已验证：`none`、托管；已准入：自托管；远程 URL：已拒绝 | 仅成功结果；PNG、JPEG data URL：所有位置已验证；GIF、WebP：已准入 | 已拒绝 | | [结构化输出](execution-tools.md#structured-output) | 所有位置已准入 | 所有位置已验证 | 已拒绝 | | [延迟函数发现](execution-tools.md#deferred-function-discovery) | 已拒绝 | 已验证：`none`、自托管；已准入：托管 | 已拒绝 | | [禁用 Web 搜索与程序化工具调用](execution-tools.md#web-search-and-programmatic-tool-calling) | 已验证：`none`；已准入：托管、自托管 | 已验证：`none`；已准入：托管、自托管 | 已验证：`none`；已准入：托管、自托管 | diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index 999b63022..d9fff0af6 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "Harness 协议" source: contracts/agents-api/harness-onboarding.md -source_hash: 23717fd39df7a88a93bb12a7b4e68862f6ea7d514054d4765673df3b3ea98b96 +source_hash: 47fd08f050420acbf5c89ef886ddf3d9de7b3d43d28e4ffd1249f1550e1cbfef --- **Harness** 是一种运行模型和工具循环的原生代理引擎（Codex、Claude Code、MiniMax Code）；**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。Harness 协议是 Runtime 与适配器之间的契约：适配器接口及其生命周期义务、声明、注册和验收。本文面向添加或维护 Harness 的开发者。 @@ -191,7 +191,7 @@ Runtime 统一解析[状态目录](../../../docs/zh/configuration.md#runtime-res - `Capabilities`，即 `proto.AgentKindCapabilities`，表示 Harness 在能力完备的主机上最多能做什么；每个字段都是 `Supported` 或 `Unsupported`，零值无效（[能力声明](../../../docs/zh/runtime-protocol.md#capability-declarations)）； - 受支持的 `Placements` 和公共 `MCPOrigins`，以及是否接受仅含空白的文本； - `Execution`，即适配器映射到原生设置的官方执行选项取值：web search 模式及是否接受搜索设置、推理强度和摘要、服务层级、程序化工具调用取值和文本详细程度。Core 准入和 Runtime 用同一检查拒绝未声明的取值（[工具策略](execution-tools.md#web-search-and-programmatic-tool-calling)）。Core 总会发送官方默认文本详细程度，因此 `TextVerbosities` 必须包含 `medium`； -- 可选的纯限制 `ValidateConfiguration`、`ValidateTools` 和 `ValidateFunctionResult`，在通用检查之外添加原生限制。验证函数为 nil 表示通用检查已足够。它们不能查询业务数据、解密凭据或控制原生进程，配置限制最先执行。存在 `ValidateConfiguration` 回调时，工具解码错误先于 `ValidateTools` 错误；不存在时，`ValidateTools` 错误先于解码错误。 +- 可选的纯限制 `ValidateConfiguration`、`ValidateTools`、`ValidateMessageInput` 和 `ValidateFunctionResult`，在通用检查之外添加原生限制。`ValidateMessageInput` 仅对携带图像的消息运行，在 `MessageImages` 能力检查之后、任何存储之前执行，决定 Harness 接受哪些 `image_url`；Core 只检查引用非空。验证函数为 nil 表示通用检查已足够。它们不能查询业务数据、解密凭据或控制原生进程，配置限制最先执行。存在 `ValidateConfiguration` 回调时，工具解码错误先于 `ValidateTools` 错误；不存在时，`ValidateTools` 错误先于解码错误。 `harnessconfig.NewRegistry` 会验证每个声明，对无效或重复的声明触发 panic：kind 和 label 必须非空，放置位置必须至少列出一个受支持的位置，MCP 来源必须是非 nil 列表，未知或重复的选项以及没有对应放置位置的来源会被拒绝，bearer 支持要求存在 MCP 来源，`Execution` 必须声明文本详细程度 `medium`。共享调度检查能力组合，绝不检查引擎名称。 diff --git a/contracts/agents-api/zh/index.md b/contracts/agents-api/zh/index.md index e4ef3753d..8d61e5b99 100644 --- a/contracts/agents-api/zh/index.md +++ b/contracts/agents-api/zh/index.md @@ -1,7 +1,7 @@ --- title: "OpenAI 兼容性" source: contracts/agents-api/index.md -source_hash: 070ba5163d6a897e5fb12fcede93df423bd0d5a5ed6f36c48cadb3e67636b115 +source_hash: f91e92737399daba6f269772ac6ceb8427dbf89d9e6198ea4181ff7ac6d88ad5 --- Core 实现下方固定版本的 OpenAI Agents API。本台账列出各项资源的覆盖情况、相对于官方服务的所有差异以及已知缺口。 @@ -31,7 +31,7 @@ Core 实现下方固定版本的 OpenAI Agents API。本台账列出各项资源 | --- | --- | --- | --- | | Agents | create, retrieve, update, list, delete | 已实现。所有固定版本设置都会保存；Session 准入只会用到其中一个子集 | [Agents](wire-semantics.md#agents) | | Sessions | create (JSON 或流式), retrieve, update, list, delete | 已实现。更新仅接受 `metadata`；删除要求 Session 处于空闲或失败状态 | [Sessions](wire-semantics.md#sessions)、[creation streaming](sessions-events.md#creation-streaming) | -| Session events | create, stream | 部分实现：支持包含文本和内嵌图像的消息、取消和函数结果；流仅支持实时模式 | [Sessions, events and history](sessions-events.md)、[message content](message-content.md) | +| Session events | create, stream | 部分实现：支持包含文本和图像的消息、取消和函数结果；流仅支持实时模式 | [Sessions, events and history](sessions-events.md)、[message content](message-content.md) | | Turns | retrieve, list | 已实现；Session Turn 路由仅承载根级 Turns | [Turns and Items](sessions-events.md#turns-and-items) | | Items | list | 部分实现：支持消息、命令、MCP 调用、functions、web search、reasoning 和 Subagent 协调 Items；其他原生变体不会被投影 | [Turns and Items](sessions-events.md#turns-and-items) | | Artifacts | retrieve, list, delete, content | 已实现 | [Environment files and Artifacts](environment-files.md) | @@ -123,7 +123,7 @@ Core 自身字段位于 `x_agents_core` 中（[Core extensions](../../../docs/zh - 固定版本的 Codex 可能会丢失在订阅其流之前发出的命令输出。 - Claude Code 和 MiniMax Code 都不报告公共用量。 - 对于原生副作用，Core 不提供崩溃安全或恰好一次保证；已认领的工作若不重放，会在重启后失败。 -- 图像必须是内嵌的 PNG 或 JPEG data URI；远程 URL、`file_id` 和 `detail` 会被拒绝。 +- 图像上的 `file_id` 和 `detail` 会被拒绝；每个 Harness 声明自己接受哪些 `image_url`（[消息内容](message-content.md#images)）。 **Environments 和模板** diff --git a/contracts/agents-api/zh/message-content.md b/contracts/agents-api/zh/message-content.md index 0355a72d8..82538049b 100644 --- a/contracts/agents-api/zh/message-content.md +++ b/contracts/agents-api/zh/message-content.md @@ -1,7 +1,7 @@ --- title: "消息内容" source: contracts/agents-api/message-content.md -source_hash: 603ab95069d34f2deb277dbde9a74195a9e0e410966d88670eb1c9abbfa1bff7 +source_hash: f2f33d3397f0d82d9954808d0f494ddab9749ed69396d4ab0ea114a64cd3906c --- 用户消息和函数结果共享同一内容模型：由 `input_text` 与 `input_image` 部分组成的有序列表。Core 按发送形式准确存储消息边界、部分顺序和图像引用，并在用户 Item 中原样返回。不下载、转码或修复媒体。Session 创建的 `input` 与 `events.create` 消息共享验证和准入；[Session、事件与历史](sessions-events.md#send-input)定义准入、请求限制和错误。 @@ -19,12 +19,12 @@ source_hash: 603ab95069d34f2deb277dbde9a74195a9e0e410966d88670eb1c9abbfa1bff7 ## 图像 {#images} -`input_image` 部分通过内联 data URI 携带 `image_url`：`data:image/png;base64,…` 或 `data:image/jpeg;base64,…`。base64 必须为规范形式，解码图像必须匹配声明类型。Core 不接受远程 URL、`file_id` 或 `detail`。 +`input_image` 部分携带非空的 `image_url`，即官方 API 允许的远程 URL 或 base64 data URL。Core 不接受 `file_id` 或 `detail`，其余只检查下表中 Harness 声明的规则；Core 从不下载或改写图像，模型提供商对无法读取的图像返回的错误原样返回。 | Harness | 消息图像 | | --- | --- | -| Codex | 在 Harness 支持的所有位置接受：`none`、`openai_hosted`、`self_hosted` | -| Claude Code | 在 `none`、`openai_hosted`、`self_hosted` 接受 | +| Codex | 接受任意 data URL 并原样传递，适用于 `none`、`openai_hosted`、`self_hosted`。`http` 或 `https` URL 返回 400，因为原生 Codex 拒绝远程图像 URL | +| Claude Code | 非 data URL 的引用成为原生 URL 来源；data URL 必须是规范 base64 的 `image/png`、`image/jpeg`、`image/gif` 或 `image/webp`，其他返回 400。适用于 `none`、`openai_hosted`、`self_hosted` | | MiniMax Code | 拒绝 | 任何写入之前，准入检查 Harness；Harness 无法接受的图像返回 400。Runtime 也必须报告消息图像支持：Core 仅将输入含图像的 Session 绑定到此类 Runtime，交付给不支持的 Runtime 会失败。应使用接受图像的模型。 @@ -51,8 +51,8 @@ source_hash: 603ab95069d34f2deb277dbde9a74195a9e0e410966d88670eb1c9abbfa1bff7 | Harness | 函数结果 | | --- | --- | -| Codex | 文本与有序文本/图像输出。Core 仅检查各部分格式正确，并将图像引用原样传给 Harness | -| Claude Code | 文本输出。仅成功结果可含内联 PNG 或 JPEG 图像；失败结果图像或远程引用在任何存储前返回 400，待处理调用保持开放。Harness 可在原生历史中调整图像尺寸或重新编码；公开 Item 保留提交字节 | +| Codex | 文本与有序文本/图像输出，原样传给 Harness。`http` 或 `https` 图像 URL 返回 400，因为原生 Codex 拒绝远程图像 URL | +| Claude Code | 文本输出。仅成功结果可含图像，且只能是规范 base64 的 `image/png`、`image/jpeg`、`image/gif`、`image/webp` data URL，因为原生工具结果不携带 URL；失败结果图像或远程引用在任何存储前返回 400，待处理调用保持开放。Harness 可在原生历史中调整图像尺寸或重新编码，或将无法处理的图像替换为文本说明；公开 Item 保留提交字节 | | MiniMax Code | 无公开函数 | ### 应用回执 {#application-receipts} diff --git a/docs/api/public-agent-api.md b/docs/api/public-agent-api.md index 8bda176b3..ca4648b1e 100644 --- a/docs/api/public-agent-api.md +++ b/docs/api/public-agent-api.md @@ -312,7 +312,7 @@ agents "/agents/sessions/$SESSION_ID/events" -H "Idempotency-Key: $KEY" -d '{ ::: - **When idle,** a message starts a new Turn. **While a Turn runs,** it joins that Turn (steering); it does not start a parallel task. -- **Content** is `input_text`, plus `input_image` as an inline PNG or JPEG data URI. The whole request is limited to 1 MiB. +- **Content** is `input_text`, plus `input_image` with an `image_url`; the Harness decides which references it accepts. The whole request is limited to 1 MiB. - Full rules: [Message content](../../contracts/agents-api/message-content.md). ### Cancel diff --git a/docs/zh/api/public-agent-api.md b/docs/zh/api/public-agent-api.md index 8f535efdf..ceb337941 100644 --- a/docs/zh/api/public-agent-api.md +++ b/docs/zh/api/public-agent-api.md @@ -1,7 +1,7 @@ --- title: "Agents API 指南" source: docs/api/public-agent-api.md -source_hash: 926187936e4c0aa8ac5298a1156463a15e893add8df2e4665bbc813204e4a6cd +source_hash: 47c626b6be9a45d18e566d17556321a484356e825e0d5f0dd07385ccd6e50496 --- Core 在 `/v1` 提供 OpenAI Agents API。本指南用官方 OpenAI SDK 和普通 HTTP 两种方式演示每项常见任务，并说明 Core 与 OpenAI 的差异。如果尚未启动过 Session，请先阅读[运行第一个 Session](../getting-started/quickstart.md)。 @@ -314,7 +314,7 @@ agents "/agents/sessions/$SESSION_ID/events" -H "Idempotency-Key: $KEY" -d '{ ::: - **处于空闲状态时，**消息会启动一个新 Turn。**Turn 运行时，**消息会加入该 Turn（进行引导），而不会启动并行任务。 -- **内容**使用 `input_text`，也可以使用 `input_image` 并以内联 PNG 或 JPEG 数据 URI 提供。整个请求限制为 1 MiB。 +- **内容**使用 `input_text`，也可以使用带 `image_url` 的 `input_image`，由 Harness 决定接受哪些引用。整个请求限制为 1 MiB。 - 完整规则请参阅[消息内容](../../../contracts/agents-api/zh/message-content.md)。 ### 取消 {#cancel} diff --git a/internal/agentdaemon/proto/message_images.go b/internal/agentdaemon/proto/message_images.go deleted file mode 100644 index 233c472de..000000000 --- a/internal/agentdaemon/proto/message_images.go +++ /dev/null @@ -1,39 +0,0 @@ -package proto - -import ( - "bytes" - "encoding/base64" - "errors" - "image" - _ "image/jpeg" - _ "image/png" - "strings" -) - -// ValidateInlineImages checks inline PNG/JPEG content without downloading or -// rewriting it. Callers separately qualify message/function-result support. -func (m MessageInput) ValidateInlineImages() error { - for _, message := range m { - for _, part := range message.Content { - if part.Type != "input_image" { - continue - } - if part.ImageURL == nil { - return errors.New("image input requires a reference") - } - header, encoded, ok := strings.Cut(*part.ImageURL, ",") - if !ok || (header != "data:image/png;base64" && header != "data:image/jpeg;base64") { - return errors.New("image requires inline PNG or JPEG") - } - data, err := base64.StdEncoding.Strict().DecodeString(encoded) - if err != nil || base64.StdEncoding.EncodeToString(data) != encoded { - return errors.New("image requires valid base64") - } - _, format, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil || header != "data:image/"+format+";base64" { - return errors.New("image format does not match its media type") - } - } - } - return nil -} diff --git a/internal/agentdaemon/proto/message_images_test.go b/internal/agentdaemon/proto/message_images_test.go deleted file mode 100644 index 4da4eb9a3..000000000 --- a/internal/agentdaemon/proto/message_images_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package proto - -import ( - "bytes" - "encoding/base64" - "image" - "image/jpeg" - "image/png" - "testing" -) - -func TestInlineMessageImages(t *testing.T) { - for _, format := range []string{"png", "jpeg"} { - var encoded bytes.Buffer - picture := image.NewRGBA(image.Rect(0, 0, 2, 2)) - if format == "png" { - _ = png.Encode(&encoded, picture) - } else { - _ = jpeg.Encode(&encoded, picture, nil) - } - url := "data:image/" + format + ";base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) - input := MessageInput{{Content: []InputContent{{Type: "input_image", ImageURL: &url}}}} - if err := input.ValidateInlineImages(); err != nil { - t.Fatal(format, err) - } - wrongType := "data:image/gif;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()) - input[0].Content[0].ImageURL = &wrongType - if input.ValidateInlineImages() == nil { - t.Fatal("accepted unsupported image type") - } - } - for _, url := range []string{"https://example.test/image.png", "file:///private/image.png", "data:image/png;base64,AA==", "data:image/png;base64,!", "data:image/png;base64,"} { - input := MessageInput{{Content: []InputContent{{Type: "input_image", ImageURL: &url}}}} - if input.ValidateInlineImages() == nil { - t.Fatalf("accepted %q", url) - } - // User-message qualification must not narrow the function-output union. - if input.Validate() != nil { - t.Fatalf("inline profile leaked into the shared content union: %q", url) - } - } - if err := TextInput("ordinary text").ValidateInlineImages(); err != nil { - t.Fatal(err) - } -} diff --git a/internal/harnessconfig/claudesdk/declaration.go b/internal/harnessconfig/claudesdk/declaration.go index 66f50403d..b4260573f 100644 --- a/internal/harnessconfig/claudesdk/declaration.go +++ b/internal/harnessconfig/claudesdk/declaration.go @@ -2,6 +2,7 @@ package claudesdk import ( + "encoding/base64" "encoding/json" "errors" "math" @@ -67,6 +68,7 @@ var Declaration = harnessconfig.Declaration{ WhitespaceOnlyText: proto.CapabilityUnsupported, ValidateConfiguration: validateConfiguration, ValidateTools: validateTools, + ValidateMessageInput: validateMessageInput, ValidateFunctionResult: validateFunctionResult, } @@ -153,16 +155,36 @@ func validateTools(tools []proto.FunctionTool, servers []proto.MCPHTTPServer) er func validateFunctionResult(result proto.FunctionResultPayload) error { for _, part := range result.Content { - if part.Type == "input_image" && !result.Success { + if part.Type == "input_image" && (!result.Success || !inlineImage(*part.ImageURL)) { return harnessconfig.ErrInvalidInput } } - if (proto.MessageInput{{Content: result.Content}}).ValidateInlineImages() != nil { - return harnessconfig.ErrInvalidInput + return nil +} + +// A user message image that is not a data URL becomes a native URL source. +func validateMessageInput(input proto.MessageInput) error { + for _, message := range input { + for _, part := range message.Content { + if part.Type == "input_image" && strings.HasPrefix(*part.ImageURL, "data:") && !inlineImage(*part.ImageURL) { + return harnessconfig.ErrInvalidInput + } + } } return nil } +// inlineImage reports canonical base64 in a media type the pinned SDK declares. +func inlineImage(url string) bool { + header, encoded, _ := strings.Cut(url, ",") + data, err := base64.StdEncoding.DecodeString(encoded) + switch header { + case "data:image/png;base64", "data:image/jpeg;base64", "data:image/gif;base64", "data:image/webp;base64": + return err == nil && encoded != "" && base64.StdEncoding.EncodeToString(data) == encoded + } + return false +} + func validateNativeConfig(config proto.HarnessConfig) bool { for key, value := range config { switch key { diff --git a/internal/harnessconfig/codex/declaration.go b/internal/harnessconfig/codex/declaration.go index 484be6fe3..52acb1864 100644 --- a/internal/harnessconfig/codex/declaration.go +++ b/internal/harnessconfig/codex/declaration.go @@ -4,6 +4,7 @@ package codex import ( "encoding/json" "errors" + "strings" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig" @@ -60,9 +61,13 @@ var Declaration = harnessconfig.Declaration{ ProgrammaticToolCalling: []bool{true, false}, TextVerbosities: []string{"low", "medium", "high"}, }, - Placements: []string{"none", "self_hosted", "openai_hosted"}, - MCPOrigins: []string{"service", "environment"}, - WhitespaceOnlyText: proto.CapabilitySupported, + Placements: []string{"none", "self_hosted", "openai_hosted"}, + MCPOrigins: []string{"service", "environment"}, + WhitespaceOnlyText: proto.CapabilitySupported, + ValidateMessageInput: rejectRemoteImages, + ValidateFunctionResult: func(result proto.FunctionResultPayload) error { + return rejectRemoteImages(proto.MessageInput{{Content: result.Content}}) + }, ValidateConfiguration: func(admission harnessconfig.Admission) error { if err := harnessconfig.RejectSubagentTools(admission, "function", "mcp"); err != nil { return err @@ -75,6 +80,21 @@ var Declaration = harnessconfig.Declaration{ }, } +// Native Codex rejects http and https image URLs in turns and tool results. +func rejectRemoteImages(input proto.MessageInput) error { + for _, message := range input { + for _, part := range message.Content { + if part.ImageURL == nil { + continue + } + if scheme, _, ok := strings.Cut(*part.ImageURL, ":"); ok && (strings.EqualFold(scheme, "http") || strings.EqualFold(scheme, "https")) { + return errors.New("remote image URLs are not supported; use an inline data URL instead") + } + } + } + return nil +} + func enablesProgrammaticToolCalling(tools []json.RawMessage) bool { for _, raw := range tools { var tool struct { diff --git a/internal/harnessconfig/harness.go b/internal/harnessconfig/harness.go index 2137d1cb9..7a01bbb84 100644 --- a/internal/harnessconfig/harness.go +++ b/internal/harnessconfig/harness.go @@ -93,6 +93,8 @@ type Declaration struct { ValidateConfiguration func(Admission) error // ValidateTools adds restrictions on resolved function and MCP tools. ValidateTools func([]proto.FunctionTool, []proto.MCPHTTPServer) error + // ValidateMessageInput adds restrictions on messages that carry images. + ValidateMessageInput func(proto.MessageInput) error // ValidateFunctionResult adds restrictions on submitted function results. ValidateFunctionResult func(proto.FunctionResultPayload) error } diff --git a/packages/claude-sdk-adapter/README.md b/packages/claude-sdk-adapter/README.md index d06a16da1..8d96e797b 100644 --- a/packages/claude-sdk-adapter/README.md +++ b/packages/claude-sdk-adapter/README.md @@ -74,7 +74,7 @@ The SDK descriptor advertises the validated daemon subset, including durable Tur Core owns public admission for Claude: [harness selection](../../contracts/agents-api/model-execution.md#harness-selection) chooses the engine for each Session; one [engine policy](../../contracts/agents-api/harness-onboarding.md#declare-the-harness) serves API admission, device selection and the final claim; the [supported operations](../../contracts/agents-api/harness-capabilities.md) table records Claude's medium-only verbosity and object-root function schemas; [function result images](../../contracts/agents-api/message-content.md#function-results) defines which placements accept image results; and [deployment defaults](../../contracts/agents-api/model-execution.md#deployment-defaults) define the model provider Core freezes for a Session. The adapter receives that provider as the adapter-owned `model_provider`, never in public Session configuration. Without one, a `none` host uses the daemon's own provider environment. The adapter alone selects the provider environment and removes credentials from native tool environments. -The `none` profile accepts only text, explicit model/system instructions, managed state, exact native resume and declared functions with ordered text or successful inline PNG/JPEG results, and the HTTP MCP subset described above. It rejects unsupported request options and disables built-in tools and undeclared MCP discovery. `DisableExecutionEnvironment` and `DisableSubagents` are accepted assertions about the single-Agent restrictive profile. Omission does not enable built-in tools. Explicit Subagent observation enables only the native delegation tools described under [Subagents](#subagents). Single-Agent new and resumed queries use the SDK's empty built-in tool set, explicit function MCP configuration and allowlist, strict MCP configuration and empty user/project/local setting sources. Without HTTP MCP declarations, native initialization and real provider request inventories must contain only the declared host functions. Managed operator policy may further restrict execution; it must not widen the profile. The profile limits model tool access; it does not isolate native state files or filesystem access by an explicitly supplied host function. The private factory accepts the execution controls in the [Claude declaration](../../internal/harnessconfig/claudesdk/declaration.go): disabled search, medium text verbosity and reasoning effort, which maps to the SDK's `effort` option. Search remains excluded by the native tool inventory; medium retains the SDK's default text generation, without adding instructions or changing caller input. The pinned SDK has no verbosity, summary or service-tier option. An undeclared value fails before native setup; omitting the block keeps the same restrictive profile. Use the SDK's history lookup before explicit resume; never fall back to a new Session. Native files remain device-affine under a caller-selected managed runtime directory. The launch configuration supplies trusted provider environment; request options cannot supply environment variables or business write authority. Omitted, null and empty `system_prompt` map to empty SDK instructions only at this adapter boundary; null model values and unsupported options remain rejected. +The `none` profile accepts only text, explicit model/system instructions, managed state, exact native resume and declared functions with ordered text or successful inline PNG, JPEG, GIF or WebP results, and the HTTP MCP subset described above. It rejects unsupported request options and disables built-in tools and undeclared MCP discovery. `DisableExecutionEnvironment` and `DisableSubagents` are accepted assertions about the single-Agent restrictive profile. Omission does not enable built-in tools. Explicit Subagent observation enables only the native delegation tools described under [Subagents](#subagents). Single-Agent new and resumed queries use the SDK's empty built-in tool set, explicit function MCP configuration and allowlist, strict MCP configuration and empty user/project/local setting sources. Without HTTP MCP declarations, native initialization and real provider request inventories must contain only the declared host functions. Managed operator policy may further restrict execution; it must not widen the profile. The profile limits model tool access; it does not isolate native state files or filesystem access by an explicitly supplied host function. The private factory accepts the execution controls in the [Claude declaration](../../internal/harnessconfig/claudesdk/declaration.go): disabled search, medium text verbosity and reasoning effort, which maps to the SDK's `effort` option. Search remains excluded by the native tool inventory; medium retains the SDK's default text generation, without adding instructions or changing caller input. The pinned SDK has no verbosity, summary or service-tier option. An undeclared value fails before native setup; omitting the block keeps the same restrictive profile. Use the SDK's history lookup before explicit resume; never fall back to a new Session. Native files remain device-affine under a caller-selected managed runtime directory. The launch configuration supplies trusted provider environment; request options cannot supply environment variables or business write authority. Omitted, null and empty `system_prompt` map to empty SDK instructions only at this adapter boundary; null model values and unsupported options remain rejected. ### Function server and results diff --git a/packages/claude-sdk-adapter/src/function_bridge.ts b/packages/claude-sdk-adapter/src/function_bridge.ts index 777d9ea15..eaae027d9 100644 --- a/packages/claude-sdk-adapter/src/function_bridge.ts +++ b/packages/claude-sdk-adapter/src/function_bridge.ts @@ -122,18 +122,12 @@ export class FunctionBridge { } // A matching root call confirms incorporation into native history. Native image -// decoding/resizing may change bytes; preserve text and ordered image positions. +// processing may change bytes or replace an unprocessable image with a text note; +// preserve text and ordered image positions. function matchesNativeContent(content: InputContent[], actual: unknown): boolean { const expected = nativeContent({ content }); if (!Array.isArray(expected) || !Array.isArray(actual) || actual.length !== expected.length) return false; - return expected.every((part, index) => { - const received = actual[index]; - if (part.type !== "image") return isDeepStrictEqual(part, received); - if (!received || received.type !== "image" || received.source?.type !== "base64" || - typeof received.source.media_type !== "string" || typeof received.source.data !== "string") return false; - try { - imageSource(`data:${received.source.media_type};base64,${received.source.data}`); - return true; - } catch { return false; } - }); + return expected.every((part, index) => part.type === "image" + ? actual[index]?.type === "image" || actual[index]?.type === "text" + : isDeepStrictEqual(part, actual[index])); } diff --git a/packages/claude-sdk-adapter/src/inputs.ts b/packages/claude-sdk-adapter/src/inputs.ts index f2effbffe..6cd9775b8 100644 --- a/packages/claude-sdk-adapter/src/inputs.ts +++ b/packages/claude-sdk-adapter/src/inputs.ts @@ -57,7 +57,7 @@ export class Inputs implements AsyncIterable { private enqueue(messages: MessageInput, id?: string): void { // Convert the entire batch before admitting any native input. - const contents = messages.map(nativeContent); + const contents = messages.map(message => nativeContent(message, true)); const batch: Batch = { id, unapplied: contents.length }; for (const content of contents) { const uuid = randomUUID(); diff --git a/packages/claude-sdk-adapter/src/message_input.ts b/packages/claude-sdk-adapter/src/message_input.ts index fc5454b37..52f1ee15b 100644 --- a/packages/claude-sdk-adapter/src/message_input.ts +++ b/packages/claude-sdk-adapter/src/message_input.ts @@ -20,7 +20,7 @@ export function parseMessageInput(value: unknown): MessageInput { for (const message of value) { if (!message || typeof message !== "object" || Object.keys(message).some(key => key !== "content") || !Array.isArray(message.content) || !message.content.length) throw new Error("Invalid user message."); - const content = parseInputContent(message.content); + const content = parseInputContent(message.content, true); const meaningful = content.some(part => part.type === "input_image" || !isBlankText(part.text)); if (!meaningful) throw new EmptyUserMessageError(); } @@ -28,7 +28,8 @@ export function parseMessageInput(value: unknown): MessageInput { } // Function results share content validation but permit empty arrays and empty text. -export function parseInputContent(content: unknown): InputContent[] { +// Only a user message image that is not a data URL becomes a native URL source. +export function parseInputContent(content: unknown, remote = false): InputContent[] { if (!Array.isArray(content)) throw new Error("Invalid input content."); for (const part of content) { if (!part || typeof part !== "object") throw new Error("Invalid user content."); @@ -36,22 +37,24 @@ export function parseInputContent(content: unknown): InputContent[] { continue; } else if (part.type === "input_image" && typeof part.image_url === "string" && Object.keys(part).every(key => key === "type" || key === "image_url")) { - imageSource(part.image_url); + if (!remote || part.image_url.startsWith("data:")) imageSource(part.image_url); } else throw new Error("Invalid user content."); } return content as InputContent[]; } -export function imageSource(url: string): { type: "base64"; media_type: "image/png" | "image/jpeg"; data: string } { - const match = /^data:(image\/png|image\/jpeg);base64,([A-Za-z0-9+/]+={0,2})$/.exec(url); +type MediaType = "image/png" | "image/jpeg" | "image/gif" | "image/webp"; + +export function imageSource(url: string): { type: "base64"; media_type: MediaType; data: string } { + const match = /^data:(image\/(?:png|jpeg|gif|webp));base64,([A-Za-z0-9+/]+={0,2})$/.exec(url); if (!match || Buffer.from(match[2]!, "base64").toString("base64") !== match[2]) throw new Error("Unsupported image reference."); - return { type: "base64", media_type: match[1] as "image/png" | "image/jpeg", data: match[2]! }; + return { type: "base64", media_type: match[1] as MediaType, data: match[2]! }; } -export function nativeContent(message: MessageInput[number]): SDKUserMessage["message"]["content"] { +export function nativeContent(message: MessageInput[number], remote = false): SDKUserMessage["message"]["content"] { return message.content.map(part => part.type === "input_text" ? { type: "text" as const, text: part.text } - : { type: "image" as const, source: imageSource(part.image_url) }); + : { type: "image" as const, source: remote && !part.image_url.startsWith("data:") ? { type: "url" as const, url: part.image_url } : imageSource(part.image_url) }); } export function textInput(text: string): MessageInput { return [{ content: [{ type: "input_text", text }] }]; } diff --git a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs index bd78b0e27..60cdd2bfa 100644 --- a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs +++ b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs @@ -89,10 +89,7 @@ test("successful image results confirm native preprocessing with unchanged ident { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "BAUG" } }, { type: "text", text: "after" }, ]; - for (const bad of [converted.slice(1), [...converted, converted[0]], - [converted[0], { type: "text", text: "Image could not be processed" }, converted[2]], - [converted[2], converted[1], converted[0]], - [converted[0], { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "!!" } }, converted[2]]]) { + for (const bad of [converted.slice(1), [...converted, converted[0]], [converted[2], converted[1], converted[0]]]) { await assert.rejects(bridge.consume(native(value, bad), "native"), /differs/); } await bridge.consume({ ...native(value, converted), isSynthetic: true }, "native"); diff --git a/packages/claude-sdk-adapter/tests/inputs.test.mjs b/packages/claude-sdk-adapter/tests/inputs.test.mjs index ce664b6b9..c2a48c12d 100644 --- a/packages/claude-sdk-adapter/tests/inputs.test.mjs +++ b/packages/claude-sdk-adapter/tests/inputs.test.mjs @@ -121,3 +121,18 @@ test("report whitespace-only steering as a rejected input", () => { assert.throws(() => inputs.submit({ type: "steer", input_id: "no-content", input: [{ content: [] }] }), /Invalid/); inputs.close(); }); + +test("a user message maps a remote URL and any native base64 type; a tool result takes base64 only", async () => { + const { nativeContent, parseInputContent } = await import("../dist/message_input.js"); + const content = [ + { type: "input_image", image_url: "https://example.com/a.webp" }, + { type: "input_image", image_url: "data:image/gif;base64,AQID" }, + ]; + assert.deepEqual(nativeContent({ content }, true), [ + { type: "image", source: { type: "url", url: "https://example.com/a.webp" } }, + { type: "image", source: { type: "base64", media_type: "image/gif", data: "AQID" } }, + ]); + assert.throws(() => parseInputContent([content[0]]), /Unsupported image/); + assert.throws(() => parseInputContent([{ type: "input_image", image_url: "data:image/bmp;base64,AQID" }], true), /Unsupported image/); + assert.equal(parseInputContent([content[1]]).length, 1); +}); diff --git a/services/core/internal/api/environment_creation_test.go b/services/core/internal/api/environment_creation_test.go index e9f03cd95..13cc1db50 100644 --- a/services/core/internal/api/environment_creation_test.go +++ b/services/core/internal/api/environment_creation_test.go @@ -169,7 +169,6 @@ func TestSelfHostedCreationRejectsBeforePersistence(t *testing.T) { {name: "scalar capabilities", environment: `{"type":"self_hosted","workspace_directory":"/workspace","capability_directories":"/remote/skills"}`}, {name: "output field", environment: `{"type":"self_hosted","workspace_directory":"/workspace","remote_url":"https://forged.example"}`}, {name: "none extra field", environment: `{"type":"none","workspace_directory":null}`}, - {name: "initial image", environment: validEnvironment, input: `,"input":[{"role":"user","content":[{"type":"input_image","image_url":"https://example.com/image.png"}]}]`}, {name: "initial assistant message", environment: validEnvironment, input: `,"input":[{"role":"assistant","content":[{"type":"input_text","text":"start"}]}]`}, {name: "deferred functions", environment: validEnvironment, agentFields: `,"tools":[{"type":"function","name":"lookup","description":"Find a value","parameters":{"type":"object"},"defer_loading":true}]`}, {name: "unregistered harness placement", environment: validEnvironment, engine: "unregistered"}, diff --git a/services/core/internal/api/inputs.go b/services/core/internal/api/inputs.go index 8d300e5ed..8f38a4404 100644 --- a/services/core/internal/api/inputs.go +++ b/services/core/internal/api/inputs.go @@ -110,7 +110,7 @@ func executionInputs(events []json.RawMessage) ([]sessions.Input, error) { for _, content := range message.Content { converted[0].Content = append(converted[0].Content, proto.InputContent{Type: content.Type, Text: content.Text, ImageURL: content.ImageURL}) } - if converted.Validate() != nil || converted.ValidateInlineImages() != nil { + if converted.Validate() != nil { return nil, sessions.ErrInvalidInput } } diff --git a/services/core/internal/api/inputs_test.go b/services/core/internal/api/inputs_test.go index f6cb61e63..2706d5df2 100644 --- a/services/core/internal/api/inputs_test.go +++ b/services/core/internal/api/inputs_test.go @@ -62,7 +62,7 @@ func TestPublicInputRejectsUnsupportedOrMalformedBatch(t *testing.T) { `{"events":[{"type":"agent.session.input.message","input":[{"type":null,"role":"user","content":[{"type":"input_text","text":"x"}]}]}]}`, `null`, `{}`, `{"events":null}`, `{"events":[{"type":"agent.session.input.tool_result","call_id":"x"}]}`, `{"events":[{"type":"agent.session.input.message","input":[{"role":"assistant","content":[{"type":"input_text","text":"x"}]}]}]}`, - `{"events":[{"type":"agent.session.input.message","input":[{"role":"user","content":[{"type":"input_image","image_url":"https://example.com/a.png"}]}]}]}`, + `{"events":[{"type":"agent.session.input.message","input":[{"role":"user","content":[{"type":"input_image","image_url":" "}]}]}]}`, `{"events":[{"type":"agent.session.input.cancel","input":[]}]}`, `{"events":[{"type":"agent.session.input.message","input":[{"role":"user","content":[{"type":"input_text","text":""}]}]}]}`, `{"events":[{"type":"agent.session.input.cancel"}]} {}`, diff --git a/services/core/internal/execution/function_images_test.go b/services/core/internal/execution/function_images_test.go index 500bdd853..b96953cd3 100644 --- a/services/core/internal/execution/function_images_test.go +++ b/services/core/internal/execution/function_images_test.go @@ -2,7 +2,6 @@ package execution import ( "encoding/json" - "errors" "testing" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" @@ -13,35 +12,28 @@ import ( func TestFunctionImageAdmission(t *testing.T) { image := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=" - profile, _ := builtin.Registry().Lookup("claude_sdk") for _, tc := range []struct { - name, url string - success, valid bool + kind, name, url string + success, valid bool }{ - {"success", image, true, true}, - {"native error drops images", image, false, false}, - {"remote", "https://example.test/image.png", true, false}, - {"invalid base64", "data:image/png;base64,?", true, false}, - {"invalid image", "data:image/png;base64,AQID", true, false}, + {"claude_sdk", "success", image, true, true}, + {"claude_sdk", "native error drops images", image, false, false}, + {"claude_sdk", "remote", "https://example.test/image.png", true, false}, + {"claude_sdk", "invalid base64", "data:image/png;base64,?", true, false}, + {"claude_sdk", "unsupported type", "data:image/bmp;base64,AQID", true, false}, + {"claude_sdk", "webp", "data:image/webp;base64,AQID", true, true}, + {"codex", "inline", image, true, true}, + {"codex", "remote", "HTTPS://example.test/image.png", true, false}, } { - t.Run(tc.name, func(t *testing.T) { + t.Run(tc.kind+" "+tc.name, func(t *testing.T) { + profile, _ := builtin.Registry().Lookup(tc.kind) output := []any{map[string]any{"type": "input_text", "text": "before"}, map[string]any{"type": "input_image", "image_url": tc.url}, map[string]any{"type": "input_text", "text": "after"}} raw, _ := json.Marshal(map[string]any{"call_id": "call", "result": map[string]any{"success": tc.success, "output": output}}) - err := validateDeclaredInputs(profile, []sessions.Input{{Kind: "tool_result", Payload: raw}}) - if tc.valid && err != nil || !tc.valid && !errors.Is(err, sessions.ErrInvalidInput) { + if err := validateDeclaredInputs(profile, []sessions.Input{{Kind: "tool_result", Payload: raw}}); (err == nil) != tc.valid { t.Fatal(err) } }) } - for _, placement := range []string{"self_hosted"} { - url := image - if err := profile.ValidateFunctionResult(proto.FunctionResultPayload{Success: true, Content: []proto.InputContent{{Type: "input_image", ImageURL: &url}}}); err != nil { - t.Fatal("qualified image rejected", placement, err) - } - if err := profile.ValidateFunctionResult(proto.FunctionResultPayload{Success: true, Content: proto.TextInput("text")[0].Content}); err != nil { - t.Fatal("workspace text regressed", err) - } - } // Readiness for image-bearing results cannot become a global function gate. if err := requireFunctionResultImages(nil, "unknown", proto.FunctionResultPayload{Content: proto.TextInput("text")[0].Content}); err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/message_images_test.go b/services/core/internal/execution/message_images_test.go new file mode 100644 index 000000000..0b3e71ae4 --- /dev/null +++ b/services/core/internal/execution/message_images_test.go @@ -0,0 +1,36 @@ +package execution + +import ( + "encoding/json" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig/builtin" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" +) + +// Each Harness declares its own image rule: Codex rejects remote URLs as +// native does, Claude Code admits what its native blocks express, and a Harness +// without image input rejects every image. +func TestMessageImageRulesBelongToTheHarness(t *testing.T) { + for _, tc := range []struct { + kind, url string + want bool + }{ + {"codex", "Https://example.com/a.webp", false}, + {"codex", "data:image/bmp;base64,AQID", true}, + {"claude_sdk", "https://example.com/a.webp", true}, + {"claude_sdk", "data:image/gif;base64,AQID", true}, + {"claude_sdk", "data:image/bmp;base64,AQID", false}, + {"claude_sdk", "ftp://example.com/a.png", true}, + {"claude_sdk", "data:image/png;base64,?", false}, + {"mcode", "https://example.com/a.webp", false}, + } { + profile, _ := builtin.Registry().Lookup(tc.kind) + payload, _ := json.Marshal(map[string]any{"input": []any{map[string]any{"role": "user", "content": []any{map[string]any{"type": "input_image", "image_url": tc.url}}}}}) + err := validateDeclaredInputs(profile, []sessions.Input{{Kind: "message", Payload: payload}}) + if (err == nil) != tc.want { + t.Fatalf("%s %s: %v", tc.kind, tc.url, err) + } + } +} + diff --git a/services/core/internal/execution/message_support.go b/services/core/internal/execution/message_support.go index 7867f35b6..7777af171 100644 --- a/services/core/internal/execution/message_support.go +++ b/services/core/internal/execution/message_support.go @@ -50,9 +50,12 @@ func validateMessageImages(declaration harnessconfig.Declaration, input proto.Me if !input.HasImages() { return nil } - if !declaration.Capabilities.MessageImages.IsSupported() || input.ValidateInlineImages() != nil { + if !declaration.Capabilities.MessageImages.IsSupported() { return sessions.ErrInvalidInput } + if declaration.ValidateMessageInput != nil { + return admissionError(declaration.ValidateMessageInput(input)) + } return nil } diff --git a/services/core/internal/execution/workspace_images_test.go b/services/core/internal/execution/workspace_images_test.go deleted file mode 100644 index 0044568d1..000000000 --- a/services/core/internal/execution/workspace_images_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package execution - -import ( - "encoding/json" - "errors" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/internal/harnessconfig/builtin" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" -) - -func TestImageQualificationIsIndependentOfEnvironmentSource(t *testing.T) { - image := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aXioAAAAASUVORK5CYII=" - message := sessions.Input{Kind: "message", Payload: json.RawMessage(`{"input":[{"role":"user","content":[{"type":"input_text","text":"inspect"},{"type":"input_image","image_url":"` + image + `"}]}]}`)} - result := sessions.Input{Kind: "tool_result", Payload: json.RawMessage(`{"call_id":"call","result":{"success":true,"output":[{"type":"input_image","image_url":"` + image + `"}]}}`)} - for _, kind := range []string{"codex", "claude_sdk", "mcode"} { - profile, _ := builtin.Registry().Lookup(kind) - for _, placement := range []string{"none", "openai_hosted", "self_hosted"} { - err := validateDeclaredInputs(profile, []sessions.Input{message}) - want := kind != "mcode" - if (err == nil) != want { - t.Fatalf("%s/%s: %v", kind, placement, err) - } - } - } - profile, _ := builtin.Registry().Lookup("claude_sdk") - if err := validateDeclaredInputs(profile, []sessions.Input{result}); err != nil { - t.Fatal(err) - } - result.Payload = json.RawMessage(`{"call_id":"call","result":{"success":false,"output":[{"type":"input_image","image_url":"` + image + `"}]}}`) - if err := validateDeclaredInputs(profile, []sessions.Input{result}); !errors.Is(err, sessions.ErrInvalidInput) { - t.Fatal("native failed image admitted", err) - } -} diff --git a/services/core/tests/integration/claude_execution_test.go b/services/core/tests/integration/claude_execution_test.go index cb0e10d6d..16d575c43 100644 --- a/services/core/tests/integration/claude_execution_test.go +++ b/services/core/tests/integration/claude_execution_test.go @@ -139,7 +139,7 @@ func TestClaudeInvalidImageResultRejectsWholeBatchBeforePersistence(t *testing.T } return sessions.Input{Kind: "tool_result", Payload: payload} } - batch := []sessions.Input{messageInput("Follow up"), result(`{"success":true,"output":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}`), {Kind: "cancel", Payload: json.RawMessage(`{}`)}} + batch := []sessions.Input{messageInput("Follow up"), result(`{"success":true,"output":[{"type":"input_image","image_url":"https://example.com/a.png"}]}`), {Kind: "cancel", Payload: json.RawMessage(`{}`)}} if _, err := worker.SubmitInputs(t.Context(), h.tenant, h.session.ID, "batch", batch); !errors.Is(err, sessions.ErrInvalidInput) { t.Fatal(err) } diff --git a/services/core/tests/official_client.py b/services/core/tests/official_client.py index f066072ee..a6523a2e6 100644 --- a/services/core/tests/official_client.py +++ b/services/core/tests/official_client.py @@ -188,7 +188,6 @@ def issue_key(project_id, name): expect_error(NotFoundError, lambda: b.beta.agents.sessions.list(after=first.id)) expect_error(AuthenticationError, lambda: invalid.beta.agents.sessions.retrieve(first.id)) expect_error(BadRequestError, lambda: sessions.retrieve(first.id, extra_headers={"OpenAI-Beta": ""})) - expect_error(BadRequestError, lambda: sessions.create(**{**spec, "input": [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.com/image.png"}]}]})) # A self-hosted Session carries its own write-only model provider; nothing here calls it. provider = {"protocol": "responses", "base_url": "https://model.fixture.example/v1", "api_key": "fixture-model-key"} self_hosted = sessions.create(agent=spec["agent"], environment={"type": "self_hosted", "workspace_directory": "/workspace"}, diff --git a/services/core/tests/official_environment_activity.py b/services/core/tests/official_environment_activity.py index 88d5a399d..24b3c1a70 100644 --- a/services/core/tests/official_environment_activity.py +++ b/services/core/tests/official_environment_activity.py @@ -153,7 +153,7 @@ def snapshot(name, status): assert raw.get("/agents/sessions/" + session_id + suffix, headers={"Authorization": "Bearer " + foreign}).status_code == 404 create = raw.post("/agents/sessions", json={"agent": {"model": "MiniMax-M3"}, "input": [{"role": "user", "content": [ - {"type": "input_image", "image_url": "https://example.com/image.png"}]}], + {"type": "input_image", "image_url": ""}]}], "environment": {"type": "self_hosted", "workspace_directory": settings["workspace_directory"]}}) assert create.status_code == 400 submit = raw.post("/agents/sessions/" + session_id + "/events", json={"events": [{ diff --git a/services/core/tests/official_function_images.py b/services/core/tests/official_function_images.py index f23133896..8a56dc355 100644 --- a/services/core/tests/official_function_images.py +++ b/services/core/tests/official_function_images.py @@ -77,7 +77,7 @@ def run(sid, output=None, expected=None, cancel=False, failed_text=False, valida key = "result-" + action.call_id if validate: before = items(sid) - invalid_urls = ["https://example.test/image.png", "data:image/png;base64,?", "data:image/png;base64,AQID"] + invalid_urls = ["https://example.test/image.png", "data:image/png;base64,?"] # Claude rejects malformed/remote and error images before consuming a call. if proof["kind"] == "claude_sdk": invalid = [{**result, "output": [{"type": "input_image", "image_url": u}]} for u in invalid_urls] diff --git a/services/core/tests/official_message_images.py b/services/core/tests/official_message_images.py index d7aeee37c..528f0984c 100644 --- a/services/core/tests/official_message_images.py +++ b/services/core/tests/official_message_images.py @@ -137,7 +137,7 @@ def check_items(sid): assert len(sessions.turns.list(session.id).data) == 2 check_items(session.id) before = [t.id for t in sessions.turns.list(session.id).data] - invalid = [message_event(messages([text("must not be admitted")])), message_event(messages([{"type": "input_image", "image_url": "https://example.test/image.png"}]))] + invalid = [message_event(messages([text("must not be admitted")])), message_event(messages([{"type": "input_image", "image_url": ""}]))] response = http.post(base + "/v1/agents/sessions/" + session.id + "/events", headers=headers, json={"events": invalid}) assert response.status_code == 400 assert [t.id for t in sessions.turns.list(session.id).data] == before diff --git a/services/core/tests/official_session_initial_input.py b/services/core/tests/official_session_initial_input.py index c5cdc1b84..ade5ca5f0 100644 --- a/services/core/tests/official_session_initial_input.py +++ b/services/core/tests/official_session_initial_input.py @@ -90,7 +90,7 @@ def user_texts(session_id): for fields in [{"input": [{"type": None, "role": "user", "content": [{"type": "input_text", "text": "x"}]}]}, {"input": 0}, {"input": {}}, {"input": []}, {"input": ""}, {"input": [{"role": "user", "content": []}]}, {"input": [{"role": "user", "content": [{"type": "input_text", "text": ""}]}]}, {"input": [{"role": "assistant", "content": [{"type": "input_text", "text": "x"}]}]}, - {"input": [{"role": "user", "content": [{"type": "input_image", "image_url": "https://example.com/x.png"}]}]}]: + {"input": [{"role": "user", "content": [{"type": "input_image", "image_url": ""}]}]}]: reply = raw.post(base + "/v1/agents/sessions", headers=headers, json={**spec, **fields}) assert reply.status_code == 400, (fields, reply.status_code) reply = raw.post(unsupported + "/v1/agents/sessions", headers=headers, json={**spec, "input": "x"}) diff --git a/services/core/tests/official_workspace_images_native.py b/services/core/tests/official_workspace_images_native.py deleted file mode 100644 index fa9b4bfe4..000000000 --- a/services/core/tests/official_workspace_images_native.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Common hosted image acceptance through fixed SDK, HTTP and real native tools.""" - -import base64 -import importlib.metadata -import json -from pathlib import Path -import secrets -import time - -from image_fixture import picture -from session_cleanup import delete_session - - -def verify_workspace_images(client, foreign, http, model, kind, restart, evidence): - """restart(session_id, environment_id) cold-restarts the owned Core/Runtime.""" - pin = json.loads((Path(__file__).resolve().parents[3] / "contracts/agents-api/upstream.json").read_text()) - distribution = importlib.metadata.distribution("openai") - assert distribution.version == pin["sdk_version"] - assert json.loads(distribution.read_text("direct_url.json"))["vcs_info"]["commit_id"] == pin["commit"] - sessions = client.beta.agents.sessions - root = str(client.base_url).rstrip("/") + "/agents" - headers = {"Authorization": "Bearer " + client.api_key, "OpenAI-Beta": "agents=v1"} - foreign_headers = {**headers, "Authorization": "Bearer " + foreign.api_key} - proof = {"engine": kind, "checks": [], "runs": [], "calls": []} - expected_messages = [] - sid = None - - def save(): - Path(evidence).write_text(json.dumps(proof, indent=2)) - - def check(name): - proof["checks"].append(name) - save() - print(name, flush=True) - - def text(value): - return {"type": "input_text", "text": value} - - def messages(parts): - return [{"role": "user", "content": parts}] - - def event(value): - return {"type": "agent.session.input.message", "input": value} - - def image_messages(url, path): - return messages([text("Inspect this image."), {"type": "input_image", "image_url": url}, - text("Remember its four band colors in left-to-right order.")]) + messages([ - text("Use native workspace tools to write exactly the four lowercase color names, separated by commas and no newline, to " + path + - ". Create the parent directory if necessary. Do not call get_visual. Then reply with the same colors.")]) - - def until(predicate, timeout=240): - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - result = predicate() - if result: - return result - time.sleep(0.2) - raise AssertionError("Public workflow did not reach expected state") - - def items(): - response = http.get(root + "/sessions/" + sid + "/items", headers=headers, params={"order": "asc", "limit": 100}) - assert response.status_code == 200 - raw = response.json()["data"] - assert not response.json()["has_more"], "Acceptance exceeded its bounded item page" - assert raw == [i.to_dict() for i in sessions.items.list(sid, order="asc", limit=100).data] - return raw - - def history(): - stored = items() - assert [i["content"] for i in stored if i["type"] == "message" and i.get("role") == "user"] == [m["content"] for m in expected_messages] - outputs = {i["call_id"]: i for i in stored if i["type"] == "function_call_output"} - for call in proof["calls"]: - assert outputs[call["call_id"]]["output"] == call["output"] - return stored - - def post(events, key=None, foreign_request=False): - hdr = dict(foreign_headers if foreign_request else headers) - if key: - hdr["Idempotency-Key"] = key - return http.post(root + "/sessions/" + sid + "/events", headers=hdr, json={"events": events}) - - def verify_file(path, expected, turn): - answers = [i for i in items() if i["type"] == "message" and i.get("role") == "assistant"] - answer = " ".join(p["text"] for p in answers[-1]["content"] if p["type"] == "output_text").lower() - positions = [answer.find(color) for color in expected] - assert all(p >= 0 for p in positions) and positions == sorted(positions), answer - artifacts = list(sessions.artifacts.list(sid, limit=100)) - matches = [a for a in artifacts if a.path == path and a.turn_id == turn] - assert len(matches) == 1, [a.to_dict() for a in artifacts] - artifact = matches[0] - with sessions.artifacts.with_streaming_response.content(artifact.id, session_id=sid) as response: - assert response.read() == ",".join(expected).encode() - assert any(f.path == path for f in client.beta.agents.environments.files.list(eid, path="/workspace/outputs")) - for suffix in ("", "/content"): - assert http.get(root + "/sessions/" + sid + "/artifacts/" + artifact.id + suffix, headers=foreign_headers).status_code == 404 - - def run(value, handler=None): - observed = [] - handled = False - with sessions.events.stream(sid, timeout=300) as stream: - sessions.events.create(sid, events=[event(value)]) - expected_messages.extend(value) - for received in stream: - observed.append(received.to_dict()) - if received.type == "agent.session.requires_action": - assert handler is not None and not handled - assert len(received.session.required_actions) == 1 - action = received.session.required_actions[0] - assert action.name == "get_visual" - handled = True - handler(action) - assert received.type not in {"agent.session.failed", "agent.session.turn.failed"}, received.to_dict() - if received.type == "agent.session.idle" and any(e["type"] == "agent.session.turn.created" for e in observed): - break - else: - raise AssertionError("SSE ended without idle") - proof["runs"].append(observed) - save() - types = [e["type"] for e in observed] - terminal = [t for t in types if t in {"agent.session.turn.completed", "agent.session.turn.cancelled"}] - assert len(terminal) == 1 - assert types[-1] == "agent.session.idle" - assert types.index("agent.session.turn.created") < types.index(terminal[0]) < len(types) - 1 - assert len({e["event_id"] for e in observed}) == len(observed) - assert handled == (handler is not None) - turn = sessions.turns.list(sid, order="desc").data[0] - assert terminal[0] == "agent.session.turn." + turn.status - history() - return turn - - def submit(action, output, validate=False): - result = {"type": "agent.session.input.tool_result", "turn_id": action.turn_id, - "call_id": action.call_id, "success": True, "output": output} - key = "result-" + action.call_id - before = items() - if validate: - if kind == "claude_sdk": - invalid = [{**result, "success": False}, {**result, "output": [{"type": "input_image", "image_url": "https://example.test/image.png"}]}] - for bad in invalid: - assert post([bad], key).status_code == 400 - assert items() == before - assert sessions.retrieve(sid).required_actions[0].call_id == action.call_id - assert post([result], foreign_request=True).status_code == 404 - assert post([{**result, "call_id": "unknown-call"}]).status_code in {400, 404, 409} - assert items() == before - sessions.events.create(sid, events=[result], idempotency_key=key) - assert post([result], key).status_code == 202 - assert post([{**result, "output": "conflict"}], key).status_code == 409 - proof["calls"].append(result) - - colors = ["red", "green", "blue", "yellow"] - secrets.SystemRandom().shuffle(colors) - initial = image_messages(picture(colors), "/workspace/outputs/initial.txt") - try: - session = sessions.create(agent={"model": model, "tools": [{"type": "function", "name": "get_visual", - "description": "Wait for a visual supplied by the caller.", "parameters": {"type": "object", "properties": {}, "additionalProperties": False}}]}, - environment={"type": "openai_hosted"}, input=initial) - sid, eid = session.id, session.environment.id - proof.update(session=sid, environment=eid, initial_colors=colors) - expected_messages.extend(initial) - save() - - def initial_done(): - turns = sessions.turns.list(sid, order="asc", limit=100).data - if not turns or turns[-1].status not in {"completed", "failed", "cancelled"}: - return False - assert len(turns) == 1 and turns[0].status == "completed", [t.to_dict() for t in turns] - return turns[0] - - first = until(initial_done) - verify_file("/workspace/outputs/initial.txt", colors, first.id) - history() - check("initial_png_native_workspace_files_artifact") - - jpeg = "data:image/jpeg;base64," + base64.b64encode((Path(__file__).parent / "testdata/function-bands.jpg").read_bytes()).decode() - idle = messages([{"type": "input_image", "image_url": jpeg}]) + messages([text( - "Write this image's four lowercase band colors from left to right, comma-separated with no newline, to /workspace/outputs/idle.txt using native tools. Reply with those colors. Do not call get_visual.")]) - turn = run(idle) - verify_file("/workspace/outputs/idle.txt", ["yellow", "blue", "red", "green"], turn.id) - check("prepared_image_only_jpeg_and_message_boundary") - - active_colors = colors[1:] + colors[:1] - def active(action): - incoming = image_messages(picture(active_colors), "/workspace/outputs/active.txt") - batch = [event(incoming)] - sessions.events.create(sid, events=batch, idempotency_key="active-image") - expected_messages.extend(incoming) - assert post(batch, "active-image").status_code == 202 - assert post([event(messages([text("conflict")]))], "active-image").status_code == 409 - submit(action, "The user supplied an image. Follow its instructions, then stop.") - turn = run(messages([text("Call get_visual once and wait. Then follow the incoming image instructions.")]), active) - verify_file("/workspace/outputs/active.txt", active_colors, turn.id) - check("active_image_input_retry_and_native_tools") - - result_colors = colors[2:] + colors[:2] - output = [text("Inspect this visual."), {"type": "input_image", "image_url": picture(result_colors, 15)}, text("Remember these band colors.")] - turn = run(messages([text("Call get_visual exactly once. Read its returned image and use native tools to write its four lowercase band colors in left-to-right order, comma-separated with no newline, to /workspace/outputs/result.txt. Reply with the colors. Do not call get_visual again.")]), - lambda action: submit(action, output, validate=True)) - verify_file("/workspace/outputs/result.txt", result_colors, turn.id) - check("large_function_image_result_receipt_retry_isolation_and_artifact") - - before = history() - invalid = [event(messages([text("must not persist")])), event(messages([{"type": "input_image", "image_url": "https://example.test/image.png"}]))] - assert post(invalid).status_code == 400 - assert history() == before - for suffix in ("", "/items", "/turns", "/artifacts"): - assert http.get(root + "/sessions/" + sid + suffix, headers=foreign_headers).status_code == 404 - assert post([event(initial)], foreign_request=True).status_code == 404 - assert http.get(root + "/environments/" + eid + "/files", headers=foreign_headers).status_code == 404 - check("unsupported_message_batch_is_atomic_and_foreign_resources_hidden") - - other = sessions.create(agent={"model": model}, environment={"type": "openai_hosted"}) - try: - source_artifact = list(sessions.artifacts.list(sid, limit=100))[0] - for suffix in ("", "/content"): - assert http.get(root + "/sessions/" + other.id + "/artifacts/" + source_artifact.id + suffix, headers=headers).status_code == 404 - response = http.post(root + "/sessions/" + other.id + "/events", headers=headers, json={"events": [proof["calls"][-1]]}) - assert response.status_code in {400, 404, 409} - assert sessions.items.list(other.id).data == [] - until(lambda: client.beta.agents.environments.retrieve(other.environment.id).status == "connected") - response = http.get(root + "/environments/" + other.environment.id + "/files", headers=headers, params={"path": "/workspace/outputs"}) - assert response.status_code in {200, 404} - if response.status_code == 200: - assert response.json()["data"] == [] - assert history() == before - check("same_tenant_session_result_artifact_and_workspace_isolation") - finally: - delete_session(sessions, other.id) - - restart(sid, eid) - assert history() == before - turn = run(messages([text("Recall the most recent image returned by get_visual from conversation history, without reading any file or calling get_visual. Write its four lowercase band colors in order, comma-separated with no newline, to /workspace/outputs/resumed.txt using native tools, then reply with them.")])) - verify_file("/workspace/outputs/resumed.txt", result_colors, turn.id) - assert len(sessions.turns.list(sid, limit=100).data) == 5 - check("cold_core_runtime_history_continuation_without_replay") - - pending = [] - def cancel(action): - pending.append(action.call_id) - for _ in range(2): - sessions.events.create(sid, events=[{"type": "agent.session.input.cancel"}], idempotency_key="cancel-image-pending") - turn = run(messages([text("Call get_visual exactly once and wait for its result.")]), cancel) - assert turn.status == "cancelled" and not sessions.retrieve(sid).required_actions - assert not any(i["type"] == "function_call_output" and i["call_id"] == pending[0] for i in items()) - run(messages([text("Reply only PLAIN_OK. Do not call any tools.")])) - assert len(sessions.turns.list(sid, limit=100).data) == 7 - assert any(i["type"] == "message" and i.get("role") == "assistant" and any("PLAIN_OK" in p.get("text", "") for p in i["content"]) for i in items()) - check("pending_cancel_retry_and_text_continuation") - proof["passed"] = True - return proof["checks"] - finally: - save() - if sid: - delete_session(sessions, sid)