diff --git a/alembic/versions/f2a6c81d9b47_code_execution_executor_and_provider_files.py b/alembic/versions/f2a6c81d9b47_code_execution_executor_and_provider_files.py new file mode 100644 index 0000000000..8ce480bebd --- /dev/null +++ b/alembic/versions/f2a6c81d9b47_code_execution_executor_and_provider_files.py @@ -0,0 +1,81 @@ +"""Add the workspace executor pin and let a file row stand for a provider-held file. + +Three things this branch needs, in one revision because they land together: + +- ``workspace_code_execution_policies.executor``, the workspace's pin on who + runs a provider-native code-execution declaration (``auto``, ``otari`` or + ``provider``), over the deployment's default and over the request's header. + No backfill: NULL is "no pin", the state every existing row is in, so a + deployment upgrading onto this revision keeps deciding exactly as it did. +- ``file_objects.provider``, ``file_objects.provider_instance`` and + ``file_objects.provider_container_id``, with ``storage_ref`` made nullable. A + provider-native code execution keeps what it produced in the provider's own + container, so the row records who may read that id (the provider only + authenticates the deployment's credential) and which provider, through which + configured instance, to fetch it from, with no local blob to point at. +- Three indexes on ``file_objects``. One on ``expires_at``, because the sweep + selects ``deleted_at IS NOT NULL OR expires_at < now`` and the existing + ``deleted_at`` index cannot serve the ``OR`` on its own, so without it every + tick scanned the table. Two composites for the paged listing, the tenant + predicates then the keyset its cursor pages on: + ``(user_id, workspace_id, created_at, id)`` for a keyed request, and + ``(user_id, created_at, id)`` for a master-key listing that names no + workspace, which the first cannot order. Without them every page sorts the + user's whole set. + +The ``ADD COLUMN``s need no table rebuild (see ``a4d7f1c9e2b6``), but dropping +``storage_ref``'s NOT NULL does on SQLite, which has no ``ALTER COLUMN``, so +that one statement goes through ``batch_alter_table``. + +Revision ID: f2a6c81d9b47 +Revises: d5f8b2a4c6e9 +Create Date: 2026-09-21 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "f2a6c81d9b47" +down_revision: str | Sequence[str] | None = "d5f8b2a4c6e9" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_POLICY_TABLE = "workspace_code_execution_policies" +_FILES_TABLE = "file_objects" + + +def upgrade() -> None: + op.add_column(_POLICY_TABLE, sa.Column("executor", sa.String(length=16), nullable=True)) + + op.add_column(_FILES_TABLE, sa.Column("provider", sa.String(), nullable=True)) + op.add_column(_FILES_TABLE, sa.Column("provider_instance", sa.String(), nullable=True)) + op.add_column(_FILES_TABLE, sa.Column("provider_container_id", sa.String(), nullable=True)) + with op.batch_alter_table(_FILES_TABLE) as batch: + batch.alter_column("storage_ref", existing_type=sa.String(), nullable=True) + + op.create_index("ix_file_objects_expires_at", _FILES_TABLE, ["expires_at"]) + op.create_index( + "ix_file_objects_user_workspace_created", + _FILES_TABLE, + ["user_id", "workspace_id", "created_at", "id"], + ) + op.create_index("ix_file_objects_user_created", _FILES_TABLE, ["user_id", "created_at", "id"]) + + +def downgrade() -> None: + op.drop_index("ix_file_objects_user_created", table_name=_FILES_TABLE) + op.drop_index("ix_file_objects_user_workspace_created", table_name=_FILES_TABLE) + op.drop_index("ix_file_objects_expires_at", table_name=_FILES_TABLE) + + # A provider-held row has no blob to point at, so it cannot survive the + # column going back to NOT NULL. + op.execute(sa.text("DELETE FROM file_objects WHERE storage_ref IS NULL")) + with op.batch_alter_table(_FILES_TABLE) as batch: + batch.alter_column("storage_ref", existing_type=sa.String(), nullable=False) + op.drop_column(_FILES_TABLE, "provider_container_id") + op.drop_column(_FILES_TABLE, "provider_instance") + op.drop_column(_FILES_TABLE, "provider") + + op.drop_column(_POLICY_TABLE, "executor") diff --git a/config.example.yml b/config.example.yml index 1c6d2a3762..788e7969c2 100644 --- a/config.example.yml +++ b/config.example.yml @@ -86,8 +86,15 @@ providers: # File and image normalization. See docs/files.md. # files_enabled: true # files_local_dir: "./otari-files" +# Any fsspec filesystem instead of a local directory or boto3 S3: +# files_backend: fsspec +# files_url: "gcs://my-bucket/otari-files" +# files_storage_options: { project: "my-project" } # files_max_bytes: 536870912 +# files_output_max_files: 20 +# files_output_max_bytes: 67108864 # files_retention_hours: 168 +# files_sweep_interval_sec: 3600 # vision_strategy: describe # vision_describe_model: "ollama:qwen2-vl" # model_capabilities: diff --git a/demo/code-exec/.gitignore b/demo/code-exec/.gitignore index 4c49bd78f1..354e683849 100644 --- a/demo/code-exec/.gitignore +++ b/demo/code-exec/.gitignore @@ -1 +1,2 @@ .env +plots/ diff --git a/demo/code-exec/plot_with_anthropic_sdk.py b/demo/code-exec/plot_with_anthropic_sdk.py new file mode 100644 index 0000000000..edbb1ba4c8 --- /dev/null +++ b/demo/code-exec/plot_with_anthropic_sdk.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Anthropic's own SDK, pointed at Otari, asks for a bar plot and saves it. + +The point is the model id. Nothing else in the script changes between: + + uv run python demo/code-exec/plot_with_anthropic_sdk.py anthropic:claude-sonnet-4-6 + uv run python demo/code-exec/plot_with_anthropic_sdk.py nebius:openai/gpt-oss-120b + +The first runs on Anthropic's own sandbox, because the `auto` executor leaves a +declaration with a provider that serves it natively. The second has no native +sandbox, so Otari runs the code. Either way the reply carries the same +`code_execution_tool_result` blocks, and the chart downloads from Otari's files +API with the same call: a file Otari's sandbox produced it stored, and one +Anthropic's produced it streams back from Anthropic under Anthropic's own id. + +Environment: + OTARI_URL default http://localhost:8000 + OTARI_KEY the gateway's master key or an API key (default demo-master-key) + QA_USER the user a master-key request bills (default plot-roundtrip) + QA_OUT_DIR where to write the plots (default demo/code-exec/plots) +""" + +from __future__ import annotations + +import os +import pathlib +import sys +from typing import Any + +from anthropic import Anthropic + +OTARI_URL = os.environ.get("OTARI_URL", "http://localhost:8000").rstrip("/") +OTARI_KEY = os.environ.get("OTARI_KEY", "demo-master-key") +USER = os.environ.get("QA_USER", "plot-roundtrip") +OUT_DIR = pathlib.Path(os.environ.get("QA_OUT_DIR", pathlib.Path(__file__).parent / "plots")) + +PROMPT = ( + "Use the code execution tool to draw a bar chart of these sales figures with " + "matplotlib: Jan 120, Feb 95, Mar 160, Apr 210, May 175. Save it as 'bar_plot.png' " + "in the working directory, then reply with the file id of the png you saved." +) +CODE_TOOL: Any = {"type": "code_execution_20250825", "name": "code_execution"} + +# The SDK appends /v1 to its base URL, and Otari serves its API under /api/v1. +# `default_query` is only for a master key: the files routes want to be told +# whose files to read, which an API key answers by itself. +otari = Anthropic(base_url=f"{OTARI_URL}/api", api_key=OTARI_KEY, default_query={"user": USER}) + + +def produced_file_ids(message: Any) -> list[str]: + """Every file id the run's tool results announce, in the order they appear. + + The python and the bash variants of the tool each name their outputs in + their own block type, and Otari answers in whichever the run used, so this + looks for the shape rather than for one block name. + """ + ids = [] + for block in message.content: + result = getattr(block, "content", None) + for output in getattr(result, "content", None) or []: + file_id = getattr(output, "file_id", None) + if file_id and file_id not in ids: + ids.append(file_id) + return ids + + +def download(file_id: str, model: str) -> pathlib.Path: + """Fetch a produced file from Otari, wherever its bytes actually live.""" + meta = otari.beta.files.retrieve_metadata(file_id) + body = otari.beta.files.download(file_id) + + OUT_DIR.mkdir(parents=True, exist_ok=True) + path = OUT_DIR / f"{model.replace(':', '-').replace('/', '-')}-{meta.filename}" + body.write_to_file(path) + return path + + +def main(model: str) -> int: + # No `betas`: they are Anthropic's own, and a provider with no Messages API + # of its own cannot honor one, so Otari drops them rather than refusing. + message = otari.beta.messages.create( + model=model, + max_tokens=4096, + messages=[{"role": "user", "content": PROMPT}], + tools=[CODE_TOOL], + # A master-key request must say who it bills; an API key ignores this + # in favor of its own user. + metadata={"user_id": USER}, + ) + + ran_natively = message.container is not None + print(f"{model}: code ran on {'the provider' if ran_natively else "otari's sandbox"}") + + file_ids = produced_file_ids(message) + if not file_ids: + print("no file id in the tool results:") + print(message.model_dump_json(indent=2)[:2000]) + return 1 + + for file_id in file_ids: + path = download(file_id, model) + print(f" {file_id} -> {path} ({path.stat().st_size} bytes)") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "anthropic:claude-sonnet-4-6")) diff --git a/demo/code-exec/plot_with_any_llm.py b/demo/code-exec/plot_with_any_llm.py new file mode 100644 index 0000000000..c2a4a1757f --- /dev/null +++ b/demo/code-exec/plot_with_any_llm.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""any-llm, pointed at Otari, asks for a bar plot and saves it. + +The third way to write the other two scripts. any-llm talks to Otari through +its `otari` provider, so one call shape reaches every model Otari serves and +the code-execution tool is declared in Otari's own vocabulary rather than a +provider's: + + uv run python demo/code-exec/plot_with_any_llm.py nebius:openai/gpt-oss-120b + uv run python demo/code-exec/plot_with_any_llm.py anthropic:claude-sonnet-4-6 + +`otari_code_execution` always runs on Otari's sandbox, whatever the model and +whatever the executor says, which is the trade for one vocabulary: an +`anthropic:` model here runs the code here rather than natively. Ask for the +provider's own sandbox with that provider's declaration instead, which is what +`plot_with_anthropic_sdk.py` does. + +any-llm covers the inference half only. It has no files API, so the chart is +downloaded with a plain HTTP GET against the same gateway. + +Environment: + OTARI_URL default http://localhost:8000 + OTARI_KEY the gateway's master key or an API key (default demo-master-key) + QA_USER the user a master-key request bills (default plot-roundtrip) + QA_OUT_DIR where to write the plots (default demo/code-exec/plots) +""" + +from __future__ import annotations + +import asyncio +import os +import pathlib +import re +import sys +from typing import Any + +import httpx +from any_llm import acompletion + +OTARI_URL = os.environ.get("OTARI_URL", "http://localhost:8000").rstrip("/") +OTARI_KEY = os.environ.get("OTARI_KEY", "demo-master-key") +USER = os.environ.get("QA_USER", "plot-roundtrip") +OUT_DIR = pathlib.Path(os.environ.get("QA_OUT_DIR", pathlib.Path(__file__).parent / "plots")) + +PROMPT = ( + "Use the code execution tool to draw a bar chart of these sales figures with " + "matplotlib: Jan 120, Feb 95, Mar 160, Apr 210, May 175. Save it as 'bar_plot.png' " + "in the working directory, then reply with the file id of the png you saved." +) +# Otari's own declaration, the one shape that means the same thing to every model. +CODE_TOOL: Any = {"type": "otari_code_execution"} +# A stored file's id as the tool result gives it to the model. +FILE_ID = re.compile(r"file-[0-9a-f]{32}") + +files = httpx.Client(base_url=f"{OTARI_URL}/api/v1", headers={"Otari-Key": OTARI_KEY}, timeout=60.0) + + +def save(file_id: str, model: str) -> pathlib.Path: + """Download one stored file and write it beside the other plots.""" + meta = files.get(f"/files/{file_id}", params={"user": USER}) + meta.raise_for_status() + body = files.get(f"/files/{file_id}/content", params={"user": USER}) + body.raise_for_status() + + OUT_DIR.mkdir(parents=True, exist_ok=True) + path = OUT_DIR / f"{model.replace(':', '-').replace('/', '-')}-{meta.json()['filename']}" + path.write_bytes(body.content) + return path + + +async def run(model: str) -> int: + response = await acompletion( + # The gateway is the provider; the selector it routes on travels as the model. + provider="otari", + model=model, + messages=[{"role": "user", "content": PROMPT}], + tools=[CODE_TOOL], + api_base=f"{OTARI_URL}/api/v1", + api_key=OTARI_KEY, + # A master-key request must say who it bills; an API key ignores this + # in favor of its own user. + user=USER, + ) + + # Chat Completions has no shape for a server-run tool, so the loop resolves + # inside the gateway and only the final message comes back. The model was + # given each produced file as `name (file_id: file-...)` and asked to pass + # the id on, which is what is read back here. + reply = response.choices[0].message.content or "" + file_ids = list(dict.fromkeys(FILE_ID.findall(reply))) + if not file_ids: + print(f"{model}: no stored file id in the reply: {reply[:300]}") + return 1 + + print(f"{model}: code ran on otari's sandbox") + for file_id in file_ids: + path = save(file_id, model) + print(f" {file_id} -> {path} ({path.stat().st_size} bytes)") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(run(sys.argv[1] if len(sys.argv) > 1 else "nebius:openai/gpt-oss-120b"))) diff --git a/demo/code-exec/plot_with_openai_sdk.py b/demo/code-exec/plot_with_openai_sdk.py new file mode 100644 index 0000000000..ef1ef8f805 --- /dev/null +++ b/demo/code-exec/plot_with_openai_sdk.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""OpenAI's own SDK, pointed at Otari, asks for a bar plot and saves it. + +The counterpart to `plot_with_anthropic_sdk.py`, on the Responses API: + + uv run python demo/code-exec/plot_with_openai_sdk.py openai:gpt-4o-mini + QA_EXECUTOR=otari uv run python demo/code-exec/plot_with_openai_sdk.py openai:gpt-4o-mini + +The first runs `code_interpreter` on OpenAI's own container, because the `auto` +executor leaves a declaration with a provider that serves it natively. The +second sends `X-Otari-Code-Execution: otari`, so the same request runs on +Otari's sandbox instead. Both come back as a `code_interpreter_call` item, and +either way the chart downloads from Otari's files API: one Otari's sandbox +produced it stored, and one OpenAI's produced it streams back from OpenAI's +container under OpenAI's own id. + +Swapping in an open model works the same way, but only for a provider whose +Responses API Otari supports (groq, fireworks, openrouter, huggingface, gmi). +Nebius is not one of those: `nebius:...` on this endpoint is refused with +"Provider 'nebius' does not support the Responses API", so use the Anthropic +script, or Chat Completions, for a nebius model. + +Environment: + OTARI_URL default http://localhost:8000 + OTARI_KEY the gateway's master key or an API key (default demo-master-key) + QA_EXECUTOR `auto` (default), `otari` or `provider`, sent as X-Otari-Code-Execution + QA_USER the user a master-key request bills (default plot-roundtrip) + QA_OUT_DIR where to write the plots (default demo/code-exec/plots) +""" + +from __future__ import annotations + +import os +import pathlib +import sys +from typing import Any + +from openai import OpenAI + +OTARI_URL = os.environ.get("OTARI_URL", "http://localhost:8000").rstrip("/") +OTARI_KEY = os.environ.get("OTARI_KEY", "demo-master-key") +EXECUTOR = os.environ.get("QA_EXECUTOR", "auto") +USER = os.environ.get("QA_USER", "plot-roundtrip") +OUT_DIR = pathlib.Path(os.environ.get("QA_OUT_DIR", pathlib.Path(__file__).parent / "plots")) + +PROMPT = ( + "Use the code interpreter to draw a bar chart of these sales figures with matplotlib: " + "Jan 120, Feb 95, Mar 160, Apr 210, May 175. Save it as 'bar_plot.png' in the working " + "directory, then reply with the file id of the png you saved." +) +CODE_TOOL: Any = {"type": "code_interpreter", "container": {"type": "auto"}} + +# Otari serves its API under /api/v1, which is the whole base URL for this SDK. +# `default_query` is only for a master key: the files routes want to be told +# whose files to read, which an API key answers by itself. +otari = OpenAI(base_url=f"{OTARI_URL}/api/v1", api_key=OTARI_KEY, default_query={"user": USER}) + + +def container_citations(response: Any) -> list[tuple[str, str]]: + """`(file_id, filename)` for each file the provider's container cited. + + OpenAI announces a produced file as a `container_file_citation` annotation + on the message it wrote. Otari records those ids as it answers, so the + download below reaches them through the gateway like any other file. + """ + return [ + (note.file_id, note.filename) + for item in response.output + for part in getattr(item, "content", None) or [] + for note in getattr(part, "annotations", None) or [] + if getattr(note, "type", "") == "container_file_citation" + ] + + +def produced_images(response: Any) -> list[str]: + """The file ids in the `image` outputs of a gateway-run `code_interpreter_call`. + + Otari announces a produced image as the URL it serves the file from, which + is the only shape the Responses API has for one, so the id is the last path + segment before `/content`. + """ + return [ + output.url.rsplit("/", 2)[-2] + for item in response.output + if item.type == "code_interpreter_call" + for output in item.outputs or [] + if getattr(output, "type", "") == "image" + ] + + +def save(file_id: str, name: str, model: str) -> pathlib.Path: + body = otari.files.content(file_id).read() + OUT_DIR.mkdir(parents=True, exist_ok=True) + path = OUT_DIR / f"{model.replace(':', '-').replace('/', '-')}-{name}" + path.write_bytes(body) + return path + + +def main(model: str) -> int: + response = otari.responses.create( + model=model, + input=PROMPT, + tools=[CODE_TOOL], + # A master-key request must say who it bills; an API key ignores this + # in favor of its own user. + user=USER, + extra_headers={"X-Otari-Code-Execution": EXECUTOR}, + ) + + # Otari's own ids mark a gateway-run execution; the provider's are its own. + calls = [item for item in response.output if item.type == "code_interpreter_call"] + ran_natively = bool(calls) and not calls[0].id.startswith("otari_ci_") + print(f"{model}: code ran on {'the provider' if ran_natively else "otari's sandbox"} (executor {EXECUTOR})") + + produced = container_citations(response) or [ + (file_id, otari.files.retrieve(file_id).filename) for file_id in produced_images(response) + ] + if not produced: + print(f"nothing produced: {response.model_dump_json(indent=2)[:2000]}") + return 1 + + for file_id, name in produced: + path = save(file_id, name, model) + print(f" {file_id} -> {path} ({path.stat().st_size} bytes)") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "openai:gpt-4o-mini")) diff --git a/docs/code-execution-protocol.md b/docs/code-execution-protocol.md index 6a1b5ce611..6a9097004c 100644 --- a/docs/code-execution-protocol.md +++ b/docs/code-execution-protocol.md @@ -52,7 +52,12 @@ below is unchanged either way, which is what lets the same backend serve both. Six operations, of which the first three are the whole execution path. A backend MUST implement those three; the file operations are OPTIONAL and are -used only by clients that move files in or out of a session. +used only by clients that move files in or out of a session. Otari is such a +client when a request runs with files enabled: it seeds uploads with `PutFile` +before the first call, and after each call fetches with `GetFile` what the +result block's file references name together with whatever `ListFiles` shows +appeared or changed, since not every backend fills the block's list in (see +`docs/files.md`, "Files and code execution"). | Operation | Purpose | Request | Response | |---|---|---|---| @@ -370,6 +375,7 @@ with the reference one rather than merely similar to it. |---|---|---| | `sandbox_url` | `OTARI_SANDBOX_URL` | Base URL of the backend. Unset, `otari_code_execution` requests are rejected. | | `sandbox_purpose_hint` | `OTARI_SANDBOX_PURPOSE_HINT` | Default purpose hint for the tool, when a request supplies none. | +| `code_execution_executor` | `OTARI_CODE_EXECUTION_EXECUTOR` | Who runs a provider-native code-execution declaration: `auto` (default), `otari` or `provider`. See [Built-in tools](tools.md#code-execution-executor). | See [Configuration](configuration.md) for the full settings reference and [Built-in tools](tools.md) for the user-facing view of the tool. diff --git a/docs/configuration.md b/docs/configuration.md index 9436a8a145..d1ce2a46ef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -328,6 +328,7 @@ The Tools pages and `GET /api/v1/tool-settings` show effective sandbox, web-sear and guardrail configuration. Common startup settings are: - `sandbox_url` +- `code_execution_executor` - `web_search_url` - `web_search_provider` and `web_search_provider_api_key` - `guardrails_url` diff --git a/docs/domains.md b/docs/domains.md index a2a338e7f4..a09164ce6f 100644 --- a/docs/domains.md +++ b/docs/domains.md @@ -210,9 +210,10 @@ retrieval, code execution and files. `web_retrieval_network.py`, `web_retrieval_policy.py`, `search_tool_store_service.py`, `tool_settings_service.py`, `tool_format.py`, `tool_usage.py`, `file_service.py`, `file_store.py`, - `file_extractors.py`, `tenancy/workspace_mcp_server_service.py`, + `file_extractors.py`, `files/`, `tenancy/workspace_mcp_server_service.py`, `tenancy/workspace_web_search_service.py`, `tenancy/workspace_code_execution_policy_service.py` +- Repositories: `files/` - Models: `tools.py`, `mcp.py` ### guardrails diff --git a/docs/files.md b/docs/files.md index 903c60096a..714145db63 100644 --- a/docs/files.md +++ b/docs/files.md @@ -55,7 +55,105 @@ Otari also requires pricing for that model key by default: add pricing, enable an intentionally unpriced backend. You can also inline a file as a base64 `data:` URL (`file.file_data`) or send an -`image_url` block, with or without uploading first. +`image_url` block, with or without uploading first. On the Responses API a +`input_file` or `input_image` item may sit directly in `input` as well as inside +a message. + +### Using the OpenAI or Anthropic SDK + +The five routes (`POST`/`GET /v1/files`, `GET`/`DELETE /v1/files/{id}`, +`GET /v1/files/{id}/content`) share their paths and verbs with both vendors' +Files APIs, so either official SDK works against Otari with only its base URL +changed. The response shape follows the caller: a request carrying Anthropic's +`anthropic-version` header, which its SDK sends on every call, gets Anthropic's +`FileMetadata` (`type`, `size_bytes`, `mime_type`, `downloadable`, an RFC 3339 +`created_at`); everything else gets the OpenAI file object (`object`, `bytes`, +`purpose`, an epoch `created_at`). + +Mind the base URL: Anthropic's SDK appends `/v1` itself, so it takes +`http://localhost:8000/api`, while an OpenAI-compatible client takes +`http://localhost:8000/api/v1` (see the +[API reference](api-reference.md)). + +```python +from anthropic import Anthropic +client = Anthropic(base_url="http://localhost:8000/api", api_key="") +meta = client.beta.files.upload(file=("report.pdf", open("report.pdf", "rb"), "application/pdf")) +client.beta.files.download(meta.id) # Otari serves every stored file's bytes back +``` + +Listings are cursor-paged: `limit` (default 100, at most 1000), `after` +(OpenAI) or `after_id` (Anthropic) naming the last file of the previous page, +`order` (`desc` by default), and `has_more`, `first_id`, `last_id` on the page. + +## Files and code execution + +When a request's code runs on Otari's sandbox, because it declared the +`otari_code_execution` tool or because the +[executor](tools.md#code-execution-executor) brought a provider's own +declaration here, every uploaded file it references is also seeded into the +sandbox session's working directory, so the code the model writes can open it. +The file keeps its own filename, reduced to its last path segment; a second +upload with the same name is suffixed (`data.csv`, then `data-2.csv`), and the +marker the model is given carries the name the file actually has. An Anthropic +`container_upload` block (`{"type": "container_upload", "file_id": "..."}`) is +for the sandbox only: the model is told the file is there and never sees its +contents. A `document`, `file`, or `input_file` block with a `file_id` is both +shown to the model (extracted or passed through as usual) and seeded. Without a +sandbox in the request, a `container_upload` block is read as a document. That +is also what happens when the executor leaves a provider's declaration with the +provider: whether a file is staged follows who runs the code, decided once from +the workspace pin, the header and the deployment default. + +A file the code writes into the working directory comes back as a new stored +file owned by the same user and workspace, with purpose `code_execution_output`. +Otari finds it two ways and unions them: the result block's own list of produced +files, and a listing of the workspace after each call compared with the one +before, so a backend that leaves the block's list empty (the reference container +does) still has its files collected. A seeded input the code rewrote counts as +produced. +The model sees it in the tool result as `chart.png (file_id: file-...)` and is +asked to pass that id on, and the caller downloads it with +`GET /v1/files/{id}/content`. A caller who declared Anthropic's own code tool +also gets the id in the `code_execution_tool_result` block's +`code_execution_output` entries, where Anthropic's SDK looks for it. Both directions need a sandbox backend that +implements the protocol's optional `PutFile` and `GetFile` operations, and +collecting a file the block does not name needs `ListFiles` as well; a seed the +backend refuses fails the request rather than running code over a missing input, +while an output that cannot be fetched is named without an id and the run stands. + +### A file the provider's own sandbox produced + +A declaration the [executor](tools.md#code-execution-executor) leaves with the +provider runs in the provider's container, and the file it writes stays there, +under the provider's own id. Otari copies nothing, and records a row saying +whose that file is and which provider holds it, so +`GET /v1/files/{id}/content` streams the bytes through on demand and the same +download serves a chart whichever sandbox drew it. The row is also what keeps +that safe: a provider authenticates the deployment's credential, which is +coarser than a workspace-scoped key, so the user and workspace predicate every +other file gets is applied here before anything is fetched. + +Three things follow from Otari not holding the bytes. A listing shows `0` for +the size, because the provider does not say how many bytes there are until they +are read. The provider's id is what travels, rather than one of Otari's, since +rewriting it would break a client that echoes the turn back with a container +reference the provider never issued. And such a file cannot be an *input* to a +later request: a `file_id` block naming one is dropped, because there is +nothing local to extract or seed a session with. Anthropic and OpenAI are the +providers Otari can fetch back from; a native run on any other is announced by +the provider and downloaded from it. + +One call may store at most `files_output_max_files` files and +`files_output_max_bytes` in total (20 files and 64 MB by default, the latter also +bounded by `files_max_bytes`). What a run writes is untrusted, so a file past +either cap is named in the tool result without an id rather than stored. A +produced file is streamed from the sandbox into the store and never held whole. + +> The reference `otari-sandbox-container` leaves the result block's +> file-reference list empty, so with it collection depends on `ListFiles`, which +> it implements. A backend that neither names nor lists a file does not have it +> collected. ### Who can see an uploaded file @@ -109,7 +207,19 @@ in order: See [config.example.yml](../config.example.yml) for the full list. Key knobs: - `files_enabled`, `files_backend`, `files_local_dir`, `files_max_bytes`, -`files_retention_hours`: upload storage. +`files_retention_hours`: upload storage. `files_output_max_files` and +`files_output_max_bytes` bound what one code-execution call may store from its +sandbox (see above). `files_backend` is `local` (a +directory), `s3` (boto3, `files_s3_*`), or `fsspec`: any filesystem +[fsspec](https://filesystem-spec.readthedocs.io) has an implementation for, +named by `files_url` (`gcs://bucket/prefix`, `abfs://container/prefix`, +`s3://bucket/prefix`, `sftp://host/path`, `file:///path`, ...) with the +implementation's own keyword arguments in `files_storage_options`. It is an +optional extra, `pip install otari[fsspec]`, like `otari[s3]`; install the +implementation package for the protocol as well (`gcsfs`, `adlfs`, `s3fs`, `paramiko`); +most read their standard credential environment variables on their own. An expired file answers 404 at once, +and the background sweep (`files_sweep_interval_sec`, hourly by default, `0` to +disable) then reclaims its bytes and row along with those of deleted files. - `file_understanding_enabled`: master switch for content normalization. - `vision_strategy` (`describe` | `ocr` | `off`) and `vision_describe_model`: how images are handled for text-only models. The describe model may be a local diff --git a/docs/public/openapi.json b/docs/public/openapi.json index f493fa17d9..f8cbc65236 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -3414,6 +3414,16 @@ "title": "CheckVerdictRequest", "type": "object" }, + "CodeExecutor": { + "description": "Who runs the code a request's code-execution tool asks for.\n\nThe one vocabulary shared by the deployment setting, the workspace policy,\nthe per-request header and the platform's resolve payload, so a value read\nfrom any of them means the same thing at admission.", + "enum": [ + "auto", + "otari", + "provider" + ], + "title": "CodeExecutor", + "type": "string" + }, "ConfigField": { "description": "One effective config value surfaced to the dashboard's config viewer.", "properties": { @@ -6690,7 +6700,7 @@ "description": "One tool the gateway can run itself.", "properties": { "accepted_types": { - "description": "Every `tools[].type` this deployment currently routes to the tool. Always includes the canonical `otari_*` type; for web search it also includes the provider-named keywords when interception is enabled.", + "description": "Every `tools[].type` this deployment currently routes to the tool. Always includes the canonical `otari_*` type; for web search it also includes the provider-named keywords when interception is enabled, and for code execution the provider-named keywords unless the deployment's executor is `provider`.", "items": { "type": "string" }, @@ -13659,6 +13669,20 @@ "ToolSettingField": { "description": "One editable tool/guardrail field surfaced to the dashboard.", "properties": { + "choices": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Choices" + }, "description": { "anyOf": [ { @@ -14514,6 +14538,17 @@ "web_search_url": "http://searxng:8080" }, "properties": { + "code_execution_executor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Code Execution Executor" + }, "guardrails_url": { "anyOf": [ { @@ -16657,6 +16692,16 @@ ], "title": "Exec Timeout S" }, + "executor": { + "anyOf": [ + { + "$ref": "#/components/schemas/CodeExecutor" + }, + { + "type": "null" + } + ] + }, "image": { "anyOf": [ { @@ -16726,6 +16771,7 @@ "exec_timeout_s", "image", "tools", + "executor", "created_at", "updated_at" ], @@ -16767,6 +16813,17 @@ "description": "Ceiling on one execution's runtime in seconds; only ever lowers the effective limit, so at most 60", "title": "Exec Timeout S" }, + "executor": { + "anyOf": [ + { + "$ref": "#/components/schemas/CodeExecutor" + }, + { + "type": "null" + } + ], + "description": "Who runs a provider-native code-execution declaration for this workspace: 'auto' (the provider when it runs the tool natively for the model, else this gateway's sandbox), 'otari' or 'provider'. Pins over the deployment default and over the request's X-Otari-Code-Execution header; null leaves both in charge" + }, "image": { "anyOf": [ { @@ -20600,7 +20657,7 @@ }, "/api/v1/files": { "get": { - "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.", + "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nPages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names\nthe last file of the previous page, and ``has_more`` says whether to ask\nagain. A cursor that has since been deleted or has expired is still a\nposition; one the caller never owned is a 404.", "operationId": "files-list_files", "parameters": [ { @@ -20651,6 +20708,64 @@ ], "title": "Workspace Id" } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "default": 100, + "maximum": 1000, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + { + "in": "query", + "name": "after", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After" + } + }, + { + "in": "query", + "name": "after_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "After Id" + } + }, + { + "in": "query", + "name": "order", + "required": false, + "schema": { + "default": "desc", + "enum": [ + "asc", + "desc" + ], + "title": "Order", + "type": "string" + } } ], "responses": { @@ -20691,7 +20806,7 @@ ] }, "post": { - "description": "OpenAI-compatible file upload endpoint.", + "description": "Upload a file. Answers in the OpenAI or Anthropic file shape, following the caller's headers.", "operationId": "files-create_file", "requestBody": { "content": { diff --git a/docs/public/otari.postman_collection.json b/docs/public/otari.postman_collection.json index a671f57f41..ee21cbf14b 100644 --- a/docs/public/otari.postman_collection.json +++ b/docs/public/otari.postman_collection.json @@ -1763,7 +1763,7 @@ { "name": "List Files", "request": { - "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.", + "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nPages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names\nthe last file of the previous page, and ``has_more`` says whether to ask\nagain. A cursor that has since been deleted or has expired is still a\nposition; one the caller never owned is a 404.", "header": [], "method": "GET", "url": { @@ -1793,16 +1793,40 @@ "disabled": true, "key": "workspace_id", "value": "" + }, + { + "description": "", + "disabled": true, + "key": "limit", + "value": "" + }, + { + "description": "", + "disabled": true, + "key": "after", + "value": "" + }, + { + "description": "", + "disabled": true, + "key": "after_id", + "value": "" + }, + { + "description": "", + "disabled": true, + "key": "order", + "value": "" } ], - "raw": "{{baseUrl}}/api/v1/files?user=&purpose=&workspace_id=" + "raw": "{{baseUrl}}/api/v1/files?user=&purpose=&workspace_id=&limit=&after=&after_id=&order=" } } }, { "name": "Create File", "request": { - "description": "OpenAI-compatible file upload endpoint.", + "description": "Upload a file. Answers in the OpenAI or Anthropic file shape, following the caller's headers.", "header": [], "method": "POST", "url": { diff --git a/docs/tools.md b/docs/tools.md index 13922b1144..7cc818dde6 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -22,9 +22,11 @@ Unavailable but recognized tools remain in the response with ## Who runs a tool -An `otari_*` type is executed by Otari. Other tool declarations are forwarded -to the provider, including provider-native code interpreter and web-search -types. Function tools remain the caller's responsibility. +An `otari_*` type is executed by Otari. A provider-native web-search type is +forwarded to the provider unless [interception](#web-search-interception) is on. +A provider-native code-execution type is decided by the request's +[executor](#code-execution-executor). Function tools remain the caller's +responsibility. ### Web-search interception @@ -120,6 +122,75 @@ Request it with: The sandbox speaks the [code-execution protocol](code-execution-protocol.md). A runnable example lives under `demo/code-exec/`. +### Code-execution executor + +A request written for a provider's own sandbox keeps its provider's vocabulary: +Anthropic's `{"type": "code_execution_20250825"}` on `/api/v1/messages`, OpenAI's +`{"type": "code_interpreter"}` on `/api/v1/responses`, or the bare +`{"type": "code_execution"}`. The **executor** decides who runs the code such a +declaration asks for: + +| Executor | Who runs the code | +| --- | --- | +| `auto` (default) | The provider, when it runs that tool natively for the dispatched model and wire format; otherwise Otari's sandbox. | +| `otari` | Always Otari's sandbox. | +| `provider` | Always the provider; the declaration is forwarded untouched. | + +`auto` is what makes a model swap transparent. An Anthropic Messages request +carrying `code_execution_20250825` runs natively against an Anthropic model, and +the same request against an open model runs on the sandbox, with the same +`server_tool_use` and `code_execution_tool_result` blocks coming back. An +`anthropic-beta` header travels only as far as it can be honored: against a +provider with no Messages API of its own it is dropped rather than refused, +because a beta names an Anthropic feature that provider was never going to +serve, and refusing it would make the request fail purely because its model +changed. On +Responses a claimed `code_interpreter` is answered with a `code_interpreter_call` +item. Chat Completions has no native shape, so a claimed declaration there +resolves inside the tool loop and only the final message is returned. Nothing +runs natively on Chat Completions, and the bare `code_execution` form is no +provider's, so under `auto` both always run on the sandbox. + +Three layers choose the executor. The workspace pin wins over both of the +others; the header wins over the deployment default: + +1. The deployment default, `code_execution_executor` (`OTARI_CODE_EXECUTION_EXECUTOR`), + editable on the Tools page. Unset means `auto`. +2. A [workspace policy](#per-workspace-code-policy) may pin `executor`. A pin is + a decision the request cannot argue with: a header that disagrees is refused + with 403. +3. The `X-Otari-Code-Execution` header (`auto`, `otari` or `provider`) chooses per + request where the workspace has not pinned. A value outside that vocabulary + is a 400. + +With no `sandbox_url` there is nothing to bring the code to, so a provider +declaration is always forwarded and no policy is read for it. Asking for `otari` +without a sandbox is a 400. The explicit `otari_code_execution` type is always +run by Otari, whatever the executor says. When a claimed declaration runs on the +sandbox, an `otari_code_execution` entry beside it is folded in rather than +refused; when the declaration stays with the provider, the two together are +still refused, because one request cannot address two sandboxes. + +A gateway-run execution is described back in the caller's vocabulary with ids +Otari reserves (`otari_srvtoolu_…`, `otari_ci_…`, `otari_cntr_…`). When a client +echoes such a turn on its next request, a Messages pair is folded into a text +block, and a Responses item into an assistant message, so the model keeps the +code and its output; a provider's own items carry the provider's ids and pass through +untouched. Uploaded files the request references are seeded into the sandbox and +files the code produces come back as stored files, announced in Anthropic's +`code_execution_output` entries by their `file_id`, and on Responses as an +`image` output naming the URL Otari serves each produced image from (under +`public_base_url` where it is set, otherwise under the address the request +arrived on; any other produced file is listed and downloadable by id). A file the *provider's* own sandbox produced stays with the +provider and is served by proxy under its own id. See +[Files and code execution](files.md#files-and-code-execution). A sandbox +session still lives for one request, so a `container` id from a previous turn +addresses the provider's container, not the sandbox. + +In hybrid mode the control plane's policy is consulted only once the decision +already points at the sandbox, so a declaration the provider serves natively is +never turned into a 403 for a workspace the control plane has not enabled. + ### Per-workspace code policy A workspace policy can disable code execution or narrow the deployment limits: @@ -130,6 +201,7 @@ A workspace policy can disable code execution or narrow the deployment limits: - `default_purpose_hint` - allowed tool kinds - an allowed sandbox image +- `executor`, the one field that is a choice rather than a narrowing (see above) Manage it under `/api/v1/workspaces/{workspace_id}/code-execution-policy` or from Tools. A policy diff --git a/pyproject.toml b/pyproject.toml index 16f08d78d6..b6e4a9bd70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,15 @@ ocr = [ s3 = [ "boto3>=1.42.0", ] +# The generic files backend (`FsspecFileStore`): one URL reaches whichever +# filesystem the operator has an fsspec implementation installed for. fsspec +# is already in the tree through any-llm-sdk[all], but as with s3 the extra is +# what build_file_store names and documents, so the backend does not depend on +# that implementation detail. The protocol's own package (gcsfs, adlfs, s3fs, +# paramiko, ...) is the operator's to install. +fsspec = [ + "fsspec>=2024.6.0", +] [dependency-groups] dev = [ @@ -225,6 +234,11 @@ ignore_missing_imports = true module = ["trafilatura.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +# fsspec ships no stubs; the file store drives it through a handful of calls. +module = ["fsspec", "fsspec.*"] +ignore_missing_imports = true + [[tool.mypy.overrides]] # Optional/untyped extraction deps imported at the root (e.g. `import pypdfium2`, # `from markitdown import ...`). Bare root names are required — a `foo.*` pattern diff --git a/src/gateway/AGENTS.md b/src/gateway/AGENTS.md index 2966559ce3..cc6a89dcf4 100644 --- a/src/gateway/AGENTS.md +++ b/src/gateway/AGENTS.md @@ -159,10 +159,17 @@ settlement. ## Tools, MCP, and guardrails -Only `otari_*` tool types run in the gateway; other declarations pass through -to the provider. The tool loop is in `services/mcp_loop.py`, sandbox and search -backends under `services/`, and outbound URL checks in -`services/url_safety.py`. +An `otari_*` tool type always runs in the gateway. A provider-native web-search +type passes through unless `web_search_intercept` is on. A provider-native +code-execution type is decided by the executor (`types/code_execution.py`, +resolved in `api/routes/_tools.py`): a workspace pin wins over everything, the +`X-Otari-Code-Execution` header wins over the deployment default, and `auto` +claims a declaration only when the dispatched provider does not run it +natively. The workspace policy is read once, in the request preamble, and +reused at admission; the same decision says whether a referenced upload is +staged for the sandbox. The tool loop is in +`services/mcp_loop.py`, sandbox and search backends under `services/`, and +outbound URL checks in `services/url_safety.py`. Deployment settings establish available backends. Workspace code-execution and web-search policy can disable or narrow those settings but cannot widen them. diff --git a/src/gateway/api/deps.py b/src/gateway/api/deps.py index 0f91a3172f..d6ccec63a6 100644 --- a/src/gateway/api/deps.py +++ b/src/gateway/api/deps.py @@ -11,7 +11,7 @@ from gateway.auth.models import hash_key from gateway.container import Container -from gateway.core.config import API_KEY_HEADER, X_API_KEY_HEADER, GatewayConfig +from gateway.core.config import API_KEY_HEADER, API_ROOT, X_API_KEY_HEADER, GatewayConfig from gateway.core.database import DATABASE_ERRORS, create_session, get_db from gateway.core.feature import CoreFeature from gateway.core.unit_of_work import UnitOfWork @@ -29,7 +29,9 @@ from gateway.repositories.overview.overview_repository import OverviewRepository from gateway.services.budgets import WorkspaceBudgetDefaultService from gateway.services.dashboard_session_service import SESSION_COOKIE_NAME, resolve_dashboard_session +from gateway.services.file_service import StagedFile from gateway.services.file_store import FileStore +from gateway.services.files import SandboxFileBridge from gateway.services.log_writer import LogWriter from gateway.services.master_key_service import hash_master_key, is_generated_master_key, load_master_key_hash from gateway.services.overview.overview_service import OverviewService @@ -620,6 +622,38 @@ async def get_db_if_needed( yield db +def build_sandbox_file_bridge( + *, + raw_request: Request, + config: GatewayConfig, + db: AsyncSession | None, + user_id: str | None, + workspace_id: uuid.UUID | None, + inputs: list[StagedFile], +) -> SandboxFileBridge | None: + """The file bridge a completion request's sandbox session gets, or ``None``. + + Built by the route once the billed user and workspace are resolved, over the + request's own session. ``None`` in hybrid mode, which has no local database + or file store to hold what a run produces, and when files are disabled. + Produced files are announced under ``public_base_url`` where the deployment + knows its address, and otherwise under the one the request arrived on. + """ + file_store = getattr(raw_request.app.state, "file_store", None) + if db is None or not config.files_enabled or file_store is None or user_id is None or workspace_id is None: + return None + base = (config.public_base_url or str(raw_request.base_url)).rstrip("/") + return SandboxFileBridge( + file_store=file_store, + config=config, + uow=UnitOfWork(db), + user_id=user_id, + workspace_id=workspace_id, + inputs=inputs, + base_url=f"{base}{API_ROOT}/files", + ) + + def get_unit_of_work(db: Annotated[AsyncSession, Depends(get_db)]) -> UnitOfWork: """Return the request's Unit of Work over its session. diff --git a/src/gateway/api/routes/_normalize.py b/src/gateway/api/routes/_normalize.py index dd3ed0d913..741a93984e 100644 --- a/src/gateway/api/routes/_normalize.py +++ b/src/gateway/api/routes/_normalize.py @@ -19,10 +19,57 @@ from fastapi import Request from sqlalchemy.ext.asyncio import AsyncSession +from gateway.api.routes._tools import ( + _extract_code_execution_tool, + decide_code_executor, + first_provider_code_execution_tool, + parse_code_execution_header, + provider_runs_code_natively, + resolve_code_executor_preference, +) from gateway.core.config import GatewayConfig from gateway.log_config import logger from gateway.services.content_normalizer import NormalizationStats, WireFormat, normalize_messages from gateway.services.model_capabilities import resolve_capabilities +from gateway.types.code_execution import CodeExecutor + + +def sandbox_requested( + tools: list[dict[str, Any]] | None, + *, + config: GatewayConfig, + provider: LLMProvider | None, + dialect: str, + code_execution_header: str | None, + workspace_executor: CodeExecutor | None = None, +) -> bool: + """Whether this request's code will run on the gateway's sandbox. + + True for the explicit ``otari_code_execution`` type, and for a provider's own + declaration the executor would bring here (see ``_tools.decide_code_executor``): + a file the request attaches is then staged for the sandbox rather than only + shown to the model. Decided from the same three layers admission uses, the + workspace pin (``workspace_executor``, read once in the preamble), the header + and the deployment default, against the dispatched provider. A header outside + the vocabulary answers false here and is refused at admission. + """ + explicit, remaining = _extract_code_execution_tool(tools) + if explicit is not None: + return True + keyword = first_provider_code_execution_tool(remaining) + if keyword is None or not config.sandbox_configured(): + return False + try: + requested = parse_code_execution_header(code_execution_header) + except ValueError: + return False + preference, _ = resolve_code_executor_preference( + requested=requested, workspace=workspace_executor, deployment=config.effective_code_executor() + ) + native = provider_runs_code_natively( + keyword, provider=provider.value if provider is not None else None, dialect=dialect + ) + return decide_code_executor(preference, sandbox_configured=True, native_available=native) is CodeExecutor.OTARI async def normalize_request_messages( @@ -37,9 +84,14 @@ async def normalize_request_messages( user_id: str | None, instance: str | None = None, workspace_id: uuid.UUID | None = None, + sandbox_requested: bool = False, ) -> tuple[list[dict[str, Any]], NormalizationStats]: """Normalize ``messages`` for the resolved ``provider/model``. + ``sandbox_requested`` is whether the request declared the gateway's + code-execution tool; the normalizer then records referenced uploads on the + stats for the sandbox backend to seed (see ``NormalizationStats.sandbox_inputs``). + No-ops (returns the input untouched) when file understanding is disabled or the provider couldn't be parsed — the downstream provider call surfaces an unknown model with its own status code. @@ -63,6 +115,7 @@ async def normalize_request_messages( file_store=file_store, user_id=user_id, workspace_id=workspace_id, + sandbox_requested=sandbox_requested, ) except Exception as exc: # noqa: BLE001 — never fail the request / leak the reservation logger.warning("content normalization failed; forwarding messages unchanged: %s", exc) diff --git a/src/gateway/api/routes/_pipeline.py b/src/gateway/api/routes/_pipeline.py index 99cae7c126..825fe05d93 100644 --- a/src/gateway/api/routes/_pipeline.py +++ b/src/gateway/api/routes/_pipeline.py @@ -103,14 +103,21 @@ _is_provider_web_search_tool_type, _resolve_sandbox_purpose_hint, _web_search_intercept_enabled, + decide_code_executor, + declares_code_execution, declares_native_web_search, - has_provider_code_execution_tool, + first_provider_code_execution_tool, + native_code_execution_dialect, + parse_code_execution_header, + provider_runs_code_natively, + resolve_code_executor_preference, web_search_max_results_baseline, ) from gateway.core.config import GatewayConfig from gateway.core.database import DATABASE_ERRORS, release_session from gateway.core.env import otari_env from gateway.core.metered_pricing import calculate_metered_cost +from gateway.core.unit_of_work import UnitOfWork from gateway.core.usage import ( cache_read_tokens_of, cache_tokens_in_prompt_of, @@ -142,6 +149,7 @@ refund_reservation, reserve_budget, ) +from gateway.services.files import ProviderFile, SandboxFileBridge, produced_files_for, record_provider_files from gateway.services.log_writer import LogWriter from gateway.services.mcp_client import MCPClientPool from gateway.services.mcp_loop import ( @@ -189,6 +197,7 @@ ) from gateway.services.tenancy.workspace_code_execution_policy_service import ( SERVED_TOOL_NAMES, + ResolvedCodeExecutionPolicy, resolve_workspace_code_execution_policy, ) from gateway.services.tenancy.workspace_mcp_server_service import resolve_workspace_mcp_servers @@ -234,6 +243,7 @@ streaming_generator, ) from gateway.types.attempt import Attempt +from gateway.types.code_execution import CodeExecutor from gateway.types.session_principal import SessionPrincipal ResultT = TypeVar("ResultT") @@ -317,6 +327,15 @@ def record_inline_cost_settlement(outcome: str) -> None: "otari_code_execution tool requested but no sandbox is configured on this gateway. " "Set OTARI_SANDBOX_URL on the gateway, or remove otari_code_execution from `tools`." ) +CODE_EXECUTOR_NOT_CONFIGURED_DETAIL = ( + "code execution was asked to run on this gateway but no sandbox is configured. " + "Set OTARI_SANDBOX_URL on the gateway, or let the provider run it." +) +CODE_EXECUTION_HEADER_INVALID_DETAIL = "X-Otari-Code-Execution must be one of auto, otari, provider" +CODE_EXECUTOR_PINNED_DETAIL = ( + "this workspace's code-execution policy decides who runs code; the X-Otari-Code-Execution " + "header cannot choose otherwise" +) SANDBOX_MCP_CONFLICT_DETAIL = ( "otari_code_execution and mcp_servers cannot be combined in the same request yet; " "pick one. Multi-backend dispatch is a planned refinement." @@ -786,6 +805,7 @@ async def run_tool_loop( on_first_response: Callable[[], None] | None = None, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> ResultT: ... @@ -796,6 +816,7 @@ def open_tool_loop_stream( max_iterations: int, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> AsyncIterator[ChunkT]: ... @@ -866,6 +887,8 @@ def __init__( estimate_inputs: "EstimateInputs | None" = None, request_group_id: str | None = None, organization_id: uuid.UUID | None = None, + code_execution_policy: ResolvedCodeExecutionPolicy | None = None, + code_execution_policy_loaded: bool = False, ) -> None: self.config = config self.db = db @@ -912,6 +935,12 @@ def __init__( # check; callers fall back to `resolve_provider_selector` themselves # in that case, same as before this field existed. self.resolved_provider = resolved_provider + # Standalone-only: the workspace's code-execution policy, read in the + # preamble when the request declares code execution on a deployment with + # a sandbox, so the staging decision and admission read one row once. + # ``loaded`` tells "no row" apart from "not consulted". + self.code_execution_policy = code_execution_policy + self.code_execution_policy_loaded = code_execution_policy_loaded # Standalone-only: the compiled routing plan when `model` named a policy. # `None` for a plain model or an alias, which is what keeps the # single-candidate path byte-identical to what it was. The head attempt is @@ -1642,10 +1671,11 @@ async def resolve_request_context( session_principal: SessionPrincipal | None = None, routing_signal: Callable[[], RoutingSignal] | None = None, normalize_messages: Callable[ - [str, LLMProvider | None, str, str | None, uuid.UUID | None], + [str, LLMProvider | None, str, str | None, uuid.UUID | None, CodeExecutor | None], Awaitable[tuple[int, CompletionUsage | None]], ] | None = None, + tools: list[dict[str, Any]] | None = None, ) -> RequestContext: """Run the shared handler preamble up to (and including) budget pre-debit. @@ -1695,6 +1725,8 @@ async def resolve_request_context( organization_id: uuid.UUID | None = None user_id: str | None = None workspace_id: uuid.UUID | None = None + code_execution_policy: ResolvedCodeExecutionPolicy | None = None + code_execution_policy_loaded = False rate_limit_info: RateLimitInfo | None = None reservation: ReservationHandle | None = None resolved_provider: ResolvedProvider | None = None @@ -2006,18 +2038,34 @@ async def resolve_request_context( # post-normalization size; the top-up rejects if it no longer fits. # Refund on any failure in this setup phase, which the downstream # provider-call settlement does not cover. + # The workspace's code-execution policy, read once here so one decision + # says whether an attachment is staged for the sandbox and, at admission, + # who runs the code. Only a request declaring code execution on a + # deployment with a sandbox has anything to decide. The estimate is + # already reserved, so a read that fails releases it before propagating. + if workspace_id is not None and config.sandbox_configured() and declares_code_execution(tools): + try: + code_execution_policy = await resolve_workspace_code_execution_policy(db, workspace_id) + except Exception: + await refund_reservation(db, reservation) + raise + code_execution_policy_loaded = True + if normalize_messages is not None: try: - # The key's own workspace, not the resolved one: a master-key - # request has no key and resolves to the default workspace, which - # would narrow an operator's file references to it. `fetch_file` - # reads None as "every workspace", matching the /api/v1/files routes. + # The caller's own workspace, not the resolved one: the key's for + # a keyed request, the session's for a Playground one, and None + # only for the master key, which has no key and resolves to the + # default workspace, where narrowing would hide an operator's own + # file references. `fetch_file` reads None as "every workspace", + # matching the /api/v1/files routes. post_chars, vision_usage = await normalize_messages( user_id, gate_impl, gate_model, gate_instance, - api_key.workspace_id if api_key is not None else None, + _caller_workspace_id(api_key, session_principal), + code_execution_policy.executor if code_execution_policy is not None else None, ) # Bill the vision describe side-call before the reservation # top-up: its cost is already incurred by normalize_messages, @@ -2099,6 +2147,8 @@ async def resolve_request_context( db=db, log_writer=log_writer, hybrid_mode=hybrid_mode, + code_execution_policy=code_execution_policy, + code_execution_policy_loaded=code_execution_policy_loaded, route=route, user_token=user_token, api_key_id=api_key_id, @@ -2143,6 +2193,7 @@ def __init__( sandbox_exec_timeout_s: int | None = None, sandbox_session_image: str | None = None, sandbox_allowed_tools: frozenset[str] | None = None, + code_execution_executor: CodeExecutor | None = None, use_web_search: bool, web_search_tool_entry: dict[str, Any] | None, web_search_url: str | None, @@ -2154,10 +2205,14 @@ def __init__( use_web_fetch: bool = False, web_fetch_tool_entry: dict[str, Any] | None = None, web_fetch_policy: DomainPolicy | None = None, + sandbox_files: SandboxFileBridge | None = None, ) -> None: self.config = config self.mcp_server_configs = mcp_server_configs self.use_sandbox = use_sandbox + # The uploads a sandbox session is seeded with and the store its outputs + # land in. None in hybrid mode and when files are disabled. + self.sandbox_files = sandbox_files self.sandbox_tool_entry = sandbox_tool_entry self.sandbox_url = sandbox_url self.sandbox_auth_token = sandbox_auth_token @@ -2171,6 +2226,10 @@ def __init__( # where the request's session is live, and read again at dispatch. self.sandbox_session_image = sandbox_session_image self.sandbox_allowed_tools = sandbox_allowed_tools + # Who was decided to run the request's code-execution declaration, when it + # made one: ``OTARI`` or ``PROVIDER``, never ``AUTO``. ``None`` when the + # request declared no code execution at all. + self.code_execution_executor = code_execution_executor self.use_web_search = use_web_search self.web_search_tool_entry = web_search_tool_entry self.web_search_url = web_search_url @@ -2213,6 +2272,8 @@ def build_sandbox_backend(self) -> SandboxBackend: image=self.sandbox_session_image, allowed_tools=self.sandbox_allowed_tools, tally=self.tally, + files=self.sandbox_files, + files_base_url=self.sandbox_files.base_url if self.sandbox_files is not None else None, ) @property @@ -2238,6 +2299,19 @@ def emit_native_web_search(self) -> bool: """Whether this request should get Anthropic-native server-tool blocks back.""" return self.use_web_search and declares_native_web_search(self.web_search_tool_entry) + @property + def native_code_execution_dialect(self) -> str | None: + """The wire format whose native code-execution blocks this request expects. + + Set only when the gateway runs a declaration made in a provider's own + vocabulary: the caller asked in Anthropic's or OpenAI's words and its SDK + will look for that provider's result shape, so the loop answers in it. + ``None`` for ``otari_code_execution``, whose callers get the plain result. + """ + if not self.use_sandbox: + return None + return native_code_execution_dialect(self.sandbox_tool_entry) + @property def max_web_search_uses(self) -> int | None: """The web-search use cap, when the caller supplied one. @@ -2655,6 +2729,8 @@ async def prepare_gateway_tools( mcp_server_ids: list[uuid.UUID] | None, max_tool_iterations: int | None, tools_header: str | None, + code_execution_header: str | None = None, + sandbox_files: SandboxFileBridge | None = None, ) -> ToolContext: """Guardrails, MCP server-id resolution, and gateway-tool extraction. @@ -2740,25 +2816,23 @@ async def prepare_gateway_tools( raise adapter.error(400, MCP_SERVER_NAME_COLLIDES_WITH_STORED_DETAIL, ErrorKind.INVALID_REQUEST) mcp_servers = (mcp_servers or []) + stored_servers - sandbox_tool_entry, tools_after_sandbox = _extract_code_execution_tool(tools) # Read the effective config value (dashboard override / env / YAML), falling # back to the env var so pure-env deployments are unchanged. A dashboard # override mutates ctx.config, so it hot-applies on the next request. sandbox_url: str | None = ctx.config.sandbox_url or otari_env("SANDBOX_URL") or None - use_sandbox = False - if sandbox_tool_entry is not None: - if sandbox_url is None: - raise adapter.error(400, SANDBOX_NOT_CONFIGURED_DETAIL, ErrorKind.INVALID_REQUEST) - if mcp_servers: - raise adapter.error(400, SANDBOX_MCP_CONFLICT_DETAIL, ErrorKind.INVALID_REQUEST) - # Two sandboxes, one request. Whichever way the gateway resolved it - # silently, half the caller's state would live somewhere they cannot - # address: the gateway sandbox's session is per-request and never - # named on the wire, the provider's is named by a handle the gateway - # would then have to route around. Refuse instead of picking. - if has_provider_code_execution_tool(tools_after_sandbox): - raise adapter.error(400, SANDBOX_PROVIDER_TOOL_CONFLICT_DETAIL, ErrorKind.INVALID_REQUEST) - use_sandbox = True + try: + requested_executor = parse_code_execution_header(code_execution_header) + except ValueError: + raise adapter.error(400, CODE_EXECUTION_HEADER_INVALID_DETAIL, ErrorKind.INVALID_REQUEST) from None + + # Two declarations can ask for code execution: the explicit gateway type, + # and a provider's own keyword. The first is always the gateway's to run. + # The second is the executor's decision, taken below once the workspace's + # policy has had its say, so here it is only found, not claimed. + sandbox_tool_entry, tools_after_sandbox = _extract_code_execution_tool(tools) + provider_code_entry = first_provider_code_execution_tool(tools_after_sandbox) + if sandbox_tool_entry is not None and sandbox_url is None: + raise adapter.error(400, SANDBOX_NOT_CONFIGURED_DETAIL, ErrorKind.INVALID_REQUEST) # Forwarded to the sandbox backend as `Authorization: Bearer`. Only set in # hybrid mode when the backend IS the platform (its URL is under the @@ -2774,94 +2848,125 @@ async def prepare_gateway_tools( # replaced by a *narrower* workspace one below. sandbox_session_image: str | None = ctx.config.effective_sandbox_image() sandbox_allowed_tools: frozenset[str] | None = None - if use_sandbox and ctx.hybrid_mode: - assert ctx.user_token is not None # guaranteed by the hybrid-mode preamble - assert sandbox_tool_entry is not None # use_sandbox implies the entry is present - if sandbox_url is not None and url_targets_platform(sandbox_url, ctx.config.platform.get("base_url")): - sandbox_auth_token = ctx.user_token - - # Platform owns the per-workspace code-exec policy: 403 if the workspace - # has it off, otherwise apply the workspace defaults (per-request values - # win) — the default purpose hint and the loop-iteration ceiling. The - # tools allow-list + exec timeout are re-enforced by the /api/v1/sandbox proxy. - policy = await _resolve_platform_code_execution(config=ctx.config, user_token=ctx.user_token) - # Fail closed on a malformed policy: a non-bool `enabled` is a cross-service - # contract break, not a "disabled" signal — surface it as 502, never run. - enabled = policy.get("enabled") - if not isinstance(enabled, bool): - raise adapter.error(502, MALFORMED_CODE_EXEC_POLICY_DETAIL, ErrorKind.API) - if not enabled: - raise adapter.error(403, SANDBOX_NOT_ENABLED_DETAIL, ErrorKind.PERMISSION) - if not sandbox_tool_entry.get("purpose_hint") and policy.get("default_purpose_hint"): - sandbox_tool_entry["purpose_hint"] = policy["default_purpose_hint"] - resolved_iters = policy.get("max_iterations") - # `bool` is an `int` subclass — exclude it so a JSON `true` isn't read as 1. - if isinstance(resolved_iters, int) and not isinstance(resolved_iters, bool) and resolved_iters > 0: - sandbox_max_iterations = resolved_iters - elif use_sandbox: - # Standalone's counterpart to the resolve above: the policy is a row in - # this deployment's own database, read here at admission because this is - # where the request's session is live and where the values it carries - # (the hint, the two ceilings) still have somewhere to land. The - # workspace comes off the key that authenticated the request, never off - # a header; a master-key request resolves to the deployment's default - # workspace, so an operator who has narrowed that workspace is narrowed - # by it too (`services/workspace_scope.py`). - # - # No row means no narrowing, which is what keeps a deployment that has - # configured nothing per-workspace behaving exactly as it did. A row may - # only narrow: it refuses the tool, lowers the ceilings (applied with - # `min` further down and in `ToolContext`), and fills in a hint the - # request did not give. It can never turn on a sandbox the deployment - # has not configured, which the missing-URL 400 above already settled. - assert sandbox_tool_entry is not None # use_sandbox implies the entry is present - if ctx.db is None or ctx.workspace_id is None: - # Fail closed. Both are invariants on this path today (a standalone - # request with no session is refused with `DB_UNAVAILABLE_DETAIL` - # before this, and `resolve_workspace_id` always answers, falling - # back to the default workspace), so this is unreachable, which is - # exactly why it refuses rather than falling through. What this arm - # guards is a *veto*: skipping it would serve code execution to a - # workspace whose row says `enabled=False`, silently, on the day one - # of those invariants stops holding. `_resolve_mcp_server_ids` - # refuses at the identical condition. - raise adapter.error(500, CODE_EXEC_POLICY_UNRESOLVABLE_DETAIL, ErrorKind.API) - workspace_policy = await resolve_workspace_code_execution_policy(ctx.db, ctx.workspace_id) - if workspace_policy is not None: - if not workspace_policy.enabled: + code_execution_executor: CodeExecutor | None = None + code_execution_policy: ResolvedCodeExecutionPolicy | None = None + use_sandbox = False + + # With no sandbox configured there is nothing to bring a provider's keyword + # to, so it is forwarded exactly as it always was and no policy is read for + # it: a deployment without a sandbox is a deployment the executor does not + # touch. (The explicit type was refused above.) + if sandbox_url is not None and (sandbox_tool_entry is not None or provider_code_entry is not None): + deployment_executor = ctx.config.effective_code_executor() + native_available = provider_runs_code_natively( + provider_code_entry, provider=_dispatch_provider_name(ctx), dialect=adapter.name + ) + if ctx.hybrid_mode: + # The platform's resolve answers for a workspace that may run code + # *here*, and refuses one that may not. So it is asked only once the + # decision already points here: asking it about a keyword the + # provider is about to serve natively would turn that request into a + # 403 for every workspace otari.ai has not enabled for Otari's + # sandbox, which today is most of them. The cost is that a + # platform-side pin cannot pull a natively served keyword here; that + # lands with the platform half of this work. + provisional, _ = resolve_code_executor_preference( + requested=requested_executor, workspace=None, deployment=deployment_executor + ) + provisional_executor = decide_code_executor( + provisional, sandbox_configured=True, native_available=native_available + ) + if sandbox_tool_entry is not None or provisional_executor is CodeExecutor.OTARI: + code_execution_policy = await _hybrid_code_execution_policy(adapter, ctx) + else: + code_execution_policy = await _standalone_code_execution_policy(adapter, ctx) + + executor_preference, executor_conflict = resolve_code_executor_preference( + requested=requested_executor, + workspace=code_execution_policy.executor if code_execution_policy is not None else None, + deployment=deployment_executor, + ) + # Only a provider-named keyword is the pin's to decide. An explicit + # ``otari_code_execution`` runs here whatever the executor says (it + # names no provider tool to forward), so refusing the header over it + # would 403 a request that the same body without the header serves. + if executor_conflict and provider_code_entry is not None: + raise adapter.error(403, CODE_EXECUTOR_PINNED_DETAIL, ErrorKind.PERMISSION) + code_execution_executor = decide_code_executor( + executor_preference, sandbox_configured=True, native_available=native_available + ) + + if provider_code_entry is not None and code_execution_executor is CodeExecutor.OTARI: + # The keyword is claimed: it leaves ``tools[]`` and becomes the entry + # the sandbox is configured from, so the caller's declaration shape + # (and with it the native result blocks it expects back) is kept. An + # explicit ``otari_code_execution`` beside it is the same request said + # twice; it is folded in rather than refused, contributing only the + # hint the keyword itself cannot carry. + claimed, tools_after_sandbox = _extract_code_execution_tool(tools_after_sandbox, intercept=True) + assert claimed is not None # ``provider_code_entry`` was found in the same list + if sandbox_tool_entry is not None and not claimed.get("purpose_hint"): + if sandbox_tool_entry.get("purpose_hint"): + claimed["purpose_hint"] = sandbox_tool_entry["purpose_hint"] + sandbox_tool_entry = claimed + elif provider_code_entry is not None and sandbox_tool_entry is not None: + # Two sandboxes, one request. Whichever way the gateway resolved it + # silently, half the caller's state would live somewhere they cannot + # address: the gateway sandbox's session is per-request and never + # named on the wire, the provider's is named by a handle the gateway + # would then have to route around. Refuse instead of picking. + raise adapter.error(400, SANDBOX_PROVIDER_TOOL_CONFLICT_DETAIL, ErrorKind.INVALID_REQUEST) + use_sandbox = sandbox_tool_entry is not None + elif requested_executor is CodeExecutor.OTARI and provider_code_entry is not None: + raise adapter.error(400, CODE_EXECUTOR_NOT_CONFIGURED_DETAIL, ErrorKind.INVALID_REQUEST) + + if use_sandbox: + assert sandbox_tool_entry is not None + assert sandbox_url is not None + if mcp_servers: + raise adapter.error(400, SANDBOX_MCP_CONFLICT_DETAIL, ErrorKind.INVALID_REQUEST) + if ctx.hybrid_mode: + assert ctx.user_token is not None # guaranteed by the hybrid-mode preamble + if url_targets_platform(sandbox_url, ctx.config.platform.get("base_url")): + sandbox_auth_token = ctx.user_token + # The policy narrows what the deployment allows and never widens it: a + # veto, two ceilings applied with ``min`` further down and in + # ``ToolContext``, a hint that fills in only when the request gave none, + # a tool list that only removes, and an image that must be one the + # operator curated. No policy means no narrowing, which is what keeps a + # deployment that has configured nothing per-workspace behaving exactly + # as it did. The workspace came off the key that authenticated the + # request (standalone) or off the caller's token (hybrid), never off a + # header. + if code_execution_policy is not None: + if not code_execution_policy.enabled: raise adapter.error(403, SANDBOX_NOT_ENABLED_DETAIL, ErrorKind.PERMISSION) - if not sandbox_tool_entry.get("purpose_hint") and workspace_policy.default_purpose_hint: - sandbox_tool_entry["purpose_hint"] = workspace_policy.default_purpose_hint - sandbox_max_iterations = workspace_policy.max_iterations - sandbox_exec_timeout_s = workspace_policy.exec_timeout_s - if workspace_policy.tools is not None: + if not sandbox_tool_entry.get("purpose_hint") and code_execution_policy.default_purpose_hint: + sandbox_tool_entry["purpose_hint"] = code_execution_policy.default_purpose_hint + sandbox_max_iterations = code_execution_policy.max_iterations + sandbox_exec_timeout_s = code_execution_policy.exec_timeout_s + if code_execution_policy.tools is not None: # An intersection, so it only ever removes. Nothing left to run # is refused here rather than handed to a backend advertising an # empty tool list, which the model would answer by not calling # the tool at all: an unusable-but-successful request is the - # failure mode a policy exists to make loud. - # - # Against ``SERVED_TOOL_NAMES``, which is the same set - # ``_require_runnable_tools`` refuses a write against, so the - # storable rule and the admission rule are one rule. Naming - # ``CODE_EXECUTION_TOOL_NAME`` here instead would agree only - # while that tuple has one entry: the day a second tool kind - # joins it, a policy naming only that one becomes storable and - # then 403s on every request, which is the state both guards - # exist to prevent. - if not set(workspace_policy.tools) & set(SERVED_TOOL_NAMES): + # failure mode a policy exists to make loud. Against + # ``SERVED_TOOL_NAMES``, the same set ``_require_runnable_tools`` + # refuses a write against, so the storable rule and the + # admission rule are one rule. + if not code_execution_policy.tools & set(SERVED_TOOL_NAMES): raise adapter.error(403, SANDBOX_TOOLS_EXCLUDED_DETAIL, ErrorKind.PERMISSION) - sandbox_allowed_tools = workspace_policy.tools - if workspace_policy.image is not None: + sandbox_allowed_tools = code_execution_policy.tools + if code_execution_policy.image is not None: # Re-checked against the operator's list, which the write # already checked once: an operator may shrink that list after # a workspace pinned from it, and running the un-curated image # anyway is precisely the supply-chain hole the column is # guarded for. Refuse rather than quietly serve the deployment # default, so the workspace learns its pin is dead. - if workspace_policy.image not in ctx.config.pinnable_sandbox_images(): + if code_execution_policy.image not in ctx.config.pinnable_sandbox_images(): raise adapter.error(403, SANDBOX_IMAGE_NOT_ALLOWED_DETAIL, ErrorKind.PERMISSION) - sandbox_session_image = workspace_policy.image + sandbox_session_image = code_execution_policy.image web_search_url: str | None = ctx.config.web_search_url or otari_env("WEB_SEARCH_URL") or None # Interception (claiming the provider-named web_search keywords) is opt-in and @@ -3038,6 +3143,7 @@ async def prepare_gateway_tools( sandbox_exec_timeout_s=sandbox_exec_timeout_s, sandbox_session_image=sandbox_session_image, sandbox_allowed_tools=sandbox_allowed_tools, + code_execution_executor=code_execution_executor, use_web_search=use_web_search, web_search_tool_entry=web_search_tool_entry, web_search_url=web_search_url, @@ -3053,9 +3159,155 @@ async def prepare_gateway_tools( sandbox_max_iterations or MAX_TOOL_ITERATIONS_CAP, ), tools_header=tools_header, + sandbox_files=sandbox_files if use_sandbox else None, ) +def _caller_workspace_id(api_key: APIKey | None, session_principal: SessionPrincipal | None) -> uuid.UUID | None: + """The workspace a file reference is resolved in: the key's, else the session's, else every one. + + ``None`` is the master key alone. A Playground request has no key but does + have a workspace it proved membership of, and a member's file uploaded + through a key in another of their workspaces must not resolve here. + """ + if api_key is not None: + return api_key.workspace_id + if session_principal is not None: + return session_principal.workspace_id + return None + + +def _dispatch_provider_name(ctx: RequestContext) -> str | None: + """The any-llm provider the request's first attempt dispatches to, if known. + + Standalone resolved it in the preamble; hybrid has it on the platform's first + attempt. ``None`` when neither could say, which the executor reads as "not + natively served", the answer that brings the code here rather than forwarding + a declaration nobody may honor. + """ + if ctx.resolved_provider is not None: + return ctx.resolved_provider.provider.value + if ctx.route is not None and ctx.route.attempts: + return ctx.route.attempts[0].provider + return None + + +async def _standalone_code_execution_policy( + adapter: FormatAdapter[Any, Any], + ctx: RequestContext, +) -> ResolvedCodeExecutionPolicy | None: + """The request's workspace policy: the preamble's read where it made one, else read here. + + The workspace comes off the key that authenticated the request, never off a + header; a master-key request resolves to the deployment's default workspace, + so an operator who has narrowed that workspace is narrowed by it too + (``services/workspace_scope.py``). ``None`` means no row and no narrowing. + + Fails closed when the session or the workspace is missing. Both are + invariants on this path today (a standalone request with no session is + refused with ``DB_UNAVAILABLE_DETAIL`` before this, and ``resolve_workspace_id`` + always answers), so this is unreachable, which is exactly why it refuses + rather than falling through: what it guards is a *veto*, and skipping it + would serve code execution to a workspace whose row says ``enabled=False`` + on the day one of those invariants stops holding. + """ + if ctx.code_execution_policy_loaded: + return ctx.code_execution_policy + if ctx.db is None or ctx.workspace_id is None: + raise adapter.error(500, CODE_EXEC_POLICY_UNRESOLVABLE_DETAIL, ErrorKind.API) + return await resolve_workspace_code_execution_policy(ctx.db, ctx.workspace_id) + + +async def _hybrid_code_execution_policy( + adapter: FormatAdapter[Any, Any], + ctx: RequestContext, +) -> ResolvedCodeExecutionPolicy: + """The platform's answer for the caller's workspace, in the standalone shape. + + The platform owns the per-workspace policy: ``enabled`` is its veto, the + hint and the loop ceiling its defaults (per-request values win), and + ``executor`` its pin where it sends one. A malformed ``enabled`` is a + cross-service contract break, not a "disabled" signal, so it surfaces as a + 502 and never runs. The other fields are read leniently: an unusable one + narrows nothing rather than failing a request over a default. + + The tool allow-list and the execution timeout the payload also carries are + deliberately not applied here: the platform's sandbox proxy re-enforces both + on every call, and enforcing them twice would let this gateway refuse a tool + the platform admits. + """ + assert ctx.user_token is not None # guaranteed by the hybrid-mode preamble + policy = await _resolve_platform_code_execution(config=ctx.config, user_token=ctx.user_token) + enabled = policy.get("enabled") + if not isinstance(enabled, bool): + raise adapter.error(502, MALFORMED_CODE_EXEC_POLICY_DETAIL, ErrorKind.API) + hint = policy.get("default_purpose_hint") + return ResolvedCodeExecutionPolicy( + enabled=enabled, + default_purpose_hint=hint if isinstance(hint, str) and hint else None, + max_iterations=_positive_int(policy.get("max_iterations")), + exec_timeout_s=None, + image=None, + tools=None, + executor=CodeExecutor.parse(policy.get("executor")), + ) + + +def _positive_int(value: Any) -> int | None: + """``value`` when it is a positive integer, else ``None``. + + ``bool`` is an ``int`` subclass and is excluded so a JSON ``true`` is not read as 1. + """ + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + return None + return int(value) + + +def _implementation_for(ctx: RequestContext, instance: str) -> LLMProvider | None: + """The any-llm provider behind the ``instance`` that served the request, if known.""" + if ctx.resolved_provider is not None and ctx.resolved_provider.instance == instance: + return ctx.resolved_provider.provider + candidates = [attempt.provider for attempt in ctx.plan.attempts if attempt.instance == instance] if ctx.plan else [] + for name in [*candidates, instance]: + try: + return LLMProvider(name) + except ValueError: + continue + return None + + +async def _record_provider_files(ctx: RequestContext, files: list[ProviderFile], *, instance: Any) -> None: + """Record the files a provider's own sandbox produced for this request, so ``/v1/files`` serves them. + + Standalone only, and only for a request billed to a user and workspace: the + row is what scopes a later download to them. ``instance`` is the configured + entry that served, whose credential is the one that can read the files back. + """ + if ctx.db is None or not files or not ctx.user_id or ctx.workspace_id is None or not isinstance(instance, str): + return + provider = _implementation_for(ctx, instance) + if provider is None: + return + await record_provider_files( + UnitOfWork(ctx.db), + files, + provider=provider.value, + provider_instance=instance, + user_id=ctx.user_id, + workspace_id=ctx.workspace_id, + config=ctx.config, + ) + + +async def _collecting_produced_files( + stream: AsyncIterator[ChunkT], dialect: str, sink: list[ProviderFile] +) -> AsyncIterator[ChunkT]: + """Forward ``stream`` unchanged, noting the provider-held files its events cite.""" + async for chunk in stream: + sink.extend(produced_files_for(dialect, chunk)) + yield chunk + + async def _require_tool_pricing( adapter: FormatAdapter[Any, Any], ctx: RequestContext, @@ -3621,9 +3873,23 @@ def _loop_options(tool_ctx: ToolContext) -> dict[str, Any]: The budget itself travels, not the number it was built from: every attempt of one request draws on the same one. """ - if tool_ctx.web_search_budget is None: + options: dict[str, Any] = {} + if tool_ctx.web_search_budget is not None: + options["web_search_budget"] = tool_ctx.web_search_budget + return options + + +def _sandbox_loop_options(adapter: FormatAdapter[Any, Any], tool_ctx: ToolContext) -> dict[str, Any]: + """Sandbox-loop kwargs, presence-encoded like :func:`_loop_options`. + + The native flag travels only when this request's declaration is in the + adapter's own vocabulary: an Anthropic-dated keyword on Messages, OpenAI's + ``code_interpreter`` on Responses. A caller who said ``otari_code_execution`` + gets the plain result it always has. + """ + if tool_ctx.native_code_execution_dialect != adapter.name: return {} - return {"web_search_budget": tool_ctx.web_search_budget} + return {"emit_native_code_execution": True} async def dispatch_non_stream( @@ -3648,7 +3914,13 @@ async def dispatch_non_stream( if tool_ctx.use_sandbox: async with tool_ctx.build_sandbox_backend() as backend: kwargs = adapter.inject_hints(call_kwargs, backend.purpose_hints(), header=tool_ctx.tools_header) - return await adapter.run_tool_loop(kwargs, backend, tool_ctx.max_tool_iterations, on_first_response) + return await adapter.run_tool_loop( + kwargs, + backend, + tool_ctx.max_tool_iterations, + on_first_response, + **_sandbox_loop_options(adapter, tool_ctx), + ) assert tool_ctx.use_web_search or tool_ctx.use_web_fetch async with tool_ctx.build_web_retrieval_backend() as web_backend: @@ -3694,6 +3966,7 @@ async def _eager_backend_stream( tool_ctx.max_tool_iterations, emit_native_web_search=tool_ctx.emit_native_web_search, **_loop_options(tool_ctx), + **(_sandbox_loop_options(adapter, tool_ctx) if tool_ctx.use_sandbox else {}), ): yield event finally: @@ -3830,6 +4103,7 @@ def build_streaming_response( attribution: RoutingAttribution | None = None, tool_tally: ToolUsageTally | None = None, workspace_id: uuid.UUID | None = None, + on_settled: Callable[[], Awaitable[None]] | None = None, ) -> StreamingResponse: """Wrap an already-opened upstream stream in an SSE response. @@ -3850,6 +4124,10 @@ def build_streaming_response( * ``on_error``: report/log the failure and refund the reservation. * ``on_incomplete``: client disconnected mid-stream; refund so the reservation does not leak. + + ``on_settled`` runs after a standalone stream has settled, complete or + without usage, for bookkeeping that needs the whole response to have + arrived (the files a provider's sandbox produced). Never on an error. """ platform_active = platform_correlation_id is not None first_chunk_at: float | None = None @@ -3896,6 +4174,8 @@ async def _on_complete(usage_data: CompletionUsage) -> SettledCost | None: await reconcile_reservation( db, reservation, actual_cost or Decimal(0), actual_tokens=_settled_tokens(usage_data) ) + if on_settled is not None: + await on_settled() return None async def _on_no_usage() -> None: @@ -3920,6 +4200,8 @@ async def _on_no_usage() -> None: if settlement is not None: record_inline_cost_settlement("unattached") return + if on_settled is not None: + await on_settled() if db is None or log_writer is None or reservation is None: return policy = config.stream_missing_usage_policy @@ -4257,6 +4539,13 @@ async def _absorbed(attempt: Attempt, exc: BaseException, _total: int) -> None: logger.error("Stream creation failed for %s:%s: %s", provider, model, exc) raise adapter.provider_error(exc) from exc + produced: list[ProviderFile] = [] + if ctx.db is not None: + stream = _collecting_produced_files(stream, adapter.name, produced) + + async def _record_produced() -> None: + await _record_provider_files(ctx, produced, instance=provider) + return build_streaming_response( adapter=adapter, stream=stream, @@ -4264,6 +4553,7 @@ async def _absorbed(attempt: Attempt, exc: BaseException, _total: int) -> None: model=model, config=ctx.config, db=ctx.db, + on_settled=_record_produced if ctx.db is not None else None, log_writer=ctx.log_writer, api_key_id=ctx.api_key_id, user_id=ctx.user_id, @@ -4410,6 +4700,7 @@ async def _build_for_attempt(attempt: ResolvedAttempt) -> AsyncIterator[ChunkT]: tool_ctx.max_tool_iterations, emit_native_web_search=tool_ctx.emit_native_web_search, **_loop_options(tool_ctx), + **(_sandbox_loop_options(adapter, tool_ctx) if tool_ctx.use_sandbox else {}), ) # See run_platform_non_stream: BackgroundTasks only run after a successful @@ -4922,6 +5213,7 @@ async def _absorbed(attempt: Attempt, exc: BaseException, _total: int) -> None: await reconcile_reservation( ctx.db, ctx.reservation, actual_cost or Decimal(0), actual_tokens=_settled_tokens(usage_data) ) + await _record_provider_files(ctx, produced_files_for(adapter.name, result), instance=provider) if display_model is not None: relabel_model(result, display_model) return result diff --git a/src/gateway/api/routes/_tools.py b/src/gateway/api/routes/_tools.py index c86c58f7ba..d4681b35bf 100644 --- a/src/gateway/api/routes/_tools.py +++ b/src/gateway/api/routes/_tools.py @@ -7,22 +7,24 @@ handling regardless of wire shape. The explicit ``otari_*`` tool types always trigger gateway-side execution. -Every other tool type — the short forms (``code_execution`` / ``web_search``) -and the provider-native keywords (``code_interpreter`` / -``code_execution_`` / ``web_search_``) — is left untouched in -``tools[]`` and forwarded to the upstream provider, which runs it server-side. -For code execution the keyword alone says who runs it: no flag, no env toggle. - -Web search has one opt-in exception. A client that cannot be told to say -``otari_web_search`` (Claude Code, the Anthropic SDK, anything speaking a -provider's native vocabulary) would otherwise never reach a configured gateway -backend. Setting ``web_search_intercept`` makes the gateway also claim the -provider-named web-search keywords, so those clients work unchanged. It is off -by default because turning it on silently takes a search away from a provider -that would have run it (see ``docs/tools.md``). An OpenAI ``function`` named -``web_search`` is deliberately *not* claimed even then: that is a caller's own -tool, and hijacking it means the caller's handler never fires and it never gets -back a ``tool_call`` it can dispatch. +A provider-native web-search keyword (``web_search`` / ``web_search_``) +is left untouched in ``tools[]`` and forwarded to the upstream provider unless +``web_search_intercept`` is on. It is off by default because turning it on +silently takes a search away from a provider that would have run it (see +``docs/tools.md``). An OpenAI ``function`` named ``web_search`` is deliberately +*not* claimed even then: that is a caller's own tool, and hijacking it means the +caller's handler never fires and it never gets back a ``tool_call`` it can +dispatch. + +A provider-native code-execution keyword (``code_execution``, +``code_interpreter``, ``code_execution_``) is decided by the request's +**executor** instead (:class:`gateway.types.code_execution.CodeExecutor`): the +provider, Otari's sandbox, or ``auto``, which picks the provider only when it +runs that tool natively for the dispatched model. ``auto`` is the default, and +it is what lets a request written against a frontier model's own sandbox keep +working when the model is swapped for one that has none. The deployment sets +the default, a workspace policy may pin a value, and :data:`CODE_EXECUTION_HEADER` +chooses per request where the workspace has not. """ from __future__ import annotations @@ -43,10 +45,17 @@ WebRetrievalCounter, ) from gateway.services.web_retrieval_policy import DomainPolicy +from gateway.types.code_execution import CodeExecutor if TYPE_CHECKING: from gateway.core.config import GatewayConfig +# Per-request choice of who runs a provider-native code-execution declaration. +# A header rather than a body field so the body stays the untouched payload a +# provider's own SDK sends; every SDK can add a default header without a code +# change. One of ``CodeExecutor``'s values, case-insensitive. +CODE_EXECUTION_HEADER = "X-Otari-Code-Execution" + class Tool(StrEnum): """Gateway-managed tool types — the only ``type`` values the gateway runs @@ -122,10 +131,9 @@ def declares_native_web_search(tool_entry: dict[str, Any] | None) -> bool: def _is_code_execution_tool_type(type_value: Any) -> bool: """Recognize the explicit gateway-managed code-execution tool type. - Matches only ``"otari_code_execution"``. Provider-named keywords - (``"code_execution"``, ``"code_interpreter"``, ``"code_execution_"``) - are *not* matched — they pass through unchanged to the upstream provider, - which runs the code in its own native sandbox. + Matches only ``"otari_code_execution"``. The provider-named keywords are + :func:`_is_provider_code_execution_tool_type`'s, and whether the gateway + claims one is the executor's decision, not the keyword's. """ if not isinstance(type_value, str): return False @@ -143,28 +151,148 @@ def _is_web_fetch_tool_type(type_value: Any) -> bool: # here, mirroring the web-search keywords above. _BARE_CODE_EXECUTION_TYPES = frozenset({"code_execution", "code_interpreter"}) _VERSIONED_CODE_EXECUTION_PREFIX = "code_execution_" - - -def has_provider_code_execution_tool(tools: list[dict[str, Any]] | None) -> bool: - """Whether ``tools`` still asks the provider to run code in its own sandbox. +_OPENAI_CODE_INTERPRETER_TYPE = "code_interpreter" +# The one provider each native vocabulary belongs to, and the wire format it is +# native in. Anthropic's dated ``code_execution_`` is a Messages server +# tool; OpenAI's ``code_interpreter`` is a Responses built-in tool. Neither has a +# native form on Chat Completions, and the bare ``code_execution`` short form is +# nobody's, so a request declaring it is never natively served and ``auto`` +# always runs it here. +_NATIVE_CODE_EXECUTION: dict[str, tuple[str, str]] = { + _VERSIONED_CODE_EXECUTION_PREFIX: ("anthropic", "messages"), + _OPENAI_CODE_INTERPRETER_TYPE: ("openai", "responses"), +} + + +def _is_provider_code_execution_tool_type(type_value: Any) -> bool: + """Recognize a provider-named code-execution keyword. Matched on the tool ``type`` alone, never on a caller's ``function`` named ``code_execution``: that is the caller's own tool, the same carve-out :func:`_is_web_search_tool_type` makes for a function named ``web_search``. + Does not match ``otari_code_execution``, which + :func:`_is_code_execution_tool_type` owns. + """ + if not isinstance(type_value, str): + return False + return type_value in _BARE_CODE_EXECUTION_TYPES or type_value.startswith(_VERSIONED_CODE_EXECUTION_PREFIX) + + +def _is_any_code_execution_tool_type(type_value: Any) -> bool: + """The gateway-managed type or a provider-named keyword.""" + return _is_code_execution_tool_type(type_value) or _is_provider_code_execution_tool_type(type_value) + + +def declares_code_execution(tools: list[dict[str, Any]] | None) -> bool: + """Whether ``tools`` asks for code execution in any vocabulary, the gateway's or a provider's.""" + return any( + isinstance(entry, dict) and _is_any_code_execution_tool_type(entry.get("type")) for entry in tools or [] + ) + + +def first_provider_code_execution_tool(tools: list[dict[str, Any]] | None) -> dict[str, Any] | None: + """The first provider-named code-execution entry in ``tools``, left in place.""" + for entry in tools or []: + if isinstance(entry, dict) and _is_provider_code_execution_tool_type(entry.get("type")): + return entry + return None - Called on what is left after :func:`_extract_code_execution_tool` has taken - the gateway-managed entry, so a true answer means the request asks two - separate sandboxes to run code. + +def native_code_execution_dialect(tool_entry: dict[str, Any] | None) -> str | None: + """The wire format whose native result blocks the caller expects, or ``None``. + + ``"messages"`` for Anthropic's dated keyword, which is what the Anthropic SDK + and Claude Code send and what makes them expect ``server_tool_use`` and + ``code_execution_tool_result`` blocks back. ``"responses"`` for OpenAI's + ``code_interpreter``, whose callers expect a ``code_interpreter_call`` item. + ``None`` for ``otari_code_execution`` and for the bare ``code_execution`` + short form, neither of which implies a native response shape, so those + callers keep receiving the plain tool-loop result they always have. """ - if not tools: + type_value = tool_entry.get("type") if tool_entry else None + if not isinstance(type_value, str): + return None + if type_value.startswith(_VERSIONED_CODE_EXECUTION_PREFIX): + return _NATIVE_CODE_EXECUTION[_VERSIONED_CODE_EXECUTION_PREFIX][1] + if type_value == _OPENAI_CODE_INTERPRETER_TYPE: + return _NATIVE_CODE_EXECUTION[_OPENAI_CODE_INTERPRETER_TYPE][1] + return None + + +def provider_runs_code_natively(tool_entry: dict[str, Any] | None, *, provider: str | None, dialect: str) -> bool: + """Whether the dispatched provider would run this declaration in its own sandbox. + + True only when the keyword is the provider's own vocabulary *and* the request + arrived in the wire format that vocabulary is native to: Anthropic's dated + keyword on Messages against an Anthropic model, OpenAI's ``code_interpreter`` + on Responses against an OpenAI model. Everything else (a Mistral model asked + in Anthropic's words, any keyword on Chat Completions, an unknown provider) + is a declaration the provider cannot honor, which is exactly when ``auto`` + brings the code here. + """ + if provider is None or tool_entry is None: + return False + type_value = tool_entry.get("type") + if not isinstance(type_value, str): return False - for entry in tools: - type_value = entry.get("type") if isinstance(entry, dict) else None - if not isinstance(type_value, str): - continue - if type_value in _BARE_CODE_EXECUTION_TYPES or type_value.startswith(_VERSIONED_CODE_EXECUTION_PREFIX): - return True - return False + key = _VERSIONED_CODE_EXECUTION_PREFIX if type_value.startswith(_VERSIONED_CODE_EXECUTION_PREFIX) else type_value + native = _NATIVE_CODE_EXECUTION.get(key) + return native is not None and native == (provider.lower(), dialect) + + +def parse_code_execution_header(value: str | None) -> CodeExecutor | None: + """The executor a request asked for, ``None`` when it asked for none. + + Raises ``ValueError`` for a value outside the vocabulary: a misspelled header + is a caller mistake to report, not a default to fall back to. + """ + if value is None or not value.strip(): + return None + executor = CodeExecutor.parse(value) + if executor is None: + msg = f"{CODE_EXECUTION_HEADER} must be one of {', '.join(e.value for e in CodeExecutor)}" + raise ValueError(msg) + return executor + + +def resolve_code_executor_preference( + *, + requested: CodeExecutor | None, + workspace: CodeExecutor | None, + deployment: CodeExecutor, +) -> tuple[CodeExecutor, bool]: + """Compose the three layers into one preference, and say whether they clashed. + + A workspace pin wins over the request, and the request wins over the + deployment default: the workspace's owner set the pin for a billing or data + reason a caller may not override, while the deployment default is only what + applies when nobody closer to the request said otherwise. The second value is + true when the request asked for something the workspace pinned away, so the + caller can refuse out loud rather than silently run elsewhere. + """ + if workspace is not None: + return workspace, requested is not None and requested != workspace + return requested or deployment, False + + +def decide_code_executor( + preference: CodeExecutor, + *, + sandbox_configured: bool, + native_available: bool, +) -> CodeExecutor: + """Turn a preference into who runs the code: ``OTARI`` or ``PROVIDER``. + + ``AUTO`` prefers the provider when it serves the tool natively, and with no + sandbox configured it also leaves the provider in charge, because there is + nothing to bring the code to; an explicit ``OTARI`` is returned as asked so + the caller can refuse it with the missing-sandbox detail instead. + """ + if preference is not CodeExecutor.AUTO: + return preference + if native_available or not sandbox_configured: + return CodeExecutor.PROVIDER + return CodeExecutor.OTARI # Gateway-internal fields the provider SDKs (any-llm, anthropic, openai, …) @@ -265,14 +393,36 @@ def _extract_first_matching_tool( def _extract_code_execution_tool( tools: list[dict[str, Any]] | None, + *, + intercept: bool = False, ) -> tuple[dict[str, Any] | None, list[dict[str, Any]] | None]: - """Pull the first ``{"type": "otari_code_execution"}`` entry out of ``tools``. + """Pull the first gateway-run code-execution entry out of ``tools``. + + With ``intercept`` off (the default) only the explicit + ``{"type": "otari_code_execution"}`` is extracted; provider-named keywords + stay in ``tools[]`` and reach the upstream provider unchanged. With it on, + which is what an executor decision of ``OTARI`` means, the provider-named + keywords are claimed too, so a client speaking a provider's vocabulary + reaches the gateway's sandbox. + """ + predicate = _is_any_code_execution_tool_type if intercept else _is_code_execution_tool_type + return _extract_first_matching_tool(tools, predicate) + - Only the explicit gateway-managed type is extracted (and run in the - gateway sandbox). Provider-named code-execution keywords stay in - ``tools[]`` and reach the upstream provider unchanged. +def code_execution_declaration_forms(config: GatewayConfig | None = None) -> list[str]: + """Every ``tools[].type`` this deployment may route to the sandbox. + + Advertised by ``GET /api/v1/tools``. The provider-named keywords appear + unless the deployment's executor is ``provider``; under ``auto`` they are + routed here only for a model whose provider does not run them natively, + which the listing cannot say per model, so it lists the forms the gateway + is prepared to claim. """ - return _extract_first_matching_tool(tools, _is_code_execution_tool_type) + forms = [str(Tool.CODE_EXECUTION)] + executor = config.effective_code_executor() if config is not None else CodeExecutor.AUTO + if executor is not CodeExecutor.PROVIDER: + forms += sorted(_BARE_CODE_EXECUTION_TYPES) + [f"{_VERSIONED_CODE_EXECUTION_PREFIX}"] + return forms def _extract_web_search_tool( diff --git a/src/gateway/api/routes/chat.py b/src/gateway/api/routes/chat.py index 57668c4804..8236fda48b 100644 --- a/src/gateway/api/routes/chat.py +++ b/src/gateway/api/routes/chat.py @@ -14,9 +14,15 @@ from pydantic import Field, field_validator from sqlalchemy.ext.asyncio import AsyncSession -from gateway.api.deps import ModelProviderPortDep, get_config, get_db_if_needed, get_log_writer +from gateway.api.deps import ( + ModelProviderPortDep, + build_sandbox_file_bridge, + get_config, + get_db_if_needed, + get_log_writer, +) from gateway.api.routes._helpers import latest_user_text, routing_signal_from_messages -from gateway.api.routes._normalize import normalize_request_messages +from gateway.api.routes._normalize import normalize_request_messages, sandbox_requested from gateway.api.routes._pipeline import ( NO_RESOLVABLE_PROVIDER_DETAIL, PROVIDER_ERROR_DETAIL, @@ -38,7 +44,7 @@ ) from gateway.api.routes._platform import ResolvedAttempt, SettledCost from gateway.api.routes._schema_derive import SESSION_LABEL_DESC, SESSION_LABEL_MAX_LENGTH, derive_request_base -from gateway.api.routes._tools import _strip_gateway_fields +from gateway.api.routes._tools import CODE_EXECUTION_HEADER, _strip_gateway_fields from gateway.core.config import GatewayConfig from gateway.core.usage import GatewayUsage from gateway.core.usage_source import PLAYGROUND_USAGE_ENDPOINT @@ -46,6 +52,7 @@ from gateway.models.guardrails import GuardrailConfig from gateway.models.mcp import MAX_MCP_SERVER_IDS, McpServerConfig from gateway.ports.model_provider_port import ModelProviderPort +from gateway.services.file_service import StagedFile from gateway.services.log_writer import LogWriter from gateway.services.mcp_loop import ( MAX_TOOL_ITERATIONS_CAP, @@ -57,6 +64,7 @@ from gateway.services.web_search_budget import WebSearchBudget from gateway.streaming import OPENAI_STREAM_FORMAT, StreamFormat from gateway.types.attempt import Attempt +from gateway.types.code_execution import CodeExecutor from gateway.types.session_principal import SessionPrincipal router = APIRouter(prefix="/chat", tags=["chat"]) @@ -263,11 +271,13 @@ async def run_tool_loop( on_first_response: Callable[[], None] | None = None, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> ChatCompletion: - # ``emit_native_web_search`` is accepted for interface parity and ignored: - # this format has no native vocabulary for a server-side tool call, so a - # gateway-run search stays invisible on the wire (see docs/tools.md). + # The two ``emit_native_*`` flags are accepted for interface parity and + # ignored: this format has no native vocabulary for a server-side tool + # call, so a gateway-run search or execution stays invisible on the wire + # (see docs/tools.md). # ``web_search_budget`` is not: the cap bounds what the caller is billed # for, which every format owes whether or not it can describe the search. # Standalone dispatch has no lock-in callback; only pass the kwarg on @@ -291,6 +301,7 @@ def open_tool_loop_stream( max_iterations: int, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> AsyncIterator[ChatCompletionChunk]: extra: dict[str, Any] = {} @@ -441,12 +452,17 @@ async def run_chat_completion( detail="Invalid request: model is required", ) + # Uploads the normalizer found for the code-execution sandbox, handed to the + # sandbox session once the billed user and workspace are resolved. + sandbox_inputs: list[StagedFile] = [] + async def _normalize( user_id: str, provider: LLMProvider | None, model: str, instance: str | None, workspace_id: uuid.UUID | None, + workspace_executor: CodeExecutor | None, ) -> tuple[int, CompletionUsage | None]: # Resolve uploaded file/image blocks into the wire payload (extract to # text for text-only models, inline for natively-capable ones) before @@ -463,7 +479,16 @@ async def _normalize( user_id=user_id, instance=instance, workspace_id=workspace_id, + sandbox_requested=sandbox_requested( + request.tools, + config=config, + provider=provider, + dialect=adapter.name, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + workspace_executor=workspace_executor, + ), ) + sandbox_inputs.extend(stats.sandbox_inputs) return len(str(request.messages)), stats.vision_usage() output_cap = _effective_output_cap(request.max_tokens, request.max_completion_tokens) @@ -486,6 +511,7 @@ async def _normalize( request.messages, raw_request, has_tools=bool(request.tools) ), normalize_messages=_normalize, + tools=request.tools, ) tool_ctx = await prepare_gateway_tools( @@ -499,6 +525,15 @@ async def _normalize( mcp_server_ids=request.mcp_server_ids, max_tool_iterations=request.max_tool_iterations, tools_header=request.tools_header, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + sandbox_files=build_sandbox_file_bridge( + raw_request=raw_request, + config=config, + db=db, + user_id=ctx.user_id, + workspace_id=ctx.workspace_id, + inputs=sandbox_inputs, + ), ) request_fields = _strip_gateway_fields( diff --git a/src/gateway/api/routes/files.py b/src/gateway/api/routes/files.py index 9f51ebb707..7da3c704dd 100644 --- a/src/gateway/api/routes/files.py +++ b/src/gateway/api/routes/files.py @@ -12,18 +12,24 @@ to its own key's workspace on every verb; a master-key request is the operator acting deployment-wide and sees every workspace, narrowable on the listing with ``workspace_id``, matching ``GET /api/v1/keys``. + +The same five routes serve two SDKs. OpenAI's and Anthropic's Files APIs share +their paths and verbs and differ only in the JSON they return, so the response +shape follows the caller: a request carrying Anthropic's ``anthropic-version`` +header (which its SDK sends on every call) gets ``FileMetadata``, everything +else gets the OpenAI file object. """ -import mimetypes import uuid from collections.abc import AsyncGenerator, AsyncIterator -from datetime import UTC, datetime, timedelta -from typing import Annotated, Any +from datetime import UTC, datetime +from typing import Annotated, Any, Literal from urllib.parse import quote -from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status +import httpx +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile, status from fastapi.responses import Response, StreamingResponse -from sqlalchemy import select +from sqlalchemy import and_, or_, select from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession @@ -33,8 +39,9 @@ from gateway.log_config import logger from gateway.models.api_keys import APIKey from gateway.models.tools import FileObject -from gateway.services.file_service import fetch_file +from gateway.services.file_service import expiry_for, fetch_file, guess_mime_type from gateway.services.file_store import FileStore +from gateway.services.files.provider_files import stream_provider_file from gateway.services.workspace_scope import default_workspace_id router = APIRouter(tags=["files"]) @@ -43,6 +50,22 @@ # enum (forward-compat), but normalise the empty case to "user_data". _DEFAULT_PURPOSE = "user_data" +# Listing page bounds. The default is OpenAI's; the ceiling is well under +# OpenAI's 10000 because a page is one query and one JSON body. +_DEFAULT_LIST_LIMIT = 100 +_MAX_LIST_LIMIT = 1000 + + +def _anthropic_shape(raw_request: Request) -> bool: + """Whether the caller speaks Anthropic's Files API rather than OpenAI's.""" + return "anthropic-version" in raw_request.headers or any( + beta.strip().startswith("files-api") for beta in raw_request.headers.get("anthropic-beta", "").split(",") + ) + + +def _serialize(record: FileObject, raw_request: Request) -> dict[str, Any]: + return record.to_anthropic_dict() if _anthropic_shape(raw_request) else record.to_dict() + def _request_workspace_id(auth_result: tuple[APIKey | None, bool]) -> uuid.UUID | None: """The workspace a keyed request is confined to, or ``None`` for the master key. @@ -158,18 +181,9 @@ def _content_disposition(filename: str) -> str: return f"attachment; filename=\"{ascii_name}\"; filename*=UTF-8''{encoded}" -def _guess_mime(filename: str | None, declared: str | None) -> str: - if declared and declared != "application/octet-stream": - return declared - if filename: - guessed, _ = mimetypes.guess_type(filename) - if guessed: - return guessed - return declared or "application/octet-stream" - - @router.post("/files") async def create_file( + raw_request: Request, auth_result: Annotated[tuple[APIKey | None, bool], Depends(verify_api_key_or_master_key)], db: Annotated[AsyncSession, Depends(get_db)], config: Annotated[GatewayConfig, Depends(get_config)], @@ -178,7 +192,7 @@ async def create_file( purpose: str = Form(_DEFAULT_PURPOSE), user: str | None = Form(None), ) -> dict[str, Any]: - """OpenAI-compatible file upload endpoint.""" + """Upload a file. Answers in the OpenAI or Anthropic file shape, following the caller's headers.""" if not config.files_enabled: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File uploads are disabled") @@ -197,21 +211,18 @@ async def create_file( await file_store.delete(storage_ref) raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Uploaded file is empty") - expires_at: datetime | None = None - if config.files_retention_hours is not None: - expires_at = datetime.now(UTC) + timedelta(hours=config.files_retention_hours) - + now = datetime.now(UTC) record = FileObject( id=file_id, user_id=user_id, workspace_id=workspace_id, filename=file.filename or file_id, - mime_type=_guess_mime(file.filename, file.content_type), + mime_type=guess_mime_type(file.filename, file.content_type), bytes=size, purpose=purpose or _DEFAULT_PURPOSE, storage_ref=storage_ref, - created_at=datetime.now(UTC), - expires_at=expires_at, + created_at=now, + expires_at=expiry_for(config, now), ) db.add(record) try: @@ -230,22 +241,32 @@ async def create_file( logger.info( "Stored file %s (%d bytes) for user %s in workspace %s", file_id, size, user_id, workspace_id ) - return record.to_dict() + return _serialize(record, raw_request) @router.get("/files") async def list_files( + raw_request: Request, auth_result: Annotated[tuple[APIKey | None, bool], Depends(verify_api_key_or_master_key)], db: Annotated[AsyncSession, Depends(get_db)], config: Annotated[GatewayConfig, Depends(get_config)], user: str | None = None, purpose: str | None = None, workspace_id: uuid.UUID | None = None, + limit: Annotated[int, Query(ge=1, le=_MAX_LIST_LIMIT)] = _DEFAULT_LIST_LIMIT, + after: str | None = None, + after_id: str | None = None, + order: Literal["asc", "desc"] = "desc", ) -> dict[str, Any]: """List the authenticated user's uploaded files in the request's workspace. ``workspace_id`` narrows a master-key listing to one workspace; a keyed request is already confined to its key's own and cannot widen or move it. + + Pages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names + the last file of the previous page, and ``has_more`` says whether to ask + again. A cursor that has since been deleted or has expired is still a + position; one the caller never owned is a 404. """ if not config.files_enabled: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File uploads are disabled") @@ -256,24 +277,70 @@ async def list_files( # request is confined either way, so refusing it would only add a way to get # an error instead of the same answer. scope = _request_workspace_id(auth_result) or workspace_id + # Expired rows are excluded for the same reason ``fetch_file`` excludes + # them: the sweep reclaims them on a timer, so between expiry and the next + # tick a listing would otherwise offer files that every other verb 404s. stmt = select(FileObject).where( FileObject.user_id == user_id, FileObject.deleted_at.is_(None), + or_(FileObject.expires_at.is_(None), FileObject.expires_at > datetime.now(UTC)), ) if scope is not None: stmt = stmt.where(FileObject.workspace_id == scope) if purpose is not None: stmt = stmt.where(FileObject.purpose == purpose) - stmt = stmt.order_by(FileObject.created_at.desc()) - result = await db.execute(stmt) - records = result.scalars().all() - return {"object": "list", "data": [r.to_dict() for r in records]} + cursor_id = after or after_id + if cursor_id is not None: + # A position, not a file: the row is read with the tenant predicates + # only, so a cursor that was deleted or expired between two pages (the + # usual "list, delete each, list again" loop) still says where the next + # page starts. Another user's id stays a 404. + cursor_conditions = [FileObject.id == cursor_id, FileObject.user_id == user_id] + if scope is not None: + cursor_conditions.append(FileObject.workspace_id == scope) + cursor = (await db.execute(select(FileObject).where(*cursor_conditions))).scalar_one_or_none() + if cursor is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + # (created_at, id) is the sort key, so the page after the cursor is + # everything strictly past it in that order. Spelled as two clauses + # rather than a row-value comparison, which SQLite only partly supports. + if order == "desc": + past = or_( + FileObject.created_at < cursor.created_at, + and_(FileObject.created_at == cursor.created_at, FileObject.id < cursor.id), + ) + else: + past = or_( + FileObject.created_at > cursor.created_at, + and_(FileObject.created_at == cursor.created_at, FileObject.id > cursor.id), + ) + stmt = stmt.where(past) + + if order == "desc": + stmt = stmt.order_by(FileObject.created_at.desc(), FileObject.id.desc()) + else: + stmt = stmt.order_by(FileObject.created_at.asc(), FileObject.id.asc()) + # One past the page tells us whether there is a next one without a count. + records = list((await db.execute(stmt.limit(limit + 1))).scalars().all()) + has_more = len(records) > limit + records = records[:limit] + + page: dict[str, Any] = { + "data": [_serialize(r, raw_request) for r in records], + "has_more": has_more, + "first_id": records[0].id if records else None, + "last_id": records[-1].id if records else None, + } + if not _anthropic_shape(raw_request): + page = {"object": "list", **page} + return page @router.get("/files/{file_id}") async def get_file( file_id: str, + raw_request: Request, auth_result: Annotated[tuple[APIKey | None, bool], Depends(verify_api_key_or_master_key)], db: Annotated[AsyncSession, Depends(get_db)], config: Annotated[GatewayConfig, Depends(get_config)], @@ -287,7 +354,7 @@ async def get_file( record = await fetch_file(db, file_id, user_id, workspace_id=_request_workspace_id(auth_result)) if record is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") - return record.to_dict() + return _serialize(record, raw_request) @router.get( @@ -328,11 +395,39 @@ async def get_file_content( if record is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + if record.provider is not None: + # The bytes live in the provider's own container; Otari holds the row + # that says whose they are and streams them through. + try: + body = await _prime(stream_provider_file(record, config)) + except LookupError as exc: + logger.error("No credential to read %s file %s: %s", record.provider, file_id, exc) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Failed to read file", + ) from exc + except httpx.HTTPError as exc: + logger.warning("Provider %s refused file %s: %s", record.provider, file_id, exc) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="The provider holding this file could not serve it", + ) from exc + return StreamingResponse( + body, + media_type=record.mime_type, + headers={"Content-Disposition": _content_disposition(record.filename)}, + ) + # No Content-Length: it would come from record.bytes (DB) while the body # comes from the storage backend (disk). If those ever diverge (partial # write, corruption), a length header derived from the DB value would be # wrong, and clients trust that header over what actually arrives. Chunked # transfer encoding doesn't need to declare a length up front. + if record.storage_ref is None: + # Neither a blob of ours nor a provider's: nothing can be served. + logger.error("File %s has no storage ref and no provider", file_id) + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found") + try: body = await _prime(file_store.get_stream(record.storage_ref)) except OSError as exc: @@ -352,6 +447,7 @@ async def get_file_content( @router.delete("/files/{file_id}") async def delete_file( file_id: str, + raw_request: Request, auth_result: Annotated[tuple[APIKey | None, bool], Depends(verify_api_key_or_master_key)], db: Annotated[AsyncSession, Depends(get_db)], config: Annotated[GatewayConfig, Depends(get_config)], @@ -383,8 +479,11 @@ async def delete_file( # view. Removing the blob is best-effort cleanup: a backend failure must not # turn a successful delete into a 500 (it would only leave an orphaned blob). try: - await file_store.delete(storage_ref) + if storage_ref is not None: + await file_store.delete(storage_ref) except OSError as exc: logger.warning("Soft-deleted file %s but failed to remove its blob %s: %s", file_id, storage_ref, exc) + if _anthropic_shape(raw_request): + return {"id": file_id, "type": "file_deleted"} return {"id": file_id, "object": "file", "deleted": True} diff --git a/src/gateway/api/routes/messages.py b/src/gateway/api/routes/messages.py index 299fbb5e5d..8d72a56b17 100644 --- a/src/gateway/api/routes/messages.py +++ b/src/gateway/api/routes/messages.py @@ -3,7 +3,7 @@ from collections.abc import AsyncIterator, Callable from typing import Annotated, Any, Literal -from any_llm import LLMProvider, amessages +from any_llm import AnyLLM, LLMProvider, amessages from any_llm.types.completion import CompletionUsage from any_llm.types.messages import ( MessageDeltaEvent, @@ -19,6 +19,7 @@ from gateway.api.deps import ( ModelProviderPortDep, + build_sandbox_file_bridge, extract_credential_token, get_config, get_db_if_needed, @@ -26,7 +27,7 @@ verify_api_key_or_master_key, ) from gateway.api.routes._helpers import latest_user_text, routing_signal_from_messages -from gateway.api.routes._normalize import normalize_request_messages +from gateway.api.routes._normalize import normalize_request_messages, sandbox_requested from gateway.api.routes._pipeline import ( DB_UNAVAILABLE_DETAIL, NO_RESOLVABLE_PROVIDER_DETAIL, @@ -52,26 +53,30 @@ _resolve_platform_credentials, ) from gateway.api.routes._schema_derive import SESSION_LABEL_DESC, SESSION_LABEL_MAX_LENGTH, derive_request_base -from gateway.api.routes._tools import _strip_gateway_fields +from gateway.api.routes._tools import CODE_EXECUTION_HEADER, _strip_gateway_fields from gateway.core.config import GatewayConfig from gateway.core.usage import GatewayUsage from gateway.log_config import logger from gateway.models.guardrails import GuardrailConfig from gateway.models.mcp import MAX_MCP_SERVER_IDS, McpServerConfig +from gateway.services.file_service import StagedFile from gateway.services.log_writer import LogWriter from gateway.services.mcp_loop import ToolBackend from gateway.services.mcp_loop_messages import ( MAX_TOOL_ITERATIONS_CAP, MCP_ACTIVITY_ID_PREFIX, MCP_CLIENT_BETA, + SERVER_TOOL_USE_ID_PREFIX, WEB_SEARCH_TOOL_USE_ID_PREFIX, anthropic_tool_loop, anthropic_tool_loop_stream, ) +from gateway.services.sandbox_backend import CODE_EXECUTION_TOOL_NAME from gateway.services.tool_format import inject_purpose_hints_anthropic, openai_to_anthropic_tools from gateway.services.web_search_budget import WebSearchBudget from gateway.streaming import ANTHROPIC_STREAM_FORMAT, StreamFormat from gateway.types.attempt import Attempt +from gateway.types.code_execution import CodeExecutor router = APIRouter(tags=["messages"]) @@ -87,19 +92,64 @@ def _merge_anthropic_betas(body_betas: list[str] | None, raw_request: Request) - return list(dict.fromkeys(betas)) or None -def _split_mcp_client_beta(kwargs: dict[str, Any]) -> tuple[dict[str, Any], bool]: - """Consume the client capability without forwarding it to the model provider.""" +def _serves_messages_natively(dispatch_model: Any) -> bool: + """Whether the dispatched provider has an Anthropic Messages API of its own. + + The same question any-llm asks before refusing ``betas``, asked the same + way: a provider that serves Messages natively overrides ``_amessages``, + while a bridged one inherits the base implementation that converts + Messages to Completions (and refuses what cannot survive the conversion). + ``SUPPORTS_MESSAGES`` does not answer it, being true for every provider the + bridge covers. + + An unknown or unparseable selector answers yes, so nothing is stripped on a + guess; any-llm then refuses the beta itself, as it did before. + + This reads a private any-llm attribute, which nothing public answers today + (``SUPPORTS_MESSAGES`` is true for every bridged provider). It is a shim in + the sense of CONTRIBUTING's "when the fix is upstream": the durable answer + is a public capability flag on the any-llm provider class, asked for in + https://github.com/mozilla-ai/any-llm/issues/1418, and this goes when that + lands. ``getattr`` keeps a renamed attribute from breaking a request; it + degrades to forwarding the betas, the pre-shim behavior. + """ + if not isinstance(dispatch_model, str) or not dispatch_model: + return True + try: + provider, _ = AnyLLM.split_model_provider(dispatch_model) + native = getattr(AnyLLM.get_provider_class(provider), "_amessages", None) + return native is not getattr(AnyLLM, "_amessages", None) + except Exception: # noqa: BLE001 - any-llm raises its own types for an unknown provider + return True + + +def _split_client_betas(kwargs: dict[str, Any]) -> tuple[dict[str, Any], bool]: + """Take out the betas no provider should see, and say whether MCP's was among them. + + Two never travel. The MCP client capability is the gateway's own, consumed + here. And every beta is dropped for a provider with no Messages API of its + own, where any-llm refuses the request outright: a beta names an Anthropic + feature that provider was never going to serve, so forwarding it would make + a request stop working purely because its model changed, which is the swap + `auto` exists to keep invisible. + """ betas = kwargs.get("betas") - if not isinstance(betas, list) or MCP_CLIENT_BETA not in betas: + if not isinstance(betas, list) or not betas: + return kwargs, False + + saw_mcp_beta = MCP_CLIENT_BETA in betas + provider_betas = ( + [beta for beta in betas if beta != MCP_CLIENT_BETA] if _serves_messages_natively(kwargs.get("model")) else [] + ) + if provider_betas == betas: return kwargs, False provider_kwargs = {**kwargs} - provider_betas = [beta for beta in betas if beta != MCP_CLIENT_BETA] if provider_betas: provider_kwargs["betas"] = provider_betas else: provider_kwargs.pop("betas") - return provider_kwargs, True + return provider_kwargs, saw_mcp_beta class MessagesRequest(derive_request_base(MessagesParams)): # type: ignore[misc] @@ -187,6 +237,46 @@ def _is_gateway_minted_result(block: Any) -> bool: return all(isinstance(hit, dict) and not hit.get("encrypted_content") for hit in hits) +def _is_gateway_minted_code_execution_result(block: Any) -> bool: + """Whether a ``code_execution_tool_result`` block was minted by this gateway. + + Provenance is the reserved ``server_tool_use`` id prefix, as for web search. + Anthropic's own results, which a caller echoes when the provider ran the + code, carry ``srvtoolu_`` ids and survive untouched. + """ + if not isinstance(block, dict) or block.get("type") != "code_execution_tool_result": + return False + return str(block.get("tool_use_id") or "").startswith(SERVER_TOOL_USE_ID_PREFIX) + + +def _code_execution_pair_as_text(use: dict[str, Any] | None, result: dict[str, Any]) -> dict[str, Any]: + """Fold a gateway-minted code-execution pair into one assistant ``text`` block. + + Unlike a web-search pair, whose hits are already in the transcript, an + execution's output exists nowhere else, so dropping the pair would make the + model forget what its code printed on the previous turn. A ``tool_use`` / + ``tool_result`` rewrite is not available either: ``tool_result`` must open a + user turn, which would mean splitting the echoed assistant message. A text + block keeps the code and its output in the model's view in a shape every + provider accepts. + """ + code = str(((use or {}).get("input") or {}).get("code") or "") + raw_content = result.get("content") + content: dict[str, Any] = raw_content if isinstance(raw_content, dict) else {} + parts = [f"[code executed]\n```\n{code}\n```"] if code else ["[code executed]"] + if content.get("type") == "code_execution_tool_result_error": + parts.append(f"error: {content.get('error_code') or 'unavailable'}") + else: + for label in ("stdout", "stderr"): + value = content.get(label) + if isinstance(value, str) and value: + parts.append(f"{label}:\n{value}") + return_code = content.get("return_code") + if isinstance(return_code, int) and return_code != 0: + parts.append(f"return_code: {return_code}") + return {"type": "text", "text": "\n".join(parts)} + + def _is_gateway_minted_mcp_block(block: Any) -> bool: """Whether ``block`` carries this gateway's reserved MCP activity prefix.""" if not isinstance(block, dict): @@ -215,6 +305,10 @@ def _strip_gateway_minted_blocks(messages: Any) -> Any: only alongside the result that answers it, matched by ``tool_use_id``, so a provider's pair is never split. + A gateway-minted code-execution pair is not dropped but folded into a text + block (:func:`_code_execution_pair_as_text`), because its output lives nowhere + else in the transcript. + A message left with no content is dropped: an empty ``content`` array is rejected by the API, and a turn that held nothing but our pair has nothing left to say. """ @@ -237,8 +331,25 @@ def _strip_gateway_minted_blocks(messages: Any) -> Any: for block in content if _is_gateway_minted_mcp_block(block) } - kept_blocks = [block for block in content if not _is_minted_pair_member(block, minted_web_ids, minted_mcp_ids)] - if len(kept_blocks) == len(content): + minted_code_uses = { + block.get("id"): block + for block in content + if isinstance(block, dict) + and block.get("type") == "server_tool_use" + and block.get("name") == CODE_EXECUTION_TOOL_NAME + and str(block.get("id") or "").startswith(SERVER_TOOL_USE_ID_PREFIX) + } + kept_blocks: list[Any] = [] + for block in content: + if _is_minted_pair_member(block, minted_web_ids, minted_mcp_ids): + continue + if _is_gateway_minted_code_execution_result(block): + kept_blocks.append(_code_execution_pair_as_text(minted_code_uses.get(block.get("tool_use_id")), block)) + continue + if isinstance(block, dict) and block.get("id") in minted_code_uses: + continue + kept_blocks.append(block) + if kept_blocks == content: kept_messages.append(message) continue dropped += len(content) - len(kept_blocks) @@ -481,11 +592,11 @@ def is_stream_cost_carrier(self, chunk: MessageStreamEvent) -> bool: return isinstance(chunk, MessageDeltaEvent) async def call_provider(self, kwargs: dict[str, Any]) -> MessageResponse: - provider_kwargs, _ = _split_mcp_client_beta(kwargs) + provider_kwargs, _ = _split_client_betas(kwargs) return await amessages(**provider_kwargs) # type: ignore[return-value] async def open_provider_stream(self, kwargs: dict[str, Any]) -> AsyncIterator[MessageStreamEvent]: - provider_kwargs, _ = _split_mcp_client_beta(kwargs) + provider_kwargs, _ = _split_client_betas(kwargs) return await amessages(**provider_kwargs) # type: ignore[return-value] def prepare_stream_kwargs( @@ -506,6 +617,7 @@ async def run_tool_loop( on_first_response: Callable[[], None] | None = None, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> MessageResponse: # Standalone dispatch has no lock-in callback; only pass the kwarg on @@ -515,7 +627,9 @@ async def run_tool_loop( extra["on_first_response"] = on_first_response if web_search_budget is not None: extra["web_search_budget"] = web_search_budget - provider_kwargs, _ = _split_mcp_client_beta(kwargs) + if emit_native_code_execution: + extra["emit_native_code_execution"] = True + provider_kwargs, _ = _split_client_betas(kwargs) return await anthropic_tool_loop( completion_kwargs=provider_kwargs, pool=pool, @@ -531,14 +645,17 @@ def open_tool_loop_stream( max_iterations: int, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> AsyncIterator[MessageStreamEvent]: - provider_kwargs, emit_native_mcp = _split_mcp_client_beta(kwargs) + provider_kwargs, emit_native_mcp = _split_client_betas(kwargs) extra: dict[str, Any] = {} if emit_native_mcp: extra["emit_native_mcp"] = True if web_search_budget is not None: extra["web_search_budget"] = web_search_budget + if emit_native_code_execution: + extra["emit_native_code_execution"] = True return anthropic_tool_loop_stream( completion_kwargs=provider_kwargs, pool=pool, @@ -648,12 +765,17 @@ async def create_message( # independent of whether the current request enables the same tool again. request.messages = _strip_gateway_minted_blocks(request.messages) + # Uploads the normalizer found for the code-execution sandbox, handed to the + # sandbox session once the billed user and workspace are resolved. + sandbox_inputs: list[StagedFile] = [] + async def _normalize( user_id: str, provider: LLMProvider | None, model: str, instance: str | None, workspace_id: uuid.UUID | None, + workspace_executor: CodeExecutor | None, ) -> tuple[int, CompletionUsage | None]: # Resolve uploaded file/image blocks into the Anthropic wire payload # before the cost estimate. Standalone only; no-op when the files @@ -669,7 +791,16 @@ async def _normalize( user_id=user_id, instance=instance, workspace_id=workspace_id, + sandbox_requested=sandbox_requested( + request.tools, + config=config, + provider=provider, + dialect=_ADAPTER.name, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + workspace_executor=workspace_executor, + ), ) + sandbox_inputs.extend(stats.sandbox_inputs) return len(str(request.messages)) + len(str(request.system or "")), stats.vision_usage() try: @@ -696,6 +827,7 @@ async def _normalize( request.messages, raw_request, has_tools=bool(request.tools) ), normalize_messages=_normalize, + tools=request.tools, ) except HTTPException as exc: # The hybrid preamble (platform resolve / auth) raises format-agnostic @@ -724,6 +856,15 @@ async def _normalize( mcp_server_ids=request.mcp_server_ids, max_tool_iterations=request.max_tool_iterations, tools_header=request.tools_header, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + sandbox_files=build_sandbox_file_bridge( + raw_request=raw_request, + config=config, + db=db, + user_id=ctx.user_id, + workspace_id=ctx.workspace_id, + inputs=sandbox_inputs, + ), ) # Strip gateway-internal fields, convert any caller-supplied OpenAI-shaped @@ -741,9 +882,10 @@ async def _normalize( # ``container`` addresses Anthropic's own code-execution container, and # the gateway sandbox owns execution for this request, so the provider # would be asked to attach a container no tool call will reach. - # ``prepare_gateway_tools`` has already refused the one shape where a - # provider-native code-execution tool survives alongside the sandbox, so - # dropping it here cannot strand a container the provider would have used. + # ``prepare_gateway_tools`` either claimed the provider-native declaration + # or refused the request, so no provider tool survives alongside the + # sandbox and dropping it here cannot strand a container the provider + # would have used. request_fields.pop("container", None) # ------------------------------------------------------------------ diff --git a/src/gateway/api/routes/responses.py b/src/gateway/api/routes/responses.py index e4d6b92ad8..7caa62a0b5 100644 --- a/src/gateway/api/routes/responses.py +++ b/src/gateway/api/routes/responses.py @@ -14,9 +14,15 @@ from pydantic import ConfigDict, Field from sqlalchemy.ext.asyncio import AsyncSession -from gateway.api.deps import ModelProviderPortDep, get_config, get_db_if_needed, get_log_writer +from gateway.api.deps import ( + ModelProviderPortDep, + build_sandbox_file_bridge, + get_config, + get_db_if_needed, + get_log_writer, +) from gateway.api.routes._helpers import latest_user_text, routing_signal_from_text, text_from_content -from gateway.api.routes._normalize import normalize_request_messages +from gateway.api.routes._normalize import normalize_request_messages, sandbox_requested from gateway.api.routes._pipeline import ( NO_RESOLVABLE_PROVIDER_DETAIL, PROVIDER_ERROR_DETAIL, @@ -38,15 +44,17 @@ ) from gateway.api.routes._platform import ResolvedAttempt, SettledCost, build_attempt_client_args from gateway.api.routes._schema_derive import SESSION_LABEL_DESC, SESSION_LABEL_MAX_LENGTH, derive_request_base -from gateway.api.routes._tools import _strip_gateway_fields +from gateway.api.routes._tools import CODE_EXECUTION_HEADER, _strip_gateway_fields from gateway.core.config import GatewayConfig from gateway.core.usage import GatewayUsage from gateway.log_config import logger from gateway.models.guardrails import GuardrailConfig from gateway.models.mcp import MAX_MCP_SERVER_IDS, McpServerConfig +from gateway.services.file_service import StagedFile from gateway.services.log_writer import LogWriter from gateway.services.mcp_loop import ToolBackend from gateway.services.mcp_loop_responses import ( + CODE_INTERPRETER_CALL_ID_PREFIX, MAX_TOOL_ITERATIONS_CAP, responses_tool_loop, responses_tool_loop_stream, @@ -55,6 +63,7 @@ from gateway.services.web_search_budget import WebSearchBudget from gateway.streaming import RESPONSES_STREAM_FORMAT, StreamFormat from gateway.types.attempt import Attempt +from gateway.types.code_execution import CodeExecutor router = APIRouter(tags=["responses"]) @@ -176,31 +185,70 @@ def _split_codex_input_metadata(value: Any) -> tuple[Any, bool]: # ``response.output`` to the next ``input``, and the gateway has no # ``previous_response_id`` support to do that server-side, so an echoed turn would # otherwise ship a ``web_search_call`` to a provider that never declared a -# web-search tool. Anthropic's equivalent hazard (an ``encrypted_content`` blob the -# gateway cannot sign) is why Messages emits no native server-tool blocks at all. +# web-search tool. A ``code_interpreter_call`` is recognized only when its id +# carries the gateway's own prefix, because OpenAI's own items are legitimately +# echoed to OpenAI and must survive. _GATEWAY_MINTED_ITEM_TYPES = frozenset({"web_search_call"}) +def _is_gateway_minted_item(item: Any) -> bool: + return isinstance(item, dict) and item.get("type") in _GATEWAY_MINTED_ITEM_TYPES + + +def _is_gateway_minted_code_interpreter_call(item: Any) -> bool: + if not isinstance(item, dict) or item.get("type") != "code_interpreter_call": + return False + return str(item.get("id") or "").startswith(CODE_INTERPRETER_CALL_ID_PREFIX) + + +def _code_interpreter_call_as_message(item: dict[str, Any]) -> dict[str, Any]: + """Fold a gateway-minted ``code_interpreter_call`` into an assistant message item. + + Unlike a search, whose results are already in the transcript, an execution's + logs exist nowhere else, so dropping the item would make the model forget + what its code printed on the previous turn. The Messages route folds its pair + the same way (``messages._code_execution_pair_as_text``). + """ + code = str(item.get("code") or "") + parts = [f"[code executed]\n```\n{code}\n```"] if code else ["[code executed]"] + parts.extend( + f"logs:\n{output['logs']}" + for output in item.get("outputs") or [] + if isinstance(output, dict) and output.get("type") == "logs" and output.get("logs") + ) + if item.get("status") == "failed": + parts.append("status: failed") + return {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "\n".join(parts)}]} + + def _strip_gateway_minted_items(input_data: Any) -> Any: - """Drop gateway-minted server-tool items from an inbound ``input``. - - Only touches a list input, and only removes the item types the gateway itself - emits. A caller who genuinely used a provider-native web search still had that - run upstream, so its items arrive on a response the gateway passed through - untouched; those are indistinguishable here and are dropped too. That is the - conservative direction: dropping a descriptive item loses nothing the model - needs (the search results themselves are in the transcript), while forwarding - one risks a 400 from the provider. + """Take gateway-minted server-tool items back off an inbound ``input``. + + Only touches a list input, and only the items the gateway itself emits. A + ``web_search_call`` is dropped: a caller who genuinely used a provider-native + web search still had that run upstream, so its items arrive on a response the + gateway passed through untouched; those are indistinguishable here and are + dropped too. That is the conservative direction: dropping a descriptive item + loses nothing the model needs (the search results themselves are in the + transcript), while forwarding one risks a 400 from the provider. A gateway-run + interpreter call is told apart by its id prefix, so a provider's own survives, + and is folded into a message rather than dropped + (:func:`_code_interpreter_call_as_message`). """ if not isinstance(input_data, list): return input_data - kept = [ - item - for item in input_data - if not (isinstance(item, dict) and item.get("type") in _GATEWAY_MINTED_ITEM_TYPES) - ] - if len(kept) != len(input_data): - logger.debug("Stripped %d gateway-minted output item(s) from the inbound input", len(input_data) - len(kept)) + kept: list[Any] = [] + touched = 0 + for item in input_data: + if _is_gateway_minted_code_interpreter_call(item): + kept.append(_code_interpreter_call_as_message(item)) + touched += 1 + elif _is_gateway_minted_item(item): + touched += 1 + else: + kept.append(item) + if touched: + logger.debug("Rewrote %d gateway-minted output item(s) on the inbound input", touched) return kept @@ -346,11 +394,12 @@ async def run_tool_loop( on_first_response: Callable[[], None] | None = None, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> ResponsesResponse: # ``emit_native_web_search`` is accepted for interface parity and ignored: - # this format has no native vocabulary for a server-side tool call, so a - # gateway-run search stays invisible on the wire (see docs/tools.md). + # this format announces a gateway-run search natively on every request + # (see docs/tools.md), so the Anthropic-shaped opt-in has nothing to add. # ``web_search_budget`` is not: the cap bounds what the caller is billed # for, which every format owes whether or not it can describe the search. # Standalone dispatch has no lock-in callback; only pass the kwarg on @@ -360,6 +409,8 @@ async def run_tool_loop( extra["on_first_response"] = on_first_response if web_search_budget is not None: extra["web_search_budget"] = web_search_budget + if emit_native_code_execution: + extra["emit_native_code_execution"] = True return await responses_tool_loop( completion_kwargs=kwargs, pool=pool, @@ -374,11 +425,14 @@ def open_tool_loop_stream( max_iterations: int, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> AsyncIterator[ResponseStreamEvent]: extra: dict[str, Any] = {} if web_search_budget is not None: extra["web_search_budget"] = web_search_budget + if emit_native_code_execution: + extra["emit_native_code_execution"] = True return responses_tool_loop_stream( completion_kwargs=kwargs, pool=pool, @@ -478,12 +532,17 @@ async def create_response( raw_max_output = getattr(request_body, "max_output_tokens", None) max_output_tokens = raw_max_output if isinstance(raw_max_output, int) and raw_max_output >= 0 else None + # Uploads the normalizer found for the code-execution sandbox, handed to the + # sandbox session once the billed user and workspace are resolved. + sandbox_inputs: list[StagedFile] = [] + async def _normalize( user_id: str, provider: LLMProvider | None, model: str, instance: str | None, workspace_id: uuid.UUID | None, + workspace_executor: CodeExecutor | None, ) -> tuple[int, CompletionUsage | None]: # Resolve uploaded file/image blocks into the Responses input payload # before the cost estimate. Standalone only; no-op when the files @@ -499,7 +558,16 @@ async def _normalize( user_id=user_id, instance=instance, workspace_id=workspace_id, + sandbox_requested=sandbox_requested( + request_body.tools, + config=config, + provider=provider, + dialect=_ADAPTER.name, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + workspace_executor=workspace_executor, + ), ) + sandbox_inputs.extend(stats.sandbox_inputs) chars = len(str(request_body.input)) + len(str(getattr(request_body, "instructions", "") or "")) return chars, stats.vision_usage() @@ -520,6 +588,7 @@ async def _normalize( _routing_text(request_body), raw_request, has_tools=bool(request_body.tools) ), normalize_messages=_normalize, + tools=request_body.tools, ) # Provider-support guard: an unsupported provider would just fail @@ -582,6 +651,15 @@ async def _normalize( mcp_server_ids=request_body.mcp_server_ids, max_tool_iterations=request_body.max_tool_iterations, tools_header=request_body.tools_header, + code_execution_header=raw_request.headers.get(CODE_EXECUTION_HEADER), + sandbox_files=build_sandbox_file_bridge( + raw_request=raw_request, + config=config, + db=db, + user_id=ctx.user_id, + workspace_id=ctx.workspace_id, + inputs=sandbox_inputs, + ), ) # Strip gateway-internal fields, flatten any caller-supplied function tools diff --git a/src/gateway/api/routes/tool_settings.py b/src/gateway/api/routes/tool_settings.py index 9f08f613b9..de5c418bec 100644 --- a/src/gateway/api/routes/tool_settings.py +++ b/src/gateway/api/routes/tool_settings.py @@ -58,6 +58,7 @@ apply_override, effective_value, effective_values, + field_choices, field_service, field_type, stage_override, @@ -102,6 +103,9 @@ class ToolSettingField(BaseModel): # keeping float out of the type narrows the OpenAPI contract accordingly. value: bool | int | str | None description: str | None = None + # The closed vocabulary of a ``str`` field. A dashboard offers exactly these + # rather than a text box, since the write refuses anything else. + choices: list[str] | None = None class ToolSettingsResponse(BaseModel): @@ -129,6 +133,7 @@ class UpdateToolSettingsRequest(BaseModel): sandbox_url: str | None = None sandbox_purpose_hint: str | None = None sandbox_session_image: str | None = None + code_execution_executor: str | None = None guardrails_url: str | None = None @@ -167,6 +172,7 @@ def _current_fields(config: GatewayConfig, *, include_urls: bool = True) -> Tool type=field_type(key), # type: ignore[arg-type] value=_display_value(config, key), description=GatewayConfig.model_fields[key].description, + choices=field_choices(key), ) for key in keys ] diff --git a/src/gateway/api/routes/tools.py b/src/gateway/api/routes/tools.py index b8bfdb7f7b..8f47e0a188 100644 --- a/src/gateway/api/routes/tools.py +++ b/src/gateway/api/routes/tools.py @@ -25,7 +25,7 @@ from pydantic import BaseModel, Field from gateway.api.deps import get_config, verify_catalog_reader -from gateway.api.routes._tools import Tool, web_search_declaration_forms +from gateway.api.routes._tools import Tool, code_execution_declaration_forms, web_search_declaration_forms from gateway.core.config import GatewayConfig from gateway.core.env import otari_env from gateway.core.surface import Surface @@ -56,7 +56,8 @@ class ManagedTool(BaseModel): description=( "Every `tools[].type` this deployment currently routes to the tool. Always " "includes the canonical `otari_*` type; for web search it also includes the " - "provider-named keywords when interception is enabled." + "provider-named keywords when interception is enabled, and for code execution " + "the provider-named keywords unless the deployment's executor is `provider`." ) ) input_schema: dict[str, Any] = Field( @@ -101,7 +102,9 @@ def _managed_tools(config: GatewayConfig) -> list[ManagedTool]: id=Tool.CODE_EXECUTION, description=code_execution["description"], available=sandbox_configured, - accepted_types=[str(Tool.CODE_EXECUTION)], + accepted_types=( + code_execution_declaration_forms(config) if sandbox_configured else [str(Tool.CODE_EXECUTION)] + ), input_schema=code_execution["parameters"], example={"type": Tool.CODE_EXECUTION}, ), diff --git a/src/gateway/core/config.py b/src/gateway/core/config.py index 9134fda1d1..0103c7c950 100644 --- a/src/gateway/core/config.py +++ b/src/gateway/core/config.py @@ -23,6 +23,7 @@ from gateway.core.settings_view import OMITTED, SECRET, SettingsGroup, Shown from gateway.log_config import logger from gateway.models.routing import RoutingConfig +from gateway.types.code_execution import CodeExecutor API_KEY_HEADER = "Otari-Key" # Aliases accepted for a provider instance's ``provider_type`` that map onto a @@ -118,6 +119,7 @@ "sandbox_purpose_hint", "sandbox_session_image", "sandbox_allowed_session_images", + "code_execution_executor", "web_search_url", "web_search_purpose_hint", "web_search_engines", @@ -878,7 +880,10 @@ class GatewayConfig(BudgetSettings, PricingSettings, BaseSettings): ) files_backend: Annotated[str, Shown(SettingsGroup.FILES)] = Field( default="local", - description="Blob backend for uploaded file bytes: 'local' (filesystem) or 's3'. Future: 'gcs'.", + description=( + "Blob backend for uploaded file bytes: 'local' (a directory), 's3' (boto3), or 'fsspec' " + "(any filesystem fsspec has an implementation for, named by files_url)." + ), ) files_local_dir: Annotated[str, Shown(SettingsGroup.FILES)] = Field( default="./otari-files", @@ -903,18 +908,61 @@ class GatewayConfig(BudgetSettings, PricingSettings, BaseSettings): "'us-east-1' when unset." ), ) + files_url: Annotated[str | None, Shown(SettingsGroup.FILES)] = Field( + default=None, + description=( + "Root URL for the 'fsspec' files backend, e.g. 'gcs://bucket/otari-files', " + "'abfs://container/prefix', 's3://bucket/prefix', 'sftp://host/path' or " + "'file:///var/lib/otari/files'. Needs the otari[fsspec] extra and the protocol's own " + "implementation package (gcsfs, adlfs, s3fs, paramiko, ...). Required when files_backend " + "is 'fsspec'." + ), + ) + files_storage_options: Annotated[dict[str, Any], SECRET] = Field( + default_factory=dict, + description=( + "Keyword arguments for the fsspec implementation behind files_url: credentials, " + "endpoint URLs, regions, project ids. Passed through untouched and never logged; " + "most implementations also read their standard environment variables, so this " + "can usually stay empty." + ), + ) files_max_bytes: Annotated[int, Shown(SettingsGroup.FILES)] = Field( default=512 * 1024 * 1024, ge=1, description="Maximum size in bytes for a single uploaded file.", ) + files_output_max_files: Annotated[int, Shown(SettingsGroup.FILES)] = Field( + default=20, + ge=0, + description=( + "Most files one code-execution call may have stored from its sandbox workspace. " + "Files past the count are named in the tool result but not stored." + ), + ) + files_output_max_bytes: Annotated[int, Shown(SettingsGroup.FILES)] = Field( + default=64 * 1024 * 1024, + ge=1, + description=( + "Total bytes one code-execution call may have stored from its sandbox workspace, across " + "all the files it produced. A file that would take the call past it is named but not stored." + ), + ) files_retention_hours: Annotated[int | None, Shown(SettingsGroup.FILES)] = Field( default=None, ge=1, description=( "Stop serving files older than this many hours: expired files become inaccessible " - "(404) and can no longer be referenced. Their stored bytes are not yet reclaimed " - "automatically, so periodic cleanup is an operator task. None keeps files indefinitely." + "(404) and can no longer be referenced, and the file sweep then reclaims their bytes " + "and rows. None keeps files indefinitely." + ), + ) + files_sweep_interval_sec: Annotated[int, Shown(SettingsGroup.FILES)] = Field( + default=3600, + ge=0, + description=( + "How often the background file sweep reclaims the bytes and rows of expired and " + "deleted files. 0 disables the sweep, leaving cleanup to the operator." ), ) file_understanding_enabled: Annotated[bool, Shown(SettingsGroup.VISION)] = Field( @@ -1015,6 +1063,19 @@ class GatewayConfig(BudgetSettings, PricingSettings, BaseSettings): "workspace may not pin an image at all." ), ) + code_execution_executor: Annotated[str | None, Shown(SettingsGroup.TOOLS)] = Field( + default=None, + description=( + "Who runs the code a provider-native code-execution declaration asks for " + "(Anthropic's code_execution_, OpenAI's code_interpreter, the bare code_execution). " + "'auto' (the default when unset) forwards it to the provider when that provider runs the " + "tool natively for the model, and runs it on this gateway's sandbox otherwise, so a request " + "written for a frontier model keeps working when the model is swapped. 'otari' always runs it " + "on the sandbox; 'provider' always forwards it. A workspace policy may pin a value and the " + "X-Otari-Code-Execution header may choose one per request where the workspace has not. " + "The explicit otari_code_execution type is always run by the gateway." + ), + ) web_fetch_enabled: Annotated[bool, Shown(SettingsGroup.TOOLS)] = Field( default=False, description=( @@ -1761,6 +1822,15 @@ def search_tools_without_backend_url(self) -> list[str]: and not entry.get("api_base") ] + def effective_code_executor(self) -> CodeExecutor: + """The deployment's answer to who runs a provider-named code-execution tool. + + ``auto`` when nothing is set, so an upgrade changes nothing for a request the + provider was already serving and only claims the ones it could not. + """ + configured = (self.code_execution_executor or "").strip() or otari_env("CODE_EXECUTION_EXECUTOR") + return CodeExecutor.parse(configured) or CodeExecutor.AUTO + def sandbox_configured(self) -> bool: """Whether this deployment can run ``otari_code_execution`` at all. @@ -2035,6 +2105,17 @@ def _validate_mail_transport(cls, value: str) -> str: raise ValueError(msg) return normalized + @field_validator("code_execution_executor") + @classmethod + def _validate_code_execution_executor(cls, value: str | None) -> str | None: + if value is None or not value.strip(): + return None + executor = CodeExecutor.parse(value) + if executor is None: + msg = f"code_execution_executor must be one of {[e.value for e in CodeExecutor]}, got '{value}'" + raise ValueError(msg) + return executor.value + @field_validator("vision_strategy") @classmethod def _validate_vision_strategy(cls, value: str) -> str: diff --git a/src/gateway/main.py b/src/gateway/main.py index 0534cd503c..f4a8d94084 100644 --- a/src/gateway/main.py +++ b/src/gateway/main.py @@ -33,6 +33,7 @@ from gateway.services.catalog_selectors import reset_selector_index from gateway.services.dashboard_session_service import revoke_sessions_on_master_key_change from gateway.services.file_store import build_file_store +from gateway.services.files import run_file_sweeper from gateway.services.log_writer import LogWriter, NoopLogWriter, create_log_writer from gateway.services.master_key_service import ensure_master_key from gateway.services.model_catalog_service import ( @@ -167,6 +168,13 @@ def _start_reservation_sweeper(config: GatewayConfig) -> Coroutine[Any, Any, Non ) +def _start_file_sweeper(config: GatewayConfig) -> Coroutine[Any, Any, None] | None: + """Return the file retention sweep, or None when files or the interval disable it.""" + if not config.files_enabled or config.files_sweep_interval_sec <= 0: + return None + return run_file_sweeper(config.files_sweep_interval_sec, build_file_store(config)) + + # The periodic background workers a standalone deployment runs. # A new worker is one entry here. # @@ -198,6 +206,9 @@ def _start_reservation_sweeper(config: GatewayConfig) -> Coroutine[Any, Any, Non # Not a cache reload: this returns leaked budget holds. Without it a user # whose single request leaked would hold against their budget forever. _LifespanWorker("budget reservation sweep", _start_reservation_sweeper), + # Same posture for uploaded files: expiry hides a file, this gives its + # bytes back. + _LifespanWorker("file retention sweep", _start_file_sweeper), ) diff --git a/src/gateway/models/tools.py b/src/gateway/models/tools.py index 48bd2390b6..684e0c9e36 100644 --- a/src/gateway/models/tools.py +++ b/src/gateway/models/tools.py @@ -4,7 +4,19 @@ from datetime import UTC, datetime from typing import Any -from sqlalchemy import JSON, CheckConstraint, DateTime, ForeignKey, String, Text, UniqueConstraint, Uuid, func, true +from sqlalchemy import ( + JSON, + CheckConstraint, + DateTime, + ForeignKey, + Index, + String, + Text, + UniqueConstraint, + Uuid, + func, + true, +) from sqlalchemy.orm import Mapped, mapped_column from gateway.models.base import Base, UtcDateTime @@ -86,6 +98,19 @@ class FileObject(Base): """ __tablename__ = "file_objects" + __table_args__ = ( + # The listing's shape: the tenant predicates, then the keyset sort the + # cursor pages on. Without them every page sorts the user's whole set; + # the second serves a master-key listing that names no workspace. + Index( + "ix_file_objects_user_workspace_created", + "user_id", + "workspace_id", + "created_at", + "id", + ), + Index("ix_file_objects_user_created", "user_id", "created_at", "id"), + ) id: Mapped[str] = mapped_column(primary_key=True, default=lambda: f"file-{uuid.uuid4().hex}") # Always set to the authenticated user; non-null enforces the user-scoping @@ -102,11 +127,23 @@ class FileObject(Base): mime_type: Mapped[str] = mapped_column() bytes: Mapped[int] = mapped_column() purpose: Mapped[str] = mapped_column(default="user_data") - storage_ref: Mapped[str] = mapped_column() + # Null for a file whose bytes a provider holds; see ``provider`` below. + storage_ref: Mapped[str | None] = mapped_column(nullable=True) + # Set when a provider's own sandbox produced the file, naming the any-llm + # provider whose files API serves its bytes. The row exists so the + # deployment knows who may read that id: the provider authenticates the + # deployment's credential, which is coarser than a workspace-scoped key. + provider: Mapped[str | None] = mapped_column(nullable=True) + # The configured instance the run dispatched through, whose credential is + # the one that can read the file back; None means the provider's own entry. + provider_instance: Mapped[str | None] = mapped_column(nullable=True) + # The provider's container, for a provider that keys a download on it + # (OpenAI does; Anthropic's files API takes the id alone). + provider_container_id: Mapped[str | None] = mapped_column(nullable=True) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=lambda: datetime.now(UTC), index=True ) - expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) deleted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), default=None, index=True) metadata_: Mapped[dict[str, Any]] = mapped_column("metadata", JSON, default=dict) @@ -123,6 +160,28 @@ def to_dict(self) -> dict[str, Any]: "purpose": self.purpose, } + def to_anthropic_dict(self) -> dict[str, Any]: + """Convert to the Anthropic Files API ``FileMetadata`` shape. + + Anthropic's SDK reads ``size_bytes`` and ``mime_type`` where OpenAI's + reads ``bytes`` and nothing, and takes ``created_at`` as an RFC 3339 + string rather than an epoch. ``downloadable`` is always true here: the + gateway serves every stored file's bytes back, unlike Anthropic, which + withholds user uploads. + """ + created_at = self.created_at + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=UTC) + return { + "id": self.id, + "type": "file", + "filename": self.filename, + "mime_type": self.mime_type, + "size_bytes": self.bytes, + "created_at": created_at.isoformat().replace("+00:00", "Z"), + "downloadable": True, + } + class WorkspaceMcpServer(Base): """One MCP server a workspace has configured, referenced by id from a request. @@ -252,6 +311,12 @@ class WorkspaceCodeExecutionPolicy(Base): # ``WorkspaceWebSearchConfig`` stores its domain lists that way: short, read # whole, and nothing queries into it. tools: Mapped[list[str] | None] = mapped_column(JSON, default=None) + # NULL means "no workspace pin": the deployment's ``code_execution_executor`` + # (and, where it leaves room, the request's header) decides who runs a + # provider-named code-execution declaration. A stored value is a pin the + # request cannot argue with. One of ``CodeExecutor``'s values; the service + # refuses anything else, and the column is sized for that vocabulary. + executor: Mapped[str | None] = mapped_column(String(16), default=None) # ``UtcDateTime`` for the same reason ``WorkspaceBudgetDefault`` uses it: # these are serialized with ``.isoformat()`` for the dashboard, and a plain # ``DateTime(timezone=True)`` round-trips naive on SQLite. diff --git a/src/gateway/repositories/files/__init__.py b/src/gateway/repositories/files/__init__.py new file mode 100644 index 0000000000..b1e18a129c --- /dev/null +++ b/src/gateway/repositories/files/__init__.py @@ -0,0 +1,23 @@ +"""Data access for the file rows the ``/v1/files`` API serves.""" + +from gateway.repositories.files.file_repository import ( + OutputFileRow, + delete_file_rows, + reclaimable_files, + record_output_file, +) +from gateway.repositories.files.provider_file_repository import ( + ProviderFileRow, + existing_file_ids, + record_provider_file_rows, +) + +__all__ = [ + "OutputFileRow", + "ProviderFileRow", + "delete_file_rows", + "existing_file_ids", + "reclaimable_files", + "record_output_file", + "record_provider_file_rows", +] diff --git a/src/gateway/repositories/files/file_repository.py b/src/gateway/repositories/files/file_repository.py new file mode 100644 index 0000000000..b4c9ef6473 --- /dev/null +++ b/src/gateway/repositories/files/file_repository.py @@ -0,0 +1,88 @@ +"""Data access for the file rows the ``/v1/files`` API serves and reclaims. + +The sweep's two statements and the code-execution output insert live here +rather than in the service that drives them, so the service orchestrates and +this module is the only place the queries are spelled. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Collection, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime + +from sqlalchemy import and_, delete, or_, select + +from gateway.core.unit_of_work import UnitOfWork, session_for +from gateway.models.tools import FileObject + + +@dataclass(frozen=True) +class OutputFileRow: + """One file a code-execution run produced, already resolved to its columns.""" + + file_id: str + user_id: str + workspace_id: uuid.UUID + filename: str + mime_type: str + bytes: int + purpose: str + storage_ref: str + expires_at: datetime | None + + +async def record_output_file(uow: UnitOfWork, row: OutputFileRow) -> None: + """Stage the row for a file a run wrote. Flushes; the caller's unit of work commits.""" + db = session_for(uow) + db.add( + FileObject( + id=row.file_id, + user_id=row.user_id, + workspace_id=row.workspace_id, + filename=row.filename, + mime_type=row.mime_type, + bytes=row.bytes, + purpose=row.purpose, + storage_ref=row.storage_ref, + created_at=datetime.now(UTC), + expires_at=row.expires_at, + ) + ) + await db.flush() + + +async def reclaimable_files( + uow: UnitOfWork, + *, + batch_size: int, + after: tuple[datetime, str] | None = None, + now: datetime | None = None, +) -> Sequence[FileObject]: + """One batch of soft-deleted or expired rows, in ``(created_at, id)`` order. + + ``after`` is the previous batch's last key: paging by key rather than from + the top is what keeps a row whose blob keeps failing to delete from parking + at the head and hiding everything behind it. + """ + stmt = select(FileObject).where( + or_(FileObject.deleted_at.is_not(None), FileObject.expires_at < (now or datetime.now(UTC))) + ) + if after is not None: + created_at, file_id = after + stmt = stmt.where( + or_( + FileObject.created_at > created_at, + and_(FileObject.created_at == created_at, FileObject.id > file_id), + ) + ) + stmt = stmt.order_by(FileObject.created_at, FileObject.id).limit(batch_size) + return (await session_for(uow).execute(stmt)).scalars().all() + + +async def delete_file_rows(uow: UnitOfWork, file_ids: Collection[str]) -> None: + """Remove the rows whose blobs are gone. The caller's unit of work commits.""" + if not file_ids: + return + await session_for(uow).execute(delete(FileObject).where(FileObject.id.in_(list(file_ids)))) diff --git a/src/gateway/repositories/files/provider_file_repository.py b/src/gateway/repositories/files/provider_file_repository.py new file mode 100644 index 0000000000..bab974bf88 --- /dev/null +++ b/src/gateway/repositories/files/provider_file_repository.py @@ -0,0 +1,67 @@ +"""Data access for files a provider's own sandbox holds. + +A row here carries no ``storage_ref``: it names the provider that has the +bytes, so ``GET /v1/files/{id}/content`` can stream them on demand. See +``services/provider_files.py`` for why they are recorded at all. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Collection +from dataclasses import dataclass +from datetime import datetime + +from sqlalchemy import select + +from gateway.core.unit_of_work import UnitOfWork, session_for +from gateway.models.tools import FileObject + + +@dataclass(frozen=True) +class ProviderFileRow: + """One file to record, already resolved to the columns it lands in.""" + + file_id: str + user_id: str + workspace_id: uuid.UUID + filename: str + mime_type: str + purpose: str + provider: str + provider_instance: str | None + container_id: str | None + expires_at: datetime | None + + +async def existing_file_ids(uow: UnitOfWork, file_ids: Collection[str]) -> set[str]: + """Which of ``file_ids`` already have a row, recorded or uploaded.""" + if not file_ids: + return set() + result = await session_for(uow).execute(select(FileObject.id).where(FileObject.id.in_(list(file_ids)))) + return set(result.scalars()) + + +async def record_provider_file_rows(uow: UnitOfWork, rows: Collection[ProviderFileRow]) -> None: + """Stage one row per provider-held file. Flushes; the caller's unit of work commits.""" + db = session_for(uow) + for row in rows: + db.add( + FileObject( + id=row.file_id, + user_id=row.user_id, + workspace_id=row.workspace_id, + filename=row.filename, + mime_type=row.mime_type, + # The provider holds the bytes and does not say how many until + # they are read, so a listing shows 0 rather than a guess. + bytes=0, + purpose=row.purpose, + storage_ref=None, + provider=row.provider, + provider_instance=row.provider_instance, + provider_container_id=row.container_id, + expires_at=row.expires_at, + ) + ) + await db.flush() diff --git a/src/gateway/services/content_normalizer.py b/src/gateway/services/content_normalizer.py index e7483cd8a8..d707e157b9 100644 --- a/src/gateway/services/content_normalizer.py +++ b/src/gateway/services/content_normalizer.py @@ -6,7 +6,11 @@ * **pass through** to a natively-capable provider (resolving any ``file_id`` to inline bytes first, since the upstream provider doesn't know our file ids), or * **extract to text** for a text-only model: documents via markitdown, images - via a vision side-call / OCR, scanned PDFs via rasterize-then-describe. + via a vision side-call / OCR, scanned PDFs via rasterize-then-describe, or +* **stage into the code-execution sandbox** when the request runs one: an + Anthropic ``container_upload`` block names a file for the sandbox rather than + the model, so it is recorded on the stats for the sandbox backend to seed and + replaced by a short text marker telling the model the file is there. The normalizer is format-aware (OpenAI chat, Anthropic messages, OpenAI Responses) because each wire shape names its blocks and its text block @@ -20,7 +24,7 @@ import base64 import binascii import uuid -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Literal from any_llm.types.completion import CompletionUsage @@ -29,7 +33,7 @@ from gateway.core.config import GatewayConfig from gateway.log_config import logger from gateway.services.file_extractors import extract_text_from_file, ocr_image, rasterize_pdf -from gateway.services.file_service import fetch_file, read_file_bytes +from gateway.services.file_service import StagedFile, fetch_file, read_file_bytes, sandbox_path_for from gateway.services.file_store import FileStore from gateway.services.model_capabilities import Capabilities from gateway.services.vision import describe_image @@ -39,6 +43,10 @@ # Kinds of content we normalize. _IMAGE = "image" _DOCUMENT = "document" +# Anthropic's block for a file the code-execution container should see. Not a +# kind the model reads: with a sandbox in the request it is staged, without one +# it is treated as a document. +_CONTAINER = "container_upload" @dataclass @@ -55,6 +63,24 @@ class NormalizationStats: vision_prompt_tokens: int = 0 vision_completion_tokens: int = 0 details: list[str] = field(default_factory=list) + # Uploads the request referenced for the code-execution sandbox, in message + # order and without repeats. Only filled when the caller said a sandbox runs. + sandbox_inputs: list[StagedFile] = field(default_factory=list) + + def stage(self, staged: StagedFile) -> StagedFile: + """Record ``staged`` for the sandbox and return it under its session name. + + A file referenced twice is staged once and keeps the name it got first; + the name itself is ``sandbox_path_for``'s, so two uploads named alike do + not overwrite each other in the working directory. + """ + for existing in self.sandbox_inputs: + if existing.file_id == staged.file_id: + return existing + taken = {existing.filename for existing in self.sandbox_inputs} + named = replace(staged, filename=sandbox_path_for(staged.filename, taken)) + self.sandbox_inputs.append(named) + return named @property def touched(self) -> bool: @@ -92,6 +118,9 @@ class _Source: # the block rewritten to inline data. Already-inline / remote blocks pass # through unchanged (no wasteful decode→re-encode round-trip). needs_inline: bool = False + # The stored upload behind a file_id block, so it can be staged into the + # sandbox. None for inline and remote sources, which have no file to stage. + staged: StagedFile | None = None def _text_block(fmt: WireFormat, text: str) -> dict[str, Any]: @@ -118,6 +147,19 @@ def _to_data_url(data: bytes, mime: str) -> str: return f"data:{mime};base64,{base64.b64encode(data).decode('ascii')}" +@dataclass +class _Resolved: + """A descriptor resolved to bytes. ``staged`` is set when they came from a stored upload.""" + + data: bytes | None + mime: str + filename: str | None + staged: StagedFile | None = None + + def source(self, kind: str) -> _Source: + return _Source(kind, self.data, self.mime, self.filename, None, self.staged is not None, self.staged) + + async def _resolve_from_ref( ref: dict[str, Any], *, @@ -125,12 +167,13 @@ async def _resolve_from_ref( file_store: FileStore | None, user_id: str | None, workspace_id: uuid.UUID | None, -) -> tuple[bytes, str, str | None, bool] | None: + read_bytes: bool = True, +) -> _Resolved | None: """Resolve a ``{file_data|url|file_id, filename}`` descriptor to bytes. - Returns ``(data, mime, filename, from_file_id)``; ``from_file_id`` is True - when the bytes came from a stored upload (so a native model needs the block - rewritten to inline data). + ``read_bytes=False`` resolves a ``file_id`` to its record without loading + the blob, for a block that is only staged into the sandbox and never shown + to the model. """ filename = ref.get("filename") file_id = ref.get("file_id") @@ -139,14 +182,21 @@ async def _resolve_from_ref( if record is None: logger.warning("content normalizer: file_id %s not found for user %s", file_id, user_id) return None - data = await read_file_bytes(file_store, record) - return data, record.mime_type, record.filename, True + if record.storage_ref is None: + # A file a provider's own sandbox produced. Otari serves its bytes + # by proxy on download but never holds them, so there is nothing to + # show the model or seed a session with. + logger.warning("content normalizer: file_id %s is held by its provider, not readable here", file_id) + return None + staged = StagedFile(record.id, record.filename, record.mime_type, record.storage_ref) + data = await read_file_bytes(file_store, record) if read_bytes else None + return _Resolved(data, record.mime_type, record.filename, staged) url = ref.get("file_data") or ref.get("url") if isinstance(url, str) and url.startswith("data:"): decoded, mime = _decode_data_url(url) if decoded is not None: - return decoded, mime, filename, False + return _Resolved(decoded, mime, filename) return None @@ -158,10 +208,25 @@ async def _classify( file_store: FileStore | None, user_id: str | None, workspace_id: uuid.UUID | None, + sandbox_requested: bool = False, ) -> _Source | None: - """Identify an image/document block and resolve its bytes, or return None.""" + """Identify an image/document/container block and resolve its bytes, or return None.""" btype = block.get("type") + # --- sandbox input blocks ------------------------------------------ + if fmt == "anthropic" and btype == _CONTAINER: + # Read the bytes only when there is no sandbox to stage into and the + # block falls back to being a document the model reads. + resolved = await _resolve_from_ref( + block, + db=db, + file_store=file_store, + user_id=user_id, + workspace_id=workspace_id, + read_bytes=not sandbox_requested, + ) + return resolved.source(_CONTAINER) if resolved else None + # --- image blocks --------------------------------------------------- if (fmt in ("openai", "responses") and btype in ("image_url", "input_image")) or ( fmt == "anthropic" and btype == "image" @@ -176,7 +241,7 @@ async def _classify( src, db=db, file_store=file_store, user_id=user_id, workspace_id=workspace_id ) if resolved: - return _Source(_IMAGE, resolved[0], resolved[1], resolved[2], None, resolved[3]) + return resolved.source(_IMAGE) return _Source(_IMAGE, None, "image/png", None, src.get("url")) # openai / responses image image_url = block.get("image_url") @@ -186,7 +251,7 @@ async def _classify( block, db=db, file_store=file_store, user_id=user_id, workspace_id=workspace_id ) if resolved: - return _Source(_IMAGE, resolved[0], resolved[1], resolved[2], None, resolved[3]) + return resolved.source(_IMAGE) if isinstance(url, str): data, mime = _decode_data_url(url) return _Source(_IMAGE, data, mime or "image/png", None, None if data else url) @@ -208,14 +273,14 @@ async def _classify( src, db=db, file_store=file_store, user_id=user_id, workspace_id=workspace_id ) if resolved: - return _Source(_DOCUMENT, resolved[0], resolved[1], resolved[2], None, resolved[3]) + return resolved.source(_DOCUMENT) return _Source(_DOCUMENT, None, "application/pdf", None, src.get("url")) ref = block.get("file", block) if btype == "file" else block resolved = await _resolve_from_ref( ref, db=db, file_store=file_store, user_id=user_id, workspace_id=workspace_id ) if resolved: - return _Source(_DOCUMENT, resolved[0], resolved[1], resolved[2], None, resolved[3]) + return resolved.source(_DOCUMENT) return None return None @@ -314,12 +379,19 @@ async def _normalize_block( file_store: FileStore | None, user_id: str | None, workspace_id: uuid.UUID | None, + sandbox_requested: bool = False, ) -> Any: if not isinstance(block, dict): return block try: src = await _classify( - block, fmt, db=db, file_store=file_store, user_id=user_id, workspace_id=workspace_id + block, + fmt, + db=db, + file_store=file_store, + user_id=user_id, + workspace_id=workspace_id, + sandbox_requested=sandbox_requested, ) except Exception as exc: # noqa: BLE001 — never fail the request over a block logger.warning("content normalizer: failed to classify block: %s", exc) @@ -327,6 +399,13 @@ async def _normalize_block( if src is None: return block + staged = stats.stage(src.staged) if sandbox_requested and src.staged is not None else None + if src.kind == _CONTAINER: + if staged is not None: + # The sandbox gets the bytes; the model gets told where they are. + return _text_block(fmt, f"[File available in the code execution sandbox: {staged.filename}]") + src.kind = _DOCUMENT + native = caps.image if src.kind == _IMAGE else caps.pdf if native: # Only rewrite when bytes came from a stored file_id (the provider can't @@ -354,6 +433,21 @@ async def _normalize_block( return _text_block(fmt, text) +# Content parts the Responses API also accepts as bare ``input`` items. +_RESPONSES_ITEM_TYPES = frozenset({"input_file", "input_image"}) + + +def _wrap_bare_responses_item(item: Any) -> Any: + """Put a bare item the normalizer turned into text back into a valid position. + + A file or image item is valid at the top level of a Responses ``input``, but + the text it extracts to is not: ``input_text`` only lives inside a message. + """ + if isinstance(item, dict) and item.get("type") == "input_text": + return {"role": "user", "content": [item]} + return item + + async def normalize_messages( messages: list[dict[str, Any]], *, @@ -364,6 +458,7 @@ async def normalize_messages( file_store: FileStore | None, user_id: str | None, workspace_id: uuid.UUID | None = None, + sandbox_requested: bool = False, ) -> tuple[list[dict[str, Any]], NormalizationStats]: """Return (possibly-rewritten messages, stats). @@ -371,8 +466,15 @@ async def normalize_messages( workspace of the API key that authenticated the request. ``None`` means "any", which is what a master-key request gets: the operator acting deployment-wide. + ``sandbox_requested`` says the request runs the gateway's code-execution + sandbox. Every stored upload the messages reference is then also recorded on + ``stats.sandbox_inputs`` for the sandbox to seed, and ``container_upload`` + blocks are staged instead of read. + Messages whose ``content`` is a plain string are returned untouched (the - common, zero-overhead path). Only list-content messages are walked. + common, zero-overhead path). Only list-content messages are walked, plus, on + the Responses format, a file or image item placed directly in ``input`` + rather than inside a message. The Responses endpoint accepts a bare-string ``input``; iterating that would walk it character-by-character, so non-list ``messages`` are returned as-is. @@ -381,26 +483,30 @@ async def normalize_messages( if not config.file_understanding_enabled or not isinstance(messages, list): return messages, stats + async def _block(block: Any) -> Any: + return await _normalize_block( + block, + fmt, + caps, + config, + stats, + db=db, + file_store=file_store, + user_id=user_id, + workspace_id=workspace_id, + sandbox_requested=sandbox_requested, + ) + out: list[dict[str, Any]] = [] for message in messages: content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): - out.append(message) + if fmt == "responses" and isinstance(message, dict) and message.get("type") in _RESPONSES_ITEM_TYPES: + out.append(_wrap_bare_responses_item(await _block(message))) + else: + out.append(message) continue - new_content = [ - await _normalize_block( - block, - fmt, - caps, - config, - stats, - db=db, - file_store=file_store, - user_id=user_id, - workspace_id=workspace_id, - ) - for block in content - ] + new_content = [await _block(block) for block in content] out.append({**message, "content": new_content}) if stats.touched: diff --git a/src/gateway/services/file_service.py b/src/gateway/services/file_service.py index cbbd44bf7d..69cc812649 100644 --- a/src/gateway/services/file_service.py +++ b/src/gateway/services/file_service.py @@ -1,22 +1,32 @@ """Shared data-access helpers for uploaded files. -Used by both the ``/v1/files`` route and the content normalizer, which resolves -``file_id`` references in chat messages back to bytes. Centralising the -user-scoping, workspace-scoping, soft-delete and expiry rules here keeps the two -call sites consistent. +Used by the ``/v1/files`` route, the content normalizer (which resolves +``file_id`` references in chat messages back to bytes), and the code-execution +sandbox (which seeds uploads into a session and stores what a run produced). +Centralising the user-scoping, workspace-scoping, soft-delete and expiry rules +here keeps every call site consistent. """ from __future__ import annotations +import mimetypes import uuid -from datetime import UTC, datetime +from collections.abc import Collection +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import PurePosixPath from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from gateway.core.config import GatewayConfig from gateway.models.tools import FileObject from gateway.services.file_store import FileStore +# The purpose stamped on a file the code-execution sandbox produced, so a +# listing can tell a run's artifact from a user's upload. +CODE_EXECUTION_OUTPUT_PURPOSE = "code_execution_output" + def _is_expired(record: FileObject) -> bool: if record.expires_at is None: @@ -56,5 +66,72 @@ async def fetch_file( async def read_file_bytes(file_store: FileStore, record: FileObject) -> bytes: - """Load the raw bytes for ``record`` from the blob backend.""" + """Load the raw bytes for ``record`` from the blob backend. + + Raises ``FileNotFoundError`` for a row whose bytes a provider holds; those + are served by proxy on download and never read into a request here. + """ + if record.storage_ref is None: + raise FileNotFoundError(f"{record.id} is held by provider {record.provider!r}, not the blob store") return await file_store.get(record.storage_ref) + + +def guess_mime_type(filename: str | None, declared: str | None = None) -> str: + """The media type for ``filename``: the declared one when it says something, else by extension.""" + if declared and declared != "application/octet-stream": + return declared + if filename: + guessed, _ = mimetypes.guess_type(filename) + if guessed: + return guessed + return declared or "application/octet-stream" + + +def expiry_for(config: GatewayConfig, now: datetime | None = None) -> datetime | None: + """When a file stored now stops being served, or ``None`` when files are kept indefinitely.""" + if config.files_retention_hours is None: + return None + return (now or datetime.now(UTC)) + timedelta(hours=config.files_retention_hours) + + +@dataclass(frozen=True) +class StagedFile: + """An uploaded file a request asked the code-execution sandbox to see. + + Recorded by the content normalizer while it walks the messages, and consumed + by the sandbox backend when it opens the session. Carries the storage ref + rather than the bytes so a large upload is read once, at staging time, and + never held across the normalizer's whole pass. + """ + + file_id: str + # The name the file has inside the session's working directory, which is + # also what the model is told; see ``sandbox_path_for``. + filename: str + mime_type: str + storage_ref: str + + +def sandbox_path_for(filename: str, taken: Collection[str]) -> str: + """The name a staged upload gets inside the session's working directory. + + The upload's own name reduced to its last path segment, so a name carrying + separators neither nests nor escapes, and suffixed ``-2``, ``-3``, ... when + an earlier attachment already took it, so two uploads named alike are both + there rather than one overwriting the other. A name with no usable segment + becomes ``file``. + """ + base = PurePosixPath(filename.replace("\\", "/")).name + if base in ("", ".", ".."): + base = "file" + if base not in taken: + return base + stem, dot, ext = base.rpartition(".") + if not dot or not stem: + stem, ext = base, "" + else: + ext = f".{ext}" + n = 2 + while f"{stem}-{n}{ext}" in taken: + n += 1 + return f"{stem}-{n}{ext}" diff --git a/src/gateway/services/file_store.py b/src/gateway/services/file_store.py index 765fe4b6ea..7f38881de7 100644 --- a/src/gateway/services/file_store.py +++ b/src/gateway/services/file_store.py @@ -11,18 +11,20 @@ instead of buffering an entire file, which is what actually bounds memory use for concurrent large uploads (see issue #156). -Only a local-filesystem backend ships today; ``S3FileStore`` / ``GCSFileStore`` -can implement the same :class:`FileStore` protocol without touching callers. +Three backends implement the :class:`FileStore` protocol: a local directory, +S3 through boto3, and :class:`FsspecFileStore`, which reaches any filesystem +`fsspec `_ has an implementation for +(GCS, Azure, SFTP, HDFS, WebDAV, and S3 again) from one ``files_url``. """ from __future__ import annotations import asyncio import tempfile -from collections.abc import AsyncGenerator, AsyncIterator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Iterator, Mapping from contextlib import asynccontextmanager, contextmanager from pathlib import Path -from typing import IO, TYPE_CHECKING, Protocol, runtime_checkable +from typing import IO, TYPE_CHECKING, Any, Protocol, runtime_checkable from gateway.core.config import GatewayConfig from gateway.log_config import logger @@ -367,6 +369,159 @@ async def delete(self, storage_ref: str) -> None: await asyncio.to_thread(self._client.delete_object, Bucket=self._bucket, Key=storage_ref) +@contextmanager +def _translate_fsspec_errors(storage_ref: str) -> Iterator[None]: + """Re-raise whatever an fsspec implementation threw as the ``OSError`` family. + + fsspec's own filesystems raise ``FileNotFoundError`` and ``PermissionError`` + for the common cases, but a third-party implementation may surface its + client's exception class instead (a botocore or google-api error), and the + route and sweep callers only know ``OSError``, exactly as they do for the S3 + backend. A missing object stays ``FileNotFoundError`` so callers can tell + "already gone" from "broken". + """ + try: + yield + except FileNotFoundError: + raise + except OSError as exc: + msg = f"fsspec operation failed for {storage_ref!r}: {exc}" + raise OSError(msg) from exc + except Exception as exc: # noqa: BLE001 — a backend's own client error + msg = f"fsspec operation failed for {storage_ref!r}: {exc}" + raise OSError(msg) from exc + + +class FsspecFileStore: + """A :class:`FileStore` over any `fsspec `_ filesystem. + + ``url`` names the root the store writes under, ``s3://bucket/otari-files``, + ``gcs://bucket/prefix``, ``abfs://container/prefix``, ``file:///var/otari``, + ``memory://`` and so on; whatever protocol fsspec can resolve with the + implementation packages installed (``s3fs``, ``gcsfs``, ``adlfs``, ...). + ``storage_options`` go to that implementation as its constructor keyword + arguments, which is where credentials, endpoints and regions live, so they + are never logged here. + + Every call goes through fsspec's synchronous API on a worker thread, the way + the S3 backend drives boto3: the async implementations exist only for a few + protocols, and the sync API is the one every implementation has. + """ + + def __init__(self, url: str, storage_options: Mapping[str, Any] | None = None) -> None: + try: + from fsspec.core import url_to_fs + except ImportError as exc: + msg = "FsspecFileStore requires fsspec. Install it with: pip install otari[fsspec]" + raise ImportError(msg) from exc + + fs, root = url_to_fs(url, **dict(storage_options or {})) + self._fs = fs + self._root = root.rstrip("/") + + def _resolve(self, storage_ref: str) -> str: + """Join ``storage_ref`` under the root, rejecting anything that could leave it. + + A server-generated ref has no ``..`` in it; this is defense-in-depth for + the day one comes from elsewhere, matching the local backend. + """ + parts = storage_ref.split("/") + if not storage_ref or storage_ref.startswith("/") or any(part in ("", ".", "..") for part in parts): + msg = f"Invalid storage_ref escapes the file store root: {storage_ref!r}" + raise ValueError(msg) + return f"{self._root}/{storage_ref}" if self._root else storage_ref + + def _mkparent(self, path: str) -> None: + # Object stores have no directories and treat this as a no-op; a + # filesystem-like backend needs it before the first write into a shard. + self._fs.makedirs(path.rsplit("/", 1)[0], exist_ok=True) + + async def put(self, file_id: str, data: bytes) -> str: + ref = _shard_key(file_id) + path = self._resolve(ref) + + def _write() -> None: + self._mkparent(path) + self._fs.pipe_file(path, data) + + with _translate_fsspec_errors(ref): + await asyncio.to_thread(_write) + return ref + + async def get(self, storage_ref: str) -> bytes: + path = self._resolve(storage_ref) + with _translate_fsspec_errors(storage_ref): + data: bytes = await asyncio.to_thread(self._fs.cat_file, path) + return data + + async def put_stream(self, file_id: str, chunks: AsyncIterator[bytes]) -> tuple[str, int]: + ref = _shard_key(file_id) + path = self._resolve(ref) + total = 0 + + def _open() -> IO[bytes]: + self._mkparent(path) + handle: IO[bytes] = self._fs.open(path, "wb") + return handle + + def _discard_partial() -> None: + try: + self._fs.rm(path) + except FileNotFoundError: + pass + + with _translate_fsspec_errors(ref): + handle = await asyncio.to_thread(_open) + try: + try: + async for chunk in chunks: + total += len(chunk) + with _translate_fsspec_errors(ref): + await asyncio.to_thread(handle.write, chunk) + finally: + # Object-store handles upload on close, so the close is part of + # the write and its failure is a write failure. Shielded like the + # local backend's: this also runs while a cancellation unwinds. + with _translate_fsspec_errors(ref): + await asyncio.shield(asyncio.to_thread(handle.close)) + except BaseException: + try: + await asyncio.shield(asyncio.to_thread(_discard_partial)) + except Exception as cleanup_exc: # noqa: BLE001 + logger.warning("put_stream: failed to remove partial blob %s: %s", ref, cleanup_exc) + raise + return ref, total + + async def get_stream(self, storage_ref: str) -> AsyncGenerator[bytes, None]: + path = self._resolve(storage_ref) + with _translate_fsspec_errors(storage_ref): + handle: IO[bytes] = await asyncio.to_thread(self._fs.open, path, "rb") + try: + while True: + with _translate_fsspec_errors(storage_ref): + chunk = await asyncio.to_thread(handle.read, _STREAM_CHUNK_BYTES) + if not chunk: + break + yield chunk + finally: + try: + await asyncio.shield(asyncio.to_thread(handle.close)) + except Exception as close_exc: # noqa: BLE001 + logger.warning("get_stream: failed to close handle for %s: %s", storage_ref, close_exc) + + async def delete(self, storage_ref: str) -> None: + path = self._resolve(storage_ref) + + def _rm() -> None: + try: + self._fs.rm(path) + except FileNotFoundError: + logger.debug("file_store delete: %s already absent", storage_ref) + + with _translate_fsspec_errors(storage_ref): + await asyncio.to_thread(_rm) + + def build_file_store(config: GatewayConfig) -> FileStore: """Construct the configured :class:`FileStore` backend.""" backend = config.files_backend.strip().lower() @@ -377,5 +532,10 @@ def build_file_store(config: GatewayConfig) -> FileStore: msg = "files_s3_bucket is required when files_backend is 's3'" raise ValueError(msg) return S3FileStore(config.files_s3_bucket, config.files_s3_endpoint_url, config.files_s3_region) - msg = f"Unsupported files_backend: {config.files_backend!r} (supported: 'local', 's3')" + if backend == "fsspec": + if not config.files_url: + msg = "files_url is required when files_backend is 'fsspec'" + raise ValueError(msg) + return FsspecFileStore(config.files_url, config.files_storage_options) + msg = f"Unsupported files_backend: {config.files_backend!r} (supported: 'local', 's3', 'fsspec')" raise ValueError(msg) diff --git a/src/gateway/services/files/__init__.py b/src/gateway/services/files/__init__.py new file mode 100644 index 0000000000..bfd5bd5e64 --- /dev/null +++ b/src/gateway/services/files/__init__.py @@ -0,0 +1,26 @@ +"""File-domain services that are not part of the original flat modules. + +``file_service.py``, ``file_store.py`` and ``file_extractors.py`` next door +still serve uploads and document understanding; this package holds what has +been written since the layout rule took effect. +""" + +from gateway.services.files.file_sweeper import SweepBatch, run_file_sweeper, sweep_files +from gateway.services.files.provider_files import ( + ProviderFile, + produced_files_for, + record_provider_files, + stream_provider_file, +) +from gateway.services.files.sandbox_bridge import SandboxFileBridge + +__all__ = [ + "ProviderFile", + "SandboxFileBridge", + "SweepBatch", + "produced_files_for", + "record_provider_files", + "run_file_sweeper", + "stream_provider_file", + "sweep_files", +] diff --git a/src/gateway/services/files/file_sweeper.py b/src/gateway/services/files/file_sweeper.py new file mode 100644 index 0000000000..fae66b8e32 --- /dev/null +++ b/src/gateway/services/files/file_sweeper.py @@ -0,0 +1,95 @@ +"""Reclaiming the bytes behind expired and soft-deleted files. + +Expiry alone only hides a file (``fetch_file`` answers 404 for it, and so does +a listing); this is what gives its storage back. Runs as one of the lifespan +workers ``main.py`` starts, standalone only. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime + +from gateway.core.database import create_session +from gateway.core.unit_of_work import UnitOfWork +from gateway.log_config import logger +from gateway.repositories.files import delete_file_rows, reclaimable_files +from gateway.services.file_store import FileStore + +# Passes one tick may make before waiting again, so a large backlog drains over +# several ticks instead of holding one session open until it is done. +_MAX_SWEEP_PASSES = 10 + + +@dataclass(frozen=True) +class SweepBatch: + """What one pass of :func:`sweep_files` did, and where the next one starts.""" + + reclaimed: int + # Rows the pass looked at, reclaimed or not. A short batch means the + # backlog is drained. + seen: int + # The last row's ``(created_at, id)``, so a following pass in the same tick + # starts past it rather than re-reading rows whose blob would not delete. + cursor: tuple[datetime, str] | None + + +async def sweep_files( + uow: UnitOfWork, + file_store: FileStore, + *, + batch_size: int, + after: tuple[datetime, str] | None = None, +) -> SweepBatch: + """Reclaim one batch of expired or soft-deleted files: their bytes, then their rows. + + Does not commit: the caller's Unit of Work block does. A blob that is + already gone is not an error (the delete route removes the bytes + best-effort before this ever sees the row); any other storage failure + leaves the row in place for a later tick rather than orphaning bytes + nothing references. + """ + records = await reclaimable_files(uow, batch_size=batch_size, after=after) + reclaimed: list[str] = [] + for record in records: + try: + # A provider-held row has no blob of ours; only the row is reclaimed. + if record.storage_ref is not None: + await file_store.delete(record.storage_ref) + except FileNotFoundError: + pass + except OSError as exc: + logger.warning("file sweep: could not remove blob %s for %s: %s", record.storage_ref, record.id, exc) + continue + reclaimed.append(record.id) + if reclaimed: + await delete_file_rows(uow, reclaimed) + logger.info("file sweep: reclaimed %d file(s)", len(reclaimed)) + cursor = (records[-1].created_at, records[-1].id) if records else None + return SweepBatch(reclaimed=len(reclaimed), seen=len(records), cursor=cursor) + + +async def run_file_sweeper(interval: float, file_store: FileStore, *, batch_size: int = 200) -> None: + """Reclaim expired and deleted files on a timer, forever. Cancelled at shutdown. + + Every error is swallowed and retried on the next tick, matching the other + lifespan tasks: a storage or database blip must not kill the sweeper, + because nothing would restart it. + """ + while True: + await asyncio.sleep(interval) + try: + async with create_session() as db: + uow = UnitOfWork(db) + cursor: tuple[datetime, str] | None = None + for _ in range(_MAX_SWEEP_PASSES): + async with uow: + batch = await sweep_files(uow, file_store, batch_size=batch_size, after=cursor) + if batch.seen < batch_size: + break + cursor = batch.cursor + except asyncio.CancelledError: + raise + except Exception: + logger.warning("File sweep failed; retrying in %ss", interval, exc_info=True) diff --git a/src/gateway/services/files/provider_files.py b/src/gateway/services/files/provider_files.py new file mode 100644 index 0000000000..2564e16461 --- /dev/null +++ b/src/gateway/services/files/provider_files.py @@ -0,0 +1,282 @@ +"""Files a provider's own sandbox produced, which Otari serves by proxy. + +A provider-native code execution keeps what it wrote in the provider's +container, and answers with the provider's file id. Nothing is copied here: +the run is recorded as a ``file_objects`` row with no ``storage_ref``, naming +the provider that holds the bytes, and ``GET /v1/files/{id}/content`` streams +them from that provider on demand. So the same call serves a chart whichever +sandbox drew it, and a caller swapping one model for another changes nothing +but the model. + +The row is what makes that safe. A provider authenticates the deployment's own +credential, which is coarser than a workspace-scoped API key, so without a +record of who the run belonged to, any tenant knowing an id could read another +tenant's output. Reads go through :func:`fetch_file`, which applies the same +user and workspace predicate every other file gets. + +Ids stay the provider's throughout. Rewriting them into Otari's own would break +a client that echoes the turn back, since the container reference it carries +would name a file the provider never issued. + +The HTTP calls below are hand-rolled because any-llm has no files API; the +request for one is https://github.com/mozilla-ai/any-llm/issues/1419, and the +URLs and headers here move there when it lands. +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import httpx +from any_llm import LLMProvider + +from gateway.core.config import GatewayConfig, provider_credential_env_names +from gateway.core.unit_of_work import UnitOfWork +from gateway.log_config import logger +from gateway.repositories.files import ProviderFileRow, existing_file_ids, record_provider_file_rows +from gateway.services.file_service import CODE_EXECUTION_OUTPUT_PURPOSE, expiry_for, guess_mime_type +from gateway.services.provider_kwargs import get_provider_kwargs + +if TYPE_CHECKING: + from gateway.models.tools import FileObject + +ANTHROPIC_FILES_BASE = "https://api.anthropic.com/v1" +OPENAI_BASE = "https://api.openai.com/v1" +ANTHROPIC_FILES_BETA = "files-api-2025-04-14" +ANTHROPIC_VERSION = "2023-06-01" + +# How long to wait on the provider for one file's metadata or bytes. +_TIMEOUT = httpx.Timeout(30.0, read=300.0) + + +@dataclass(frozen=True) +class ProviderFile: + """One file a provider's sandbox produced, as its response announced it.""" + + file_id: str + filename: str | None = None + container_id: str | None = None + + +def _anthropic_files_in(blocks: list[Any]) -> list[ProviderFile]: + files: dict[str, ProviderFile] = {} + for block in blocks: + content = getattr(block, "content", None) + for output in getattr(content, "content", None) or []: + file_id = getattr(output, "file_id", None) + if isinstance(file_id, str) and file_id and file_id not in files: + files[file_id] = ProviderFile(file_id=file_id) + return list(files.values()) + + +def anthropic_produced_files(result: Any) -> list[ProviderFile]: + """Provider file ids in an Anthropic Messages response's tool results. + + Both the python and the bash variant of the tool name their outputs in a + block of their own, and neither gives a filename, so the shape is what this + looks for rather than a block name. + """ + return _anthropic_files_in(list(getattr(result, "content", None) or [])) + + +def responses_produced_files(result: Any) -> list[ProviderFile]: + """Provider file ids an OpenAI Responses reply cites from its container. + + OpenAI announces a produced file as a ``container_file_citation`` + annotation on the message it wrote, which carries the container and the + name as well as the id. + """ + files: dict[str, ProviderFile] = {} + for item in getattr(result, "output", None) or []: + for part in getattr(item, "content", None) or []: + for note in getattr(part, "annotations", None) or []: + if getattr(note, "type", None) != "container_file_citation": + continue + file_id = getattr(note, "file_id", None) + if not isinstance(file_id, str) or not file_id or file_id in files: + continue + files[file_id] = ProviderFile( + file_id=file_id, + filename=getattr(note, "filename", None), + container_id=getattr(note, "container_id", None), + ) + return list(files.values()) + + +def produced_files_for(dialect: str, obj: Any) -> list[ProviderFile]: + """Provider file ids in a completed reply, or in one streamed event, of ``dialect``. + + A Messages stream delivers a server tool result whole in its + ``content_block_start`` event; a Responses stream repeats the entire + response on ``response.completed``. Anything else (a delta, a chat + completion, which has no native code tool) names no file. + """ + kind = getattr(obj, "type", None) + if dialect == "messages": + if kind == "content_block_start": + return _anthropic_files_in([getattr(obj, "content_block", None)]) + return anthropic_produced_files(obj) + if dialect == "responses": + if kind == "response.completed": + return responses_produced_files(getattr(obj, "response", None)) + return responses_produced_files(obj) + return [] + + +def serves_files(provider: str) -> bool: + """Whether Otari knows how to fetch a produced file back from ``provider``.""" + return provider in (LLMProvider.ANTHROPIC.value, LLMProvider.OPENAI.value) + + +def _credentials( + config: GatewayConfig, provider: str, instance: str | None, workspace_id: uuid.UUID | None +) -> tuple[str, str | None]: + """The API key and base URL to read ``provider``'s files with. + + ``instance`` is the configured entry the run dispatched through, so a named + instance's own key and base URL are the ones used to read back what it + produced. Falls back to the provider SDK's own environment variable, which + is how a config with an empty provider stanza is credentialed for dispatch + too. + """ + member = LLMProvider(provider) + kwargs = get_provider_kwargs(config, member, instance, workspace_id=workspace_id) + api_key = kwargs.get("api_key") + if not api_key: + # An empty provider stanza is credentialed by the SDK's own variable, + # which is how the dispatch that produced the file was credentialed too. + for name in provider_credential_env_names(provider) or (): + if value := os.environ.get(name): + api_key = value + break + if not api_key: + raise LookupError(f"no credential configured for provider '{provider}'") + return str(api_key), kwargs.get("api_base") + + +def _request_for(record: FileObject, api_key: str, api_base: str | None) -> tuple[str, dict[str, str]]: + """The URL and headers that read ``record``'s bytes from its provider.""" + if record.provider == LLMProvider.ANTHROPIC.value: + base = (api_base or ANTHROPIC_FILES_BASE).rstrip("/") + headers = { + "x-api-key": api_key, + "anthropic-version": ANTHROPIC_VERSION, + "anthropic-beta": ANTHROPIC_FILES_BETA, + } + return f"{base}/files/{record.id}/content", headers + base = (api_base or OPENAI_BASE).rstrip("/") + # OpenAI keys a container file on its container as well as its id. + if not record.provider_container_id: + raise LookupError(f"{record.provider} file {record.id} names no container to read it from") + return ( + f"{base}/containers/{record.provider_container_id}/files/{record.id}/content", + {"Authorization": f"Bearer {api_key}"}, + ) + + +async def _fetch_filename(provider: str, file_id: str, api_key: str, api_base: str | None) -> str | None: + """Anthropic's metadata call, for the name its result block leaves out. + + One small JSON read per produced file, so a listing and a download both + name the file the way the run did. A failure is not fatal: the id still + downloads, it is just announced under its own id. + """ + if provider != LLMProvider.ANTHROPIC.value: + return None + base = (api_base or ANTHROPIC_FILES_BASE).rstrip("/") + headers = { + "x-api-key": api_key, + "anthropic-version": ANTHROPIC_VERSION, + "anthropic-beta": ANTHROPIC_FILES_BETA, + } + try: + async with httpx.AsyncClient(timeout=_TIMEOUT) as client: + response = await client.get(f"{base}/files/{file_id}", headers=headers) + response.raise_for_status() + name = response.json().get("filename") + except (httpx.HTTPError, ValueError) as exc: + logger.warning("Could not read %s metadata for %s: %s", provider, file_id, exc) + return None + return name if isinstance(name, str) and name else None + + +async def record_provider_files( + uow: UnitOfWork, + files: list[ProviderFile], + *, + provider: str, + user_id: str, + workspace_id: uuid.UUID, + config: GatewayConfig, + provider_instance: str | None = None, +) -> None: + """Record what a provider's sandbox produced, so its ids serve from Otari's files API. + + Never fatal: the caller has a completed response in hand, and a file it + cannot be handed later is a smaller failure than losing the answer. A file + id already recorded is left alone, so a conversation citing the same chart + twice keeps one row. An OpenAI file cited without its container is skipped, + since the download is keyed on the container and the row could never serve. + """ + files = list({file.file_id: file for file in files}.values()) + if provider == LLMProvider.OPENAI.value: + files = [file for file in files if file.container_id] + if not files or not serves_files(provider): + return + try: + api_key, api_base = _credentials(config, provider, provider_instance, workspace_id) + except (LookupError, ValueError) as exc: + logger.warning("Not recording %d %s file(s): %s", len(files), provider, exc) + return + + try: + async with uow: + known = await existing_file_ids(uow, [file.file_id for file in files]) + expires_at = expiry_for(config) + rows = [] + for file in files: + if file.file_id in known: + continue + filename = file.filename or await _fetch_filename(provider, file.file_id, api_key, api_base) + rows.append( + ProviderFileRow( + file_id=file.file_id, + user_id=user_id, + workspace_id=workspace_id, + filename=filename or file.file_id, + mime_type=guess_mime_type(filename), + purpose=CODE_EXECUTION_OUTPUT_PURPOSE, + provider=provider, + provider_instance=provider_instance, + container_id=file.container_id, + expires_at=expires_at, + ) + ) + if not rows: + return + async with uow: + await record_provider_file_rows(uow, rows) + except Exception as exc: # noqa: BLE001 - a recording failure must not fail the response + logger.warning("Could not record %d %s file(s): %s", len(files), provider, exc) + + +async def stream_provider_file(record: FileObject, config: GatewayConfig) -> AsyncGenerator[bytes, None]: + """Stream a provider-held file's bytes, never holding the whole body. + + Raises ``LookupError`` when the provider cannot be credentialed and + ``httpx.HTTPError`` when it refuses or fails, which the route maps to its + own status; the provider's own message never reaches the caller. + """ + api_key, api_base = _credentials(config, str(record.provider), record.provider_instance, record.workspace_id) + url, headers = _request_for(record, api_key, api_base) + async with ( + httpx.AsyncClient(timeout=_TIMEOUT) as client, + client.stream("GET", url, headers=headers) as response, + ): + response.raise_for_status() + async for chunk in response.aiter_bytes(): + yield chunk diff --git a/src/gateway/services/files/sandbox_bridge.py b/src/gateway/services/files/sandbox_bridge.py new file mode 100644 index 0000000000..348c49122b --- /dev/null +++ b/src/gateway/services/files/sandbox_bridge.py @@ -0,0 +1,101 @@ +"""Moving files between the ``/v1/files`` store and one code-execution session.""" + +from __future__ import annotations + +import asyncio +import contextlib +import uuid +from collections.abc import AsyncIterator + +from gateway.core.config import GatewayConfig +from gateway.core.unit_of_work import UnitOfWork +from gateway.repositories.files import OutputFileRow, record_output_file +from gateway.services.file_service import ( + CODE_EXECUTION_OUTPUT_PURPOSE, + StagedFile, + expiry_for, + guess_mime_type, +) +from gateway.services.file_store import FileStore + + +class SandboxFileBridge: + """Moves files between the ``/v1/files`` store and one sandbox session. + + Built per request by the route, once the billed user and workspace are + known, and handed to the sandbox backend. ``inputs`` are the uploads the + request referenced for the sandbox; :meth:`store_output` persists a file a + run produced as a new file row owned by the same user and workspace, so the + caller can download it through ``GET /v1/files/{id}/content``. ``base_url`` + is where those downloads are served from, for a loop that announces a + produced file to the caller as a URL. + + Standalone only: it needs the local database that hybrid mode does not have. + Writes go through the request's Unit of Work, ``uow``: the request session + is released before the provider is dispatched, so it holds no transaction + while the tool loop runs, and each stored file is one block of its own. + """ + + def __init__( + self, + *, + file_store: FileStore, + config: GatewayConfig, + uow: UnitOfWork, + user_id: str, + workspace_id: uuid.UUID, + inputs: list[StagedFile], + base_url: str | None = None, + ) -> None: + self._file_store = file_store + self._config = config + self._uow = uow + self._user_id = user_id + self._workspace_id = workspace_id + self.inputs = inputs + self.base_url = base_url + + @property + def max_output_files(self) -> int: + return self._config.files_output_max_files + + @property + def max_output_bytes(self) -> int: + return min(self._config.files_output_max_bytes, self._config.files_max_bytes) + + async def read_input(self, staged: StagedFile) -> bytes: + return await self._file_store.get(staged.storage_ref) + + async def store_output(self, filename: str, chunks: AsyncIterator[bytes]) -> str | None: + """Persist ``chunks`` as a new file and return its ``file_id``, or ``None`` when empty. + + Streams into the store, so a produced file is never held whole. Whatever + stops the row from landing, the blob goes with it, so nothing sits in the + store that no row and no sweep can reach. + """ + file_id = f"file-{uuid.uuid4().hex}" + storage_ref, size = await self._file_store.put_stream(file_id, chunks) + if size == 0: + await self._file_store.delete(storage_ref) + return None + row = OutputFileRow( + file_id=file_id, + user_id=self._user_id, + workspace_id=self._workspace_id, + filename=filename, + mime_type=guess_mime_type(filename), + bytes=size, + purpose=CODE_EXECUTION_OUTPUT_PURPOSE, + storage_ref=storage_ref, + expires_at=expiry_for(self._config), + ) + try: + async with self._uow: + await record_output_file(self._uow, row) + except BaseException: + # Shielded so a cancellation already in flight cannot cut the + # cleanup short and leave the orphan it exists to prevent. + with contextlib.suppress(Exception): + await asyncio.shield(self._file_store.delete(storage_ref)) + raise + return file_id diff --git a/src/gateway/services/mcp_loop_messages.py b/src/gateway/services/mcp_loop_messages.py index e11b971944..b1728a7785 100644 --- a/src/gateway/services/mcp_loop_messages.py +++ b/src/gateway/services/mcp_loop_messages.py @@ -20,7 +20,16 @@ from contextlib import aclosing from typing import TYPE_CHECKING, Any, Literal, Protocol, TypedDict, cast, runtime_checkable -from anthropic.types import ServerToolUseBlock, WebSearchResultBlock, WebSearchToolResultBlock, WebSearchToolResultError +from anthropic.types import ( + CodeExecutionOutputBlock, + CodeExecutionResultBlock, + CodeExecutionToolResultBlock, + CodeExecutionToolResultError, + ServerToolUseBlock, + WebSearchResultBlock, + WebSearchToolResultBlock, + WebSearchToolResultError, +) from anthropic.types.beta import BetaMCPToolResultBlock, BetaMCPToolUseBlock from any_llm import amessages from any_llm.types.messages import ( @@ -37,6 +46,7 @@ MaxToolIterationsExceeded, ToolBackend, ) +from gateway.services.sandbox_backend import CODE_EXECUTION_TOOL_NAME, CodeExecution from gateway.services.tool_format import openai_to_anthropic_tools from gateway.services.tool_usage import is_tool_error from gateway.services.web_retrieval_backend import WEB_RETRIEVAL_RESULT_MAX_BYTES, WEB_SEARCH_TOOL_NAME @@ -86,10 +96,12 @@ async def call_tool_outcome(self, name: str, arguments: dict[str, Any]) -> MCPTo # provider-native MCP blocks. MCP_ACTIVITY_ID_PREFIX = "otari_mcptoolu_" -# The gateway's own ``server_tool_use`` ids for web search. Anthropic issues -# ``srvtoolu_``-prefixed ids of its own, so a reserved prefix is what lets an echoed -# transcript be told apart from one describing a search the provider really ran. -WEB_SEARCH_TOOL_USE_ID_PREFIX = "otari_srvtoolu_" +# The gateway's own ``server_tool_use`` ids, for web search and code execution +# alike. Anthropic issues ``srvtoolu_``-prefixed ids of its own, so a reserved +# prefix is what lets an echoed transcript be told apart from one describing a +# call the provider really ran. +SERVER_TOOL_USE_ID_PREFIX = "otari_srvtoolu_" +WEB_SEARCH_TOOL_USE_ID_PREFIX = SERVER_TOOL_USE_ID_PREFIX # Anthropic beta capability a caller must declare before the Messages stream # includes the beta-only MCP activity block vocabulary. @@ -218,14 +230,61 @@ def _max_uses_exceeded_result( return {"type": "tool_result", "tool_use_id": tool_use_id, "content": MAX_USES_EXCEEDED_ERROR} -def _native_blocks_for_call(pool: ToolBackend, name: str, arguments: dict[str, Any]) -> list[Any]: - """Native blocks describing one completed gateway tool call, if it has any. +def _native_code_execution_blocks(execution: CodeExecution) -> list[Any]: + """A ``server_tool_use`` / ``code_execution_tool_result`` pair for one gateway execution. - Only ``web_search`` does: a sandbox or MCP call has no Anthropic block that - would be honest to emit (``code_execution_tool_result`` would claim Anthropic's - own container ran the code), so those stay invisible, as they do on Responses. + Emitted for a caller that declared code execution in Anthropic's own + vocabulary and whose request the gateway's sandbox ran instead. The result + block is the contract's own shape, which mirrors Anthropic's, so a client + parsing Anthropic responses reads it with no translation. A call the backend + never answered is reported in the vocabulary's error shape rather than + dropped, because the model was told about the failure and the client should + see the same story. """ - if name != WEB_SEARCH_TOOL_NAME: + tool_use_id = f"{SERVER_TOOL_USE_ID_PREFIX}{uuid.uuid4().hex}" + content: CodeExecutionResultBlock | CodeExecutionToolResultError + if execution.result is None: + content = CodeExecutionToolResultError(type="code_execution_tool_result_error", error_code="unavailable") + else: + result = execution.result.content + content = CodeExecutionResultBlock( + type="code_execution_result", + stdout=result.stdout, + stderr=result.stderr, + return_code=result.return_code if result.return_code is not None else 0, + # The ids are the ones ``/v1/files`` serves, not the sandbox's own: a + # produced file that was not stored has no id the caller could use. + content=[ + CodeExecutionOutputBlock(type="code_execution_output", file_id=file_id) + for file_id in execution.file_ids.values() + ], + ) + return [ + ServerToolUseBlock( + id=tool_use_id, + name=cast('Literal["code_execution"]', CODE_EXECUTION_TOOL_NAME), + input={"code": execution.code}, + type="server_tool_use", + ), + CodeExecutionToolResultBlock(tool_use_id=tool_use_id, type="code_execution_tool_result", content=content), + ] + + +def _native_blocks_for_call(pool: ToolBackend, name: str, arguments: dict[str, Any], *, is_error: bool) -> list[Any]: + """Native blocks describing one gateway tool call, if its tool has any. + + A web search contributes a pair only when it succeeded: a failed search has + nothing to cite. A code execution contributes one either way, because a + program that exited non-zero is a result the vocabulary can carry, and one + the backend never ran has an error shape of its own. An MCP call has no + Anthropic block that would be honest to emit and stays invisible. + """ + if name == CODE_EXECUTION_TOOL_NAME: + take_executions = getattr(pool, "take_executions", None) + if take_executions is None: + return [] + return [block for execution in take_executions() for block in _native_code_execution_blocks(execution)] + if is_error or name != WEB_SEARCH_TOOL_NAME: return [] take_last_results = getattr(pool, "take_last_results", None) if take_last_results is None: @@ -270,11 +329,10 @@ async def _execute_tool_uses( from ``BaseException`` and skip the ``Exception`` clause. Same idiom as :func:`gateway.services.mcp_loop._execute_mcp_calls`. - When ``native_blocks`` is given, each *successful* call appends the native - server-tool blocks describing it. A failed call contributes none: the model - still gets the ``[tool error]`` text, but there is no result to cite. Collecting - immediately after each awaited call is what makes the backend's single-slot - result buffer safe, since the calls run one at a time. + When ``native_blocks`` is given, each call appends the native server-tool + blocks describing it, per :func:`_native_blocks_for_call`. Collecting + immediately after each awaited call is what makes the backend's result buffer + safe, since the calls run one at a time. """ out: list[dict[str, Any]] = [] for block in blocks: @@ -293,8 +351,8 @@ async def _execute_tool_uses( else: if capped and budget is not None: budget.record(text) - if native_blocks is not None and not is_tool_error(text): - native_blocks.extend(_native_blocks_for_call(pool, block.name, arguments)) + if native_blocks is not None: + native_blocks.extend(_native_blocks_for_call(pool, block.name, arguments, is_error=is_tool_error(text))) out.append({"type": "tool_result", "tool_use_id": block.id, "content": text}) return out @@ -533,8 +591,8 @@ async def _execute_stream_owned_events( ) if capped and budget is not None: budget.record(text) - if not is_error and native_blocks is not None: - native_blocks.extend(_native_blocks_for_call(pool, name, parsed_input)) + if native_blocks is not None: + native_blocks.extend(_native_blocks_for_call(pool, name, parsed_input, is_error=is_error)) results.append({"type": "tool_result", "tool_use_id": spec["id"], "content": text}) if activity_id is not None: @@ -554,9 +612,9 @@ class _MessagesToolLoopStrategy: ``amessages`` is resolved as a module global at call time so tests can monkeypatch ``gateway.services.mcp_loop_messages.amessages``. - Native web-search and MCP activity emission are per-request capabilities, - so a request that wants either gets its own strategy instance rather than - sharing the module-level default one. + Native web-search, code-execution and MCP activity emission are per-request + capabilities, so a request that wants any gets its own strategy instance + rather than sharing the module-level default one. """ transcript_key = "messages" @@ -565,10 +623,12 @@ def __init__( self, *, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, emit_native_mcp: bool = False, budget: WebSearchBudget | None = None, ) -> None: self._emit_native_web_search = emit_native_web_search + self._emit_native_code_execution = emit_native_code_execution self._emit_native_mcp = emit_native_mcp # Absent unless the caller capped the searches, so the shared instance in # ``_strategy_for`` stays free of per-request state. @@ -576,7 +636,7 @@ def __init__( def _native_sink(self, sink: list[Any]) -> list[Any] | None: """``sink`` when native emission is on, else ``None`` (collect nothing).""" - return sink if self._emit_native_web_search else None + return sink if self._emit_native_web_search or self._emit_native_code_execution else None def coerce_transcript(self, value: Any) -> list[Any]: return list(value or []) @@ -933,6 +993,7 @@ def _strategy_for( budget: WebSearchBudget | None, *, emit_native_mcp: bool = False, + emit_native_code_execution: bool = False, ) -> _MessagesToolLoopStrategy: """The shared strategy, or a per-request one when any of the options is set. @@ -940,10 +1001,11 @@ def _strategy_for( module-level instance; a request wanting neither native emission nor a cap has nothing per-request to hold and keeps reusing it. """ - if not emit_native_web_search and not emit_native_mcp and budget is None: + if not emit_native_web_search and not emit_native_mcp and not emit_native_code_execution and budget is None: return _MESSAGES_STRATEGY return _MessagesToolLoopStrategy( emit_native_web_search=emit_native_web_search, + emit_native_code_execution=emit_native_code_execution, emit_native_mcp=emit_native_mcp, budget=budget, ) @@ -956,6 +1018,7 @@ async def anthropic_tool_loop( max_iterations: int, on_first_response: Callable[[], None] | None = None, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> MessageResponse: """Non-streaming Anthropic Messages tool-use loop. @@ -979,10 +1042,16 @@ async def anthropic_tool_loop( :func:`gateway.services._tool_loop.run_tool_loop`. With ``emit_native_web_search``, the returned content is prefixed with a - ``server_tool_use`` / ``web_search_tool_result`` pair per gateway-run search. + ``server_tool_use`` / ``web_search_tool_result`` pair per gateway-run search; + with ``emit_native_code_execution``, a ``server_tool_use`` / + ``code_execution_tool_result`` pair per gateway-run execution. """ return await run_tool_loop( - strategy=_strategy_for(emit_native_web_search, web_search_budget), + strategy=_strategy_for( + emit_native_web_search, + web_search_budget, + emit_native_code_execution=emit_native_code_execution, + ), completion_kwargs=completion_kwargs, pool=pool, max_iterations=max_iterations, @@ -996,6 +1065,7 @@ async def anthropic_tool_loop_stream( pool: ToolBackend, max_iterations: int, emit_native_web_search: bool = False, + emit_native_code_execution: bool = False, emit_native_mcp: bool = False, web_search_budget: WebSearchBudget | None = None, ) -> AsyncGenerator[MessageStreamEvent, None]: @@ -1034,6 +1104,7 @@ async def anthropic_tool_loop_stream( emit_native_web_search, web_search_budget, emit_native_mcp=emit_native_mcp, + emit_native_code_execution=emit_native_code_execution, ), completion_kwargs=completion_kwargs, pool=pool, diff --git a/src/gateway/services/mcp_loop_responses.py b/src/gateway/services/mcp_loop_responses.py index 33554d2888..ca5f023b7a 100644 --- a/src/gateway/services/mcp_loop_responses.py +++ b/src/gateway/services/mcp_loop_responses.py @@ -23,24 +23,28 @@ from __future__ import annotations import json +import uuid from collections.abc import AsyncGenerator, AsyncIterator, Callable from contextlib import aclosing from typing import TYPE_CHECKING, Any from any_llm import aresponses -from openai.types.responses import ResponseFunctionWebSearch +from openai.types.responses import ResponseCodeInterpreterToolCall, ResponseFunctionWebSearch +from openai.types.responses.response_code_interpreter_tool_call import OutputImage, OutputLogs from openai.types.responses.response_function_web_search import ActionSearch from openai.types.responses.response_output_item_added_event import ResponseOutputItemAddedEvent from openai.types.responses.response_output_item_done_event import ResponseOutputItemDoneEvent from gateway.log_config import logger from gateway.services._tool_loop import StreamAction, run_tool_loop, run_tool_loop_stream +from gateway.services.file_service import guess_mime_type from gateway.services.mcp_loop import ( DEFAULT_MAX_TOOL_ITERATIONS, MAX_TOOL_ITERATIONS_CAP, MaxToolIterationsExceeded, ToolBackend, ) +from gateway.services.sandbox_backend import CodeExecution from gateway.services.tool_format import openai_to_responses_tools from gateway.services.web_retrieval_backend import WEB_SEARCH_TOOL_NAME from gateway.services.web_search_budget import MAX_USES_EXCEEDED_ERROR, WebSearchBudget, is_capped_search @@ -215,19 +219,76 @@ def _web_search_call_item(call_id: str, query: str) -> ResponseFunctionWebSearch ) -def _web_search_items_for( +# The gateway's own ``code_interpreter_call`` item ids. OpenAI issues ``ci_`` +# ids, so a reserved prefix is what lets an echoed item be told apart from one +# describing a run OpenAI's own interpreter did (see ``routes/responses.py``). +CODE_INTERPRETER_CALL_ID_PREFIX = "otari_ci_" + + +def _produced_image_outputs(execution: CodeExecution, files_base_url: str | None) -> list[OutputImage]: + """``image`` outputs for the images a run produced, in the caller's vocabulary. + + OpenAI's only shape for a produced file here is a URL, so an image is + announced as the address Otari serves it from and anything else is left to + the files API, where every produced file is listed and downloadable by id. + """ + if not files_base_url: + return [] + return [ + OutputImage(type="image", url=f"{files_base_url}/{file_id}/content") + for filename, file_id in execution.file_ids.items() + if guess_mime_type(filename).startswith("image/") + ] + + +def _code_interpreter_call_item( + execution: CodeExecution, container_id: str, files_base_url: str | None = None +) -> ResponseCodeInterpreterToolCall: + """The Responses API's native "the server ran code" output item, for one gateway execution. + + Emitted for a caller that declared ``code_interpreter`` and whose request the + gateway's sandbox ran instead. ``outputs`` carries the run's logs, which is + what OpenAI's interpreter reports too, and an ``image`` entry per produced + image (see :func:`_produced_image_outputs`). + """ + outputs: list[OutputLogs | OutputImage] | None = None + status: str = "failed" + if execution.result is not None: + result = execution.result.content + logs = "".join(part for part in (result.stdout, result.stderr) if part) + outputs = [OutputLogs(type="logs", logs=logs)] if logs else None + images = _produced_image_outputs(execution, files_base_url) + if images: + outputs = [*(outputs or []), *images] + status = "completed" if result.return_code in (None, 0) else "failed" + return ResponseCodeInterpreterToolCall( + id=f"{CODE_INTERPRETER_CALL_ID_PREFIX}{uuid.uuid4().hex}", + code=execution.code, + container_id=container_id, + outputs=outputs, + status=status, # type: ignore[arg-type] + type="code_interpreter_call", + ) + + +def _native_items_for( owned: list[Any], + pool: ToolBackend, refused: set[str] | None = None, -) -> list[ResponseFunctionWebSearch]: - """Native items for the gateway-run searches among ``owned``. - - Only ``web_search`` maps to a Responses item the gateway can emit honestly. A - sandbox or MCP call has no native equivalent (``code_interpreter_call`` means - OpenAI's own interpreter ran, which would be a lie), so those stay invisible. - A call the ``max_uses`` cap refused is invisible for the same reason: no search - ran, so there is nothing to announce. + *, + emit_code_execution: bool = False, +) -> list[Any]: + """Native items for the gateway-run calls among ``owned``. + + ``web_search`` always maps to a ``web_search_call``, since the item needs only + an id, a query and a status. A call the ``max_uses`` cap refused is invisible: + no search ran, so there is nothing to announce. ``code_execution`` maps to a + ``code_interpreter_call`` only for a caller that declared the tool in + OpenAI's vocabulary (``emit_code_execution``), read off the executions the + sandbox backend kept for the calls just awaited. An MCP call has no native + equivalent and stays invisible. """ - items: list[ResponseFunctionWebSearch] = [] + items: list[Any] = [] for item in owned: if getattr(item, "name", None) != WEB_SEARCH_TOOL_NAME: continue @@ -238,9 +299,20 @@ def _web_search_items_for( except json.JSONDecodeError: query = "" items.append(_web_search_call_item(getattr(item, "call_id", "") or "", query)) + items.extend(_code_interpreter_items(pool, emit=emit_code_execution)) return items +def _code_interpreter_items(pool: ToolBackend, *, emit: bool) -> list[ResponseCodeInterpreterToolCall]: + """``code_interpreter_call`` items for the executions the backend kept, when asked.""" + take_executions = getattr(pool, "take_executions", None) + if not emit or take_executions is None: + return [] + container_id = str(getattr(pool, "container_id", "") or "") + files_base_url = getattr(pool, "files_base_url", None) + return [_code_interpreter_call_item(execution, container_id, files_base_url) for execution in take_executions()] + + def _compaction_items(output: list[Any]) -> list[Any]: """Return provider compaction items in output order.""" return [item for item in output if getattr(item, "type", None) == "compaction"] @@ -265,6 +337,7 @@ async def _execute_stream_owned( pool: ToolBackend, *, budget: WebSearchBudget | None = None, + emit_code_execution: bool = False, ) -> list[dict[str, Any]]: """Run the stream's gateway-owned function calls, returning their output items. @@ -297,6 +370,7 @@ async def _execute_stream_owned( if capped and budget is not None: budget.record(text) results.append({"type": "function_call_output", "call_id": spec["call_id"], "output": text}) + state.code_interpreter_items.extend(_code_interpreter_items(pool, emit=emit_code_execution)) return results @@ -356,6 +430,9 @@ def __init__(self) -> None: # ``call_id``s the max_uses cap refused this iteration, so their native # ``web_search_call`` item is not emitted. self.refused_call_ids: set[str] = set() + # Native items for this iteration's gateway-run code executions, minted + # right after the calls ran and drained by ``synthetic_events``. + self.code_interpreter_items: list[ResponseCodeInterpreterToolCall] = [] # Output items the gateway runs itself. Their events are swallowed: the # client can never be sent a ``function_call_output`` for a call the # gateway consumed, so showing it the call is a dead end. @@ -375,10 +452,11 @@ class _ResponsesToolLoopStrategy: transcript_key = "input_data" - def __init__(self, *, budget: WebSearchBudget | None = None) -> None: + def __init__(self, *, budget: WebSearchBudget | None = None, emit_native_code_execution: bool = False) -> None: # Absent unless the caller capped the searches, which keeps the shared # instance in ``_strategy_for`` free of per-request state. self._budget = budget + self._emit_native_code_execution = emit_native_code_execution def coerce_transcript(self, value: Any) -> list[Any]: return _coerce_input_to_list(value) @@ -393,10 +471,11 @@ async def call(self, kwargs: dict[str, Any]) -> Response: return result def new_usage_accumulator(self) -> dict[str, Any]: - # ``searches`` collects the gateway-run searches so the final response can - # announce them natively; ``compactions`` keeps replay state produced by - # hidden iterations available to the caller. See ``fold_usage``. - return {"input": 0, "output": 0, "total": 0, "searches": [], "compactions": []} + # ``native_items`` collects the gateway-run searches and executions so the + # final response can announce them natively; ``compactions`` keeps replay + # state produced by hidden iterations available to the caller. See + # ``fold_usage``. + return {"input": 0, "output": 0, "total": 0, "native_items": [], "compactions": []} def accumulate_usage(self, acc: dict[str, Any], result: Response) -> None: if result.usage: @@ -406,10 +485,10 @@ def accumulate_usage(self, acc: dict[str, Any], result: Response) -> None: def fold_usage(self, result: Response, acc: dict[str, Any]) -> None: _fold_usage(result, acc["input"], acc["output"], acc["total"]) - # Prepend a native ``web_search_call`` item per gateway-run search. The - # loop consumed the raw ``function_call`` items, so without this the caller - # has no way to know a search happened; they come first because they did. - hidden_output = list(acc["compactions"]) + list(acc["searches"]) + # Prepend a native item per gateway-run search or execution. The loop + # consumed the raw ``function_call`` items, so without this the caller has + # no way to know the call happened; they come first because they did. + hidden_output = list(acc["compactions"]) + list(acc["native_items"]) if hidden_output: try: result.output = hidden_output + list(result.output or []) @@ -426,11 +505,18 @@ def exit_after_split(self, result: Response) -> bool: return False async def execute_owned( - self, pool: ToolBackend, owned: list[Any], acc: Any = None + self, pool: ToolBackend, owned: list[Any], acc: dict[str, Any] | None = None ) -> list[dict[str, Any]]: - # ``acc`` is accepted for interface parity and unused: this format has no - # native vocabulary for a server-side tool call to report on a mixed batch. - return await _execute_function_calls(pool, owned, budget=self._budget) + # Mixed-batch exit: the owned subset runs for its side effects. Collect + # its native items too, since ``fold_usage`` runs on that path and + # prepends them, so the caller still sees the search or run it paid for. + refused: set[str] = set() + outputs = await _execute_function_calls(pool, owned, budget=self._budget, refused_call_ids=refused) + if acc is not None: + acc["native_items"].extend( + _native_items_for(owned, pool, refused, emit_code_execution=self._emit_native_code_execution) + ) + return outputs def filter_owned(self, result: Response, owned: list[Any], pool: ToolBackend) -> None: # Mixed batch: the owned subset was executed for its side effects; @@ -477,7 +563,11 @@ async def advance_transcript( transcript.extend(outputs) if acc is not None: acc["compactions"].extend(_compaction_items(output)) - acc["searches"].extend(_web_search_items_for(owned, refused_call_ids)) + acc["native_items"].extend( + _native_items_for( + owned, pool, refused_call_ids, emit_code_execution=self._emit_native_code_execution + ) + ) # ---- streaming hooks ---- @@ -508,6 +598,9 @@ def new_stream_accumulator(self) -> dict[str, Any]: "next_sequence": 0, "next_output_index": 0, "compactions": [], + # The native items announced mid-stream, kept so the terminal + # ``response.completed`` lists what the client already saw. + "native_items": [], } def observe( @@ -627,14 +720,16 @@ async def finalize_exit( pool: ToolBackend, acc: dict[str, Any], ) -> AsyncIterator[ResponseStreamEvent]: - del acc # Mixed batch: the gateway's function_call items were withheld from the # stream, so run them for their side effects rather than dropping the model's - # request. Matches the non-streaming loop's mixed-batch handling. + # request. Matches the non-streaming loop's mixed-batch handling, and like + # it announces the runs natively before the round exits. if state.owned_specs: - await _execute_stream_owned(state, pool, budget=self._budget) - return - yield # pragma: no cover - makes this a no-event async iterator + await _execute_stream_owned( + state, pool, budget=self._budget, emit_code_execution=self._emit_native_code_execution + ) + for event in self.synthetic_events(state, acc): + yield event def terminal_events(self, state: _ResponsesStreamState, acc: dict[str, Any]) -> list[ResponseStreamEvent]: if state.deferred_completed is None: @@ -645,7 +740,7 @@ def terminal_events(self, state: _ResponsesStreamState, acc: dict[str, Any]) -> # client just accumulated, and hand it a call it cannot dispatch. hidden = _hidden_call_ids(state) folded = _without_output_items(state.deferred_completed, hidden) if hidden else state.deferred_completed - folded = _prepend_output_items(folded, acc["compactions"]) + folded = _prepend_output_items(folded, [*acc["compactions"], *acc.get("native_items", [])]) folded = _maybe_fold_response_completed_usage(folded, acc["output_tokens"]) # The terminal event is the last thing the client sees, so it continues the # same sequence as the events forwarded before it. @@ -671,10 +766,12 @@ def synthetic_events( what an OpenAI-hosted search would have emitted, and unlike the Anthropic equivalent it is expressible without forging provider-signed content. - Only ``web_search`` is announced. A sandbox or MCP call has no native item - that would be honest to emit, so it stays invisible on the wire. + A gateway-run code execution is announced as a ``code_interpreter_call`` + for a caller that declared the tool in OpenAI's vocabulary. An MCP call + has no native item and stays invisible on the wire. """ events: list[ResponseStreamEvent] = [] + items: list[Any] = [] for spec in state.owned_specs: if spec.get("name") != WEB_SEARCH_TOOL_NAME: continue @@ -684,7 +781,11 @@ def synthetic_events( query = str(json.loads(spec.get("arguments") or "{}").get("query") or "") except json.JSONDecodeError: query = "" - item = _web_search_call_item(spec.get("call_id") or "", query) + items.append(_web_search_call_item(spec.get("call_id") or "", query)) + items.extend(state.code_interpreter_items) + state.code_interpreter_items = [] + acc.setdefault("native_items", []).extend(items) + for item in items: output_index = acc["next_output_index"] acc["next_output_index"] += 1 for event_cls, event_type in ( @@ -726,7 +827,11 @@ async def advance_stream_transcript( } ) transcript.extend(_items_to_dicts(replay_items)) - transcript.extend(await _execute_stream_owned(state, pool, budget=self._budget)) + transcript.extend( + await _execute_stream_owned( + state, pool, budget=self._budget, emit_code_execution=self._emit_native_code_execution + ) + ) return yield # pragma: no cover - makes this a no-event async iterator @@ -734,15 +839,18 @@ async def advance_stream_transcript( _RESPONSES_STRATEGY = _ResponsesToolLoopStrategy() -def _strategy_for(budget: WebSearchBudget | None) -> _ResponsesToolLoopStrategy: - """The shared strategy, or a per-request one when the caller capped searches. +def _strategy_for( + budget: WebSearchBudget | None, *, emit_native_code_execution: bool = False +) -> _ResponsesToolLoopStrategy: + """The shared strategy, or a per-request one when either option is set. - Only a capped request has anything per-request to hold, so every other request - keeps reusing the single module-level instance. + Only a capped request, or one owed native interpreter items, has anything + per-request to hold, so every other request keeps reusing the single + module-level instance. """ - if budget is None: + if budget is None and not emit_native_code_execution: return _RESPONSES_STRATEGY - return _ResponsesToolLoopStrategy(budget=budget) + return _ResponsesToolLoopStrategy(budget=budget, emit_native_code_execution=emit_native_code_execution) async def responses_tool_loop( @@ -752,6 +860,7 @@ async def responses_tool_loop( max_iterations: int, on_first_response: Callable[[], None] | None = None, web_search_budget: WebSearchBudget | None = None, + emit_native_code_execution: bool = False, ) -> Response: """Non-streaming OpenAI Responses tool-use loop. @@ -775,7 +884,7 @@ async def responses_tool_loop( reasoning items that can't be replayed against another provider. """ return await run_tool_loop( - strategy=_strategy_for(web_search_budget), + strategy=_strategy_for(web_search_budget, emit_native_code_execution=emit_native_code_execution), completion_kwargs=completion_kwargs, pool=pool, max_iterations=max_iterations, @@ -789,6 +898,7 @@ async def responses_tool_loop_stream( pool: ToolBackend, max_iterations: int, web_search_budget: WebSearchBudget | None = None, + emit_native_code_execution: bool = False, ) -> AsyncGenerator[ResponseStreamEvent, None]: """Streaming OpenAI Responses tool-use loop. @@ -809,7 +919,7 @@ async def responses_tool_loop_stream( # instead of waiting for event-loop async-generator finalization. async with aclosing( run_tool_loop_stream( - strategy=_strategy_for(web_search_budget), + strategy=_strategy_for(web_search_budget, emit_native_code_execution=emit_native_code_execution), completion_kwargs=completion_kwargs, pool=pool, max_iterations=max_iterations, diff --git a/src/gateway/services/sandbox_backend.py b/src/gateway/services/sandbox_backend.py index 29f493f8ff..3a97387962 100644 --- a/src/gateway/services/sandbox_backend.py +++ b/src/gateway/services/sandbox_backend.py @@ -20,6 +20,12 @@ timeout_seconds: int}`` → returns ``{result_block: {…}}`` * ``DELETE /sessions/{id}`` → tears the session down +* ``POST /sessions/{id}/files``, ``GET /sessions/{id}/files/list`` and + ``GET /sessions/{id}/files?path=…`` + → seed the request's uploads into the workspace + before the first call, then after each call list + the workspace and fetch what appeared or changed, + when a :class:`SandboxFiles` bridge is attached Session lifecycle is per-request: enter creates a session, exit destroys it. State does not persist across separate chat-completion @@ -37,8 +43,11 @@ from __future__ import annotations import logging +import uuid +from collections.abc import AsyncIterator from contextlib import AsyncExitStack -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Protocol import httpx from opentelemetry import trace @@ -50,10 +59,16 @@ if TYPE_CHECKING: from types import TracebackType + from gateway.services.file_service import StagedFile + logger = logging.getLogger(__name__) tracer = trace.get_tracer(__name__) CODE_EXECUTION_TOOL_NAME = "code_execution" +# The gateway's own container ids. OpenAI issues ``cntr_``-prefixed ids and +# Anthropic ``container_``-prefixed ones, so a reserved prefix is what lets an +# echoed native item be told apart from one describing a provider's container. +CONTAINER_ID_PREFIX = "otari_cntr_" # The code-execution tool kinds a policy may name, which is the vocabulary the # hosted ``CodeExecutionConfig.tools`` uses and the one the protocol's ``tool`` # field carries on the wire. This backend serves the first of them and no more, @@ -80,10 +95,47 @@ _DEFAULT_PURPOSE_HINT = ( "Prefer `code_execution` for any computation, data analysis, date " "arithmetic, statistics, or anything that benefits from exact output. " - "Python with numpy/pandas/scipy/sympy/matplotlib pre-installed." + "Python with numpy/pandas/scipy/sympy/matplotlib pre-installed. Files the " + "user attached are in the working directory under their own names. A file " + "you write there comes back with a file_id; give the user that file_id so " + "they can download it." ) +class SandboxFiles(Protocol): + """What the backend needs to move files in and out of a session. + + Implemented by :class:`gateway.services.files.SandboxFileBridge`; + a Protocol so the backend does not depend on the database-backed store and + a test can hand it a stub. + """ + + @property + def inputs(self) -> list[StagedFile]: + """The uploads to seed into the session, in message order.""" + ... + + @property + def max_output_files(self) -> int: + """Most produced files one call may store; the rest are named but not stored.""" + ... + + @property + def max_output_bytes(self) -> int: + """Total bytes one call may store across its produced files.""" + ... + + async def read_input(self, staged: StagedFile) -> bytes: ... + + async def store_output(self, filename: str, chunks: AsyncIterator[bytes]) -> str | None: + """Persist a produced file from ``chunks``, returning the ``file_id`` a caller downloads it by. + + ``None`` for an empty file, which is not worth a row. An exception the + chunk source raises propagates after the partial blob is removed. + """ + ... + + def code_execution_tool_definition() -> dict[str, Any]: """The OpenAI-shaped function definition the model is given for code execution. @@ -115,6 +167,25 @@ def code_execution_tool_definition() -> dict[str, Any]: } +@dataclass(frozen=True) +class CodeExecution: + """One executed call, kept for a loop that answers in a provider's native vocabulary. + + ``result`` is the backend's structured result block, so the loop can mint an + Anthropic ``code_execution_tool_result`` or an OpenAI ``code_interpreter_call`` + from the real stdout, stderr and exit code rather than re-parsing the string + the model was given. ``None`` when the call never produced one (the backend + was unreachable), which the loop renders as its vocabulary's error shape. + """ + + code: str + result: ResultBlock | None + # Produced filename -> the ``file_id`` it was stored under in ``/v1/files``, + # for the files a bridge collected. A produced file with no entry here is + # one the caller cannot download, so a native block does not announce it. + file_ids: dict[str, str] = field(default_factory=dict) + + class SandboxNotReachableError(RuntimeError): """Raised when the sandbox container can't be reached or returns malformed data.""" @@ -152,6 +223,29 @@ def _contract_violation(exc: ValidationError) -> str: return f"response does not match the code-execution contract ({fields})" +class _OutputOverBudget(Exception): + """A produced file ran past the bytes this call may still store.""" + + +class _CountedChunks: + """An async iterator over ``source`` that counts bytes and stops past ``budget``.""" + + def __init__(self, source: AsyncIterator[bytes], budget: int) -> None: + self._source = source + self._budget = budget + self.total = 0 + + def __aiter__(self) -> _CountedChunks: + return self + + async def __anext__(self) -> bytes: + chunk = await self._source.__anext__() + self.total += len(chunk) + if self.total > self._budget: + raise _OutputOverBudget + return chunk + + class SandboxBackend: """Async context manager that owns one sandbox session for a request's lifetime. @@ -174,8 +268,17 @@ def __init__( image: str | None = None, allowed_tools: frozenset[str] | None = None, tally: ToolUsageTally | None = None, + files: SandboxFiles | None = None, + files_base_url: str | None = None, ) -> None: self._sandbox_url = sandbox_url.rstrip("/") + # Where this deployment serves ``/v1/files`` from: a loop answering in + # OpenAI's vocabulary needs a URL to announce a produced image with. + # None outside a request (tests, direct use), which announces none. + self.files_base_url = files_base_url + # The request's file bridge, or None when it has no uploads to seed and + # nowhere to keep what a run produces (hybrid mode, tests, direct use). + self._files = files # Per-request accounting, owned by the route and passed in. None when the # backend runs outside a billed request (tests, direct use). self._tally = tally @@ -202,6 +305,17 @@ def __init__( self._client: httpx.AsyncClient | None = None self._session_id: str | None = None self._stack: AsyncExitStack = AsyncExitStack() + # The calls executed since the last ``take_executions``, in order. A loop + # that mints native result blocks drains this right after the awaited + # calls it made, which is what keeps a batch's blocks paired with the + # right calls: every tool loop runs its calls one at a time and in order. + self._executions: list[CodeExecution] = [] + # The workspace as last listed, path -> (size, modified_at). What a call + # produced is whatever differs from this afterwards; see ``_collect_outputs``. + self._workspace: dict[str, tuple[int, float | None]] = {} + # Minted per backend, so per request: what a Responses caller sees as the + # ``container_id`` of every interpreter call this request ran. + self.container_id = f"{CONTAINER_ID_PREFIX}{uuid.uuid4().hex}" async def __aenter__(self) -> SandboxBackend: try: @@ -224,8 +338,151 @@ async def __aenter__(self) -> SandboxBackend: except (httpx.HTTPError, ValueError) as exc: await self._stack.aclose() raise SandboxNotReachableError(f"failed to create sandbox session at {self._sandbox_url}: {exc}") from exc + try: + await self._seed_inputs() + if self._files is not None: + self._workspace = await self._list_workspace() + except BaseException: + # The session exists but the request cannot run as asked; release it + # rather than leaving it to the backend's idle reclaim. + await self.__aexit__(None, None, None) + raise return self + async def _seed_inputs(self) -> None: + """Write every staged upload into the session workspace before the model runs. + + A refused seed is terminal for the request: the code the model writes + would look for a file that is not there, and a run over a silently + missing input is worse than no run. + """ + if self._files is None or not self._files.inputs: + return + assert self._client is not None and self._session_id is not None + for staged in self._files.inputs: + try: + data = await self._files.read_input(staged) + except OSError as exc: + raise SandboxNotReachableError(f"could not read attachment {staged.file_id} for the sandbox") from exc + try: + response = await self._client.post( + f"{self._sandbox_url}/sessions/{self._session_id}/files", + files={"file": (staged.filename, data, staged.mime_type)}, + data={"path": staged.filename}, + ) + response.raise_for_status() + except httpx.HTTPError as exc: + raise SandboxNotReachableError(f"sandbox refused attachment {staged.file_id}: {exc}") from exc + logger.info("sandbox session %s seeded with file %s", self._session_id, staged.file_id) + + async def _list_workspace(self) -> dict[str, tuple[int, float | None]]: + """The session workspace's files, path -> (size, modified_at); empty when unlistable. + + ``ListFiles`` is optional in the contract, so a backend without it (404, + or any other failure) simply leaves the diff empty and the result block's + own file list as the only source of produced files. + """ + assert self._client is not None and self._session_id is not None + try: + response = await self._client.get(f"{self._sandbox_url}/sessions/{self._session_id}/files/list") + response.raise_for_status() + entries = response.json().get("files") + except (httpx.HTTPError, ValueError, AttributeError) as exc: + logger.debug("sandbox session %s workspace not listable: %s", self._session_id, exc) + return {} + listed: dict[str, tuple[int, float | None]] = {} + for entry in entries if isinstance(entries, list) else []: + if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): + continue + size = entry.get("size_bytes") + modified = entry.get("modified_at") + listed[entry["path"]] = ( + size if isinstance(size, int) else -1, + float(modified) if isinstance(modified, int | float) else None, + ) + return listed + + async def _produced_files(self, block: ResultBlock) -> list[str]: + """The files this call produced: what the block names, plus what the workspace diff shows. + + The contract's result block carries a list of produced files, but not + every backend fills it in (the reference container reports files only + through ``ListFiles``), so the two sources are unioned: the block's names + first, in its order, then every path that appeared or changed since the + last listing. A seeded input the code rewrote counts as produced. + """ + names = [ref.filename for ref in block.content.content if ref.filename] + if self._files is None: + return names + after = await self._list_workspace() + if after: + names += [path for path, stamp in after.items() if path not in names and self._workspace.get(path) != stamp] + self._workspace = after + return names + + async def _collect_outputs(self, block: ResultBlock) -> tuple[list[str], dict[str, str]]: + """Fetch the files a run produced and store each: the names produced, and filename to file_id. + + Best-effort per file: one that cannot be fetched or stored is still named + in the rendered result, just without an id, and the run itself stands. + What a run writes is untrusted, so one call may store at most + ``max_output_files`` files and ``max_output_bytes`` in total; the rest + are named only. + """ + if self._files is None: + return [], {} + assert self._client is not None and self._session_id is not None + produced = await self._produced_files(block) + max_files = self._files.max_output_files + if len(produced) > max_files: + logger.warning("sandbox produced %d files; storing the first %d", len(produced), max_files) + ids: dict[str, str] = {} + budget = self._files.max_output_bytes + for filename in produced[:max_files]: + try: + stored = await self._store_output(filename, budget) + except httpx.HTTPError as exc: + logger.warning("sandbox output %r could not be fetched: %s", filename, exc) + continue + except _OutputOverBudget: + logger.warning( + "sandbox output %r skipped: over the %d byte budget left for this call", filename, budget + ) + continue + except Exception as exc: # noqa: BLE001 — a storage failure must not fail the run + logger.warning("sandbox output %r could not be stored: %s", filename, exc) + continue + if stored is None: + continue + file_id, size = stored + ids[filename] = file_id + budget -= size + return produced, ids + + async def _store_output(self, filename: str, budget: int) -> tuple[str, int] | None: + """Stream one produced file from the sandbox into the store, returning its id and size. + + Never holds the file whole: the bytes go from the sandbox's response to + the store as they arrive, and the count is checked on the way, so a file + past ``budget`` is abandoned mid-stream (the store removes the partial + blob). A declared ``Content-Length`` past it is refused before a byte is + read. ``None`` for an empty file. + """ + assert self._client is not None and self._session_id is not None and self._files is not None + async with self._client.stream( + "GET", f"{self._sandbox_url}/sessions/{self._session_id}/files", params={"path": filename} + ) as response: + response.raise_for_status() + declared = response.headers.get("content-length", "") + if declared.isdigit() and int(declared) > budget: + raise _OutputOverBudget + counted = _CountedChunks(response.aiter_bytes(), budget) + file_id = await self._files.store_output(filename, counted) + if file_id is None: + logger.warning("sandbox output %r skipped: empty", filename) + return None + return file_id, counted.total + async def __aexit__( self, _exc_type: type[BaseException] | None, @@ -262,6 +519,16 @@ def _serves_code_execution(self) -> bool: """ return self._allowed_tools is None or CODE_EXECUTION_TOOL_NAME in self._allowed_tools + def take_executions(self) -> list[CodeExecution]: + """The calls executed since the last take, in order, clearing them. + + Consumed by a loop building native result blocks right after the calls it + awaited. Clearing means a later loop round cannot attribute an earlier + round's executions to its own calls. + """ + executions, self._executions = self._executions, [] + return executions + async def call_tool(self, name: str, arguments: dict[str, Any]) -> str: """Execute code and record the call on the request's tally. @@ -270,21 +537,23 @@ async def call_tool(self, name: str, arguments: dict[str, Any]) -> str: """ if not self.owns_tool(name): raise KeyError(f"SandboxBackend does not own tool {name!r}") + code = str(arguments.get("code") or "") try: - result = await self._exec_tool(arguments) + result, block, file_ids = await self._exec_tool(code) except Exception: + self._executions.append(CodeExecution(code=code, result=None)) if self._tally is not None: self._tally.record_failure(CODE_EXECUTION_TOOL_NAME) raise + self._executions.append(CodeExecution(code=code, result=block, file_ids=file_ids)) if self._tally is not None: self._tally.record_result(CODE_EXECUTION_TOOL_NAME, result) return result - async def _exec_tool(self, arguments: dict[str, Any]) -> str: + async def _exec_tool(self, code: str) -> tuple[str, ResultBlock, dict[str, str]]: if self._client is None or self._session_id is None: raise RuntimeError("SandboxBackend not entered as an async context manager") - code = arguments.get("code") or "" payload = { "tool": CODE_EXECUTION_TOOL_NAME, "input": {"code": code}, @@ -327,13 +596,16 @@ async def _exec_tool(self, arguments: dict[str, Any]) -> str: span.set_status(trace.StatusCode.ERROR, str(exc)) raise SandboxNotReachableError(f"sandbox exec failed: {exc}") from exc - result = _flatten_result_block(exec_response.result_block) + produced, file_ids = await self._collect_outputs(exec_response.result_block) + result = _flatten_result_block(exec_response.result_block, file_ids, produced) if result.startswith("[tool error]"): span.set_status(trace.StatusCode.ERROR, result) - return result + return result, exec_response.result_block, file_ids -def _flatten_result_block(block: ResultBlock) -> str: +def _flatten_result_block( + block: ResultBlock, file_ids: dict[str, str] | None = None, produced: list[str] | None = None +) -> str: """Render the structured result as a single string for the model. The tool loop hands the model one string per tool call, so the block's @@ -341,11 +613,17 @@ def _flatten_result_block(block: ResultBlock) -> str: ``return_code`` or a non-empty ``stderr``; the contract has no top-level ``is_error`` flag. - Passing the full structured result through to the caller (file refs as - content blocks, per-step exit codes) is a future enhancement that lands - alongside the Anthropic-content-block lift. + ``produced`` is every file the run wrote, as the block and the workspace + diff found them; ``file_ids`` maps those that were stored to the ``file_id`` + they were stored under, so the model can hand the user something + downloadable. A produced file with no id is listed by name alone. Passing + the full structured result through to the caller (file refs as content + blocks, per-step exit codes) is a future enhancement that lands alongside + the Anthropic-content-block lift. """ content = block.content + file_ids = file_ids or {} + produced = produced or [] parts: list[str] = [] if content.stdout: @@ -354,8 +632,11 @@ def _flatten_result_block(block: ResultBlock) -> str: parts.append(f"stderr:\n{content.stderr}") if content.return_code not in (None, 0): parts.append(f"return_code: {content.return_code}") - if content.content: - parts.append("files: " + ", ".join(ref.filename or "?" for ref in content.content)) + listed = [ref.filename or "?" for ref in content.content] + listed += [name for name in [*produced, *file_ids] if name not in listed] + if listed: + names = [f"{name} (file_id: {file_ids[name]})" if name in file_ids else name for name in listed] + parts.append("files: " + ", ".join(names)) flattened = "\n".join(parts) if not flattened: diff --git a/src/gateway/services/tenancy/workspace_code_execution_policy_service.py b/src/gateway/services/tenancy/workspace_code_execution_policy_service.py index 010d0e99b4..89d53a6d06 100644 --- a/src/gateway/services/tenancy/workspace_code_execution_policy_service.py +++ b/src/gateway/services/tenancy/workspace_code_execution_policy_service.py @@ -7,6 +7,9 @@ ``default_purpose_hint`` applies only when the request names none. ``tools`` intersects the tool kinds the sandbox backend serves, and an empty result refuses the request. ``image`` may name only an image on the operator's allow-list, because a settable image is a supply-chain surface. +``executor`` pins who runs a provider-named code-execution declaration (``auto``, ``otari`` or ``provider``) +over the deployment default and the request's header. It is a choice rather than a narrowing, and it grants +no sandbox the deployment has not configured. No row means no narrowing. Reads and writes both require an owner or admin of the organization or of the workspace. :func:`resolve_workspace_code_execution_policy` is a plain read with no identity, and the workspace comes from the key. @@ -32,6 +35,7 @@ from gateway.services.tenancy import authorization from gateway.services.tenancy.errors import SandboxImageNotAllowedError, SandboxToolsUnrunnableError from gateway.services.tenancy.organization_service import OrganizationService +from gateway.types.code_execution import CodeExecutor # The two ceilings a workspace value is floored against, which are also the # largest values worth storing: a policy may only narrow, so a number above the @@ -105,6 +109,15 @@ class WorkspaceCodeExecutionPolicyUpdate(BaseModel): "null exposes whatever it serves" ), ) + executor: CodeExecutor | None = Field( + default=None, + description=( + "Who runs a provider-native code-execution declaration for this workspace: 'auto' (the " + "provider when it runs the tool natively for the model, else this gateway's sandbox), " + "'otari' or 'provider'. Pins over the deployment default and over the request's " + "X-Otari-Code-Execution header; null leaves both in charge" + ), + ) @field_validator("tools") @classmethod @@ -168,6 +181,7 @@ class WorkspaceCodeExecutionPolicyPublic(BaseModel): exec_timeout_s: int | None image: str | None tools: list[str] | None + executor: CodeExecutor | None created_at: str | None updated_at: str | None @@ -191,6 +205,7 @@ def unconfigured( exec_timeout_s=None, image=None, tools=None, + executor=None, created_at=None, updated_at=None, ) @@ -215,6 +230,7 @@ def from_model( exec_timeout_s=policy.exec_timeout_s, image=policy.image, tools=list(policy.tools) if policy.tools is not None else None, + executor=CodeExecutor.parse(policy.executor), created_at=policy.created_at.isoformat(), updated_at=policy.updated_at.isoformat(), ) @@ -238,6 +254,11 @@ class ResolvedCodeExecutionPolicy: # ever asks whether a tool kind is in it, and an immutable one cannot be # edited by a backend it is handed to. tools: frozenset[str] | None + # The workspace's pin on who runs code, or ``None`` for "the deployment and + # the request decide". Parsed on the way out, so a stored value outside the + # vocabulary (which the write refuses) reads as no pin rather than failing + # every request. + executor: CodeExecutor | None = None async def resolve_workspace_code_execution_policy( @@ -260,6 +281,7 @@ async def resolve_workspace_code_execution_policy( exec_timeout_s=policy.exec_timeout_s, image=policy.image, tools=frozenset(policy.tools) if policy.tools is not None else None, + executor=CodeExecutor.parse(policy.executor), ) @@ -391,6 +413,7 @@ def _apply( policy.exec_timeout_s = request.exec_timeout_s policy.image = _blank_to_none(request.image) policy.tools = request.tools + policy.executor = request.executor.value if request.executor is not None else None async def clear_policy(self, *, user: User, workspace_id: uuid.UUID) -> WorkspaceCodeExecutionPolicyPublic: """Drop the workspace's policy, returning it to the deployment's behavior. diff --git a/src/gateway/services/tool_settings_service.py b/src/gateway/services/tool_settings_service.py index 23cde095e5..a5a4dc8945 100644 --- a/src/gateway/services/tool_settings_service.py +++ b/src/gateway/services/tool_settings_service.py @@ -35,6 +35,7 @@ from gateway.log_config import logger from gateway.models.platform import RuntimeSetting from gateway.services.runtime_settings_service import SettingValue +from gateway.types.code_execution import CodeExecutor WEB_SEARCH_URL = "web_search_url" WEB_SEARCH_ENGINES = "web_search_engines" @@ -45,6 +46,7 @@ SANDBOX_URL = "sandbox_url" SANDBOX_PURPOSE_HINT = "sandbox_purpose_hint" SANDBOX_SESSION_IMAGE = "sandbox_session_image" +CODE_EXECUTION_EXECUTOR = "code_execution_executor" GUARDRAILS_URL = "guardrails_url" @@ -54,11 +56,14 @@ class _ToolSpec: ``type`` is one of ``"url" | "str" | "int" | "bool"``. Every field is nullable (an empty value clears the override); ``ge`` is an inclusive lower - bound for ``int`` fields, mirroring the ``GatewayConfig`` field's constraint. + bound for ``int`` fields, mirroring the ``GatewayConfig`` field's constraint, + and ``choices`` closes a ``str`` field to a fixed vocabulary, which the + dashboard renders as a select. """ type: str ge: int | None = None + choices: tuple[str, ...] | None = None # The tool/guardrail config fields the dashboard may edit. These are the ``*_url`` @@ -73,6 +78,7 @@ class _ToolSpec: WEB_SEARCH_PURPOSE_HINT: _ToolSpec("str"), SANDBOX_PURPOSE_HINT: _ToolSpec("str"), SANDBOX_SESSION_IMAGE: _ToolSpec("str"), + CODE_EXECUTION_EXECUTOR: _ToolSpec("str", choices=tuple(executor.value for executor in CodeExecutor)), WEB_SEARCH_MAX_RESULTS: _ToolSpec("int", ge=1), WEB_SEARCH_EXTRACT: _ToolSpec("bool"), WEB_SEARCH_INTERCEPT: _ToolSpec("bool"), @@ -91,6 +97,7 @@ class _ToolSpec: SANDBOX_URL: "sandbox", SANDBOX_PURPOSE_HINT: "sandbox", SANDBOX_SESSION_IMAGE: "sandbox", + CODE_EXECUTION_EXECUTOR: "sandbox", GUARDRAILS_URL: "guardrails", } @@ -166,6 +173,12 @@ def validate_value(key: str, value: SettingValue) -> SettingValue: raise ValueError(msg) if spec.type == "url": return validate_url(value) + if spec.choices is not None: + normalized = value.strip().lower() + if normalized not in spec.choices: + msg = f"{key} must be one of {', '.join(spec.choices)}." + raise ValueError(msg) + return normalized return value @@ -283,3 +296,9 @@ def field_service(key: str) -> str: def field_type(key: str) -> str: """The display/validation type of a field ('url' | 'str' | 'int' | 'bool').""" return _TOOL_SPECS[key].type + + +def field_choices(key: str) -> list[str] | None: + """The closed vocabulary of a ``str`` field, or ``None`` for free text.""" + choices = _TOOL_SPECS[key].choices + return list(choices) if choices is not None else None diff --git a/src/gateway/types/code_execution.py b/src/gateway/types/code_execution.py index 7141c2a5b8..0205a5d8f9 100644 --- a/src/gateway/types/code_execution.py +++ b/src/gateway/types/code_execution.py @@ -24,6 +24,7 @@ from __future__ import annotations +from enum import StrEnum from typing import Annotated, Any from pydantic import BaseModel, BeforeValidator, ConfigDict, Field @@ -31,12 +32,41 @@ __all__ = [ "CodeExecutionFileRef", "CodeExecutionResult", + "CodeExecutor", "ExecResponse", "ResultBlock", "SessionHandle", ] +class CodeExecutor(StrEnum): + """Who runs the code a request's code-execution tool asks for. + + The one vocabulary shared by the deployment setting, the workspace policy, + the per-request header and the platform's resolve payload, so a value read + from any of them means the same thing at admission. + """ + + AUTO = "auto" + """The provider when it runs this tool natively for this model, else Otari.""" + OTARI = "otari" + """Always the configured code-execution backend, whatever the provider offers.""" + PROVIDER = "provider" + """Always the provider: the declaration is forwarded untouched.""" + + @classmethod + def parse(cls, value: object) -> CodeExecutor | None: + """The member for a stored or wire value, or ``None`` for anything else.""" + if isinstance(value, cls): + return value + if not isinstance(value, str): + return None + try: + return cls(value.strip().lower()) + except ValueError: + return None + + def _rendered_text(value: Any) -> Any: """Coerce whatever a backend put in a render-only field into text. @@ -95,6 +125,7 @@ class SessionHandle(_ContractModel): class CodeExecutionFileRef(_ContractModel): """A file the execution produced (a chart, a generated CSV).""" + file_id: _RenderedStr = "" filename: _RenderedStr = "" diff --git a/tests/integration/test_code_execution_executor.py b/tests/integration/test_code_execution_executor.py new file mode 100644 index 0000000000..6c25411b1f --- /dev/null +++ b/tests/integration/test_code_execution_executor.py @@ -0,0 +1,523 @@ +"""The executor decision on the request path: who runs a provider-native code declaration. + +The pure logic is covered by ``tests/unit/test_code_executor.py``. These pin the +wiring in ``prepare_gateway_tools``: which declarations are claimed, how the +deployment default, the workspace pin and the request header compose, and what a +claimed request hands the tool loop. +""" + +from __future__ import annotations + +from typing import Any, cast +from unittest.mock import AsyncMock, patch + +import pytest +from any_llm.types.messages import MessageResponse, MessageUsage, TextBlock +from fastapi.testclient import TestClient +from openai.types.responses import Response, ResponseUsage +from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails + +from gateway.core.config import API_ROOT + +_SANDBOX_URL = "http://127.0.0.1:9999/sandbox" +_ANTHROPIC = "anthropic:claude-3-5-sonnet-20241022" +_OPENAI = "openai:gpt-4o-mini" +_DATED = {"type": "code_execution_20250825", "name": "code_execution"} +_BARE = {"type": "code_execution"} +_INTERPRETER = {"type": "code_interpreter"} +_HEADER = "X-Otari-Code-Execution" + + +def _text_response(text: str = "ok") -> MessageResponse: + return MessageResponse( + id="msg_test", + type="message", + role="assistant", + model="claude-3-5-sonnet-20241022", + content=[TextBlock(type="text", text=text, citations=None)], + stop_reason=cast(Any, "end_turn"), + stop_sequence=None, + usage=MessageUsage(input_tokens=5, output_tokens=2), + ) + + +def _responses_response() -> Response: + return Response( + id="resp_test", + created_at=0.0, + model="fake", + object="response", + status=cast(Any, "completed"), + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseUsage( + input_tokens=5, + input_tokens_details=InputTokensDetails(cached_tokens=0), + output_tokens=2, + output_tokens_details=OutputTokensDetails(reasoning_tokens=0), + total_tokens=7, + ), + error=None, + incomplete_details=None, + instructions=None, + metadata=None, + temperature=None, + top_p=None, + ) + + +class _Seen: + """What one request did: forwarded to the provider, or claimed by the sandbox loop.""" + + def __init__(self) -> None: + self.provider_kwargs: dict[str, Any] | None = None + self.loop_kwargs: dict[str, Any] | None = None + self.loop_extra: dict[str, Any] | None = None + self.backend_kwargs: dict[str, Any] | None = None + + @property + def forwarded_tool_types(self) -> set[str]: + assert self.provider_kwargs is not None, "the provider was never called" + return {tool["type"] for tool in self.provider_kwargs.get("tools") or []} + + +def _post_messages(client: TestClient, headers: dict[str, str], body: dict[str, Any]) -> tuple[Any, _Seen]: + seen = _Seen() + + async def fake_amessages(**kwargs: Any) -> MessageResponse: + seen.provider_kwargs = kwargs + return _text_response("via-provider") + + async def fake_loop(*, completion_kwargs: Any, pool: Any, max_iterations: int, **extra: Any) -> MessageResponse: + seen.loop_kwargs = completion_kwargs + seen.loop_extra = extra + return _text_response("via-sandbox-loop") + + def fake_sandbox(**kwargs: Any) -> Any: + seen.backend_kwargs = kwargs + backend = AsyncMock() + backend.purpose_hints = lambda: [] + return AsyncMock(__aenter__=AsyncMock(return_value=backend), __aexit__=AsyncMock(return_value=None)) + + with ( + patch("gateway.api.routes.messages.amessages", new=fake_amessages), + patch("gateway.api.routes.messages.anthropic_tool_loop", new=fake_loop), + patch("gateway.api.routes._pipeline.SandboxBackend", new=fake_sandbox), + ): + response = client.post(f"{API_ROOT}/messages", json=body, headers=headers) + return response, seen + + +def _post_responses(client: TestClient, headers: dict[str, str], body: dict[str, Any]) -> tuple[Any, _Seen]: + seen = _Seen() + + async def fake_aresponses(**kwargs: Any) -> Response: + seen.provider_kwargs = kwargs + return _responses_response() + + async def fake_loop(*, completion_kwargs: Any, pool: Any, max_iterations: int, **extra: Any) -> Response: + seen.loop_kwargs = completion_kwargs + seen.loop_extra = extra + return _responses_response() + + def fake_sandbox(**kwargs: Any) -> Any: + seen.backend_kwargs = kwargs + backend = AsyncMock() + backend.purpose_hints = lambda: [] + return AsyncMock(__aenter__=AsyncMock(return_value=backend), __aexit__=AsyncMock(return_value=None)) + + with ( + patch("gateway.api.routes.responses.aresponses", new=fake_aresponses), + patch("gateway.api.routes.responses.responses_tool_loop", new=fake_loop), + patch("gateway.api.routes._pipeline.SandboxBackend", new=fake_sandbox), + ): + response = client.post(f"{API_ROOT}/responses", json=body, headers=headers) + return response, seen + + +def _messages_body(model: str, *tools: dict[str, Any]) -> dict[str, Any]: + return { + "model": model, + "messages": [{"role": "user", "content": "compute"}], + "max_tokens": 100, + "tools": list(tools), + } + + +def _default_workspace_id(client: TestClient, master_key_header: dict[str, str]) -> str: + listed = client.get(f"{API_ROOT}/workspaces", headers=master_key_header) + assert listed.status_code == 200 + workspace_id: str = listed.json()["data"][0]["id"] + return workspace_id + + +def _pin_executor(client: TestClient, master_key_header: dict[str, str], executor: str) -> None: + workspace_id = _default_workspace_id(client, master_key_header) + response = client.put( + f"{API_ROOT}/workspaces/{workspace_id}/code-execution-policy", + json={"enabled": True, "executor": executor}, + headers=master_key_header, + ) + assert response.status_code == 200, response.text + assert response.json()["executor"] == executor + + +# --- auto: the default ----------------------------------------------------------------- + + +def test_auto_leaves_anthropics_own_declaration_with_anthropic( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """The case an upgrade must not change: the provider runs what it runs natively.""" + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages(client, api_key_header, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution_20250825"} + assert seen.loop_kwargs is None + + +def test_auto_brings_anthropics_declaration_here_for_a_model_that_cannot_run_it( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """The transparent model swap: same request, other model, the sandbox runs the code.""" + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages(client, api_key_header, _messages_body(_OPENAI, _DATED)) + + assert response.status_code == 200, response.text + assert seen.provider_kwargs is None + assert seen.loop_kwargs is not None + assert "tools" not in seen.loop_kwargs or not seen.loop_kwargs["tools"], "the claimed declaration was forwarded" + # The caller spoke Anthropic's vocabulary, so it is answered in it. + assert seen.loop_extra is not None + assert seen.loop_extra.get("emit_native_code_execution") is True + + +def test_auto_claims_the_bare_keyword_which_no_provider_owns( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages(client, api_key_header, _messages_body(_ANTHROPIC, _BARE)) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + # The bare form implies no native response shape, so the plain result is kept. + assert seen.loop_extra is not None + assert "emit_native_code_execution" not in seen.loop_extra + + +def test_without_a_sandbox_a_provider_declaration_is_forwarded_and_nothing_else_happens( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OTARI_SANDBOX_URL", raising=False) + + response, seen = _post_messages(client, api_key_header, _messages_body(_OPENAI, _BARE)) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution"} + + +# --- the request header ---------------------------------------------------------------- + + +def test_the_header_can_bring_anthropics_declaration_here_even_for_anthropic( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages( + client, {**api_key_header, _HEADER: "otari"}, _messages_body(_ANTHROPIC, _DATED) + ) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + assert seen.loop_extra is not None + assert seen.loop_extra.get("emit_native_code_execution") is True + + +def test_the_header_can_leave_a_claimed_keyword_with_the_provider( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages( + client, {**api_key_header, _HEADER: "provider"}, _messages_body(_ANTHROPIC, _BARE) + ) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution"} + + +def test_a_header_outside_the_vocabulary_is_refused( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, _ = _post_messages(client, {**api_key_header, _HEADER: "anthropic"}, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 400 + assert response.json()["detail"]["error"]["type"] == "invalid_request_error" + assert _HEADER in response.json()["detail"]["error"]["message"] + + +def test_asking_for_otari_without_a_sandbox_is_refused( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OTARI_SANDBOX_URL", raising=False) + + response, _ = _post_messages(client, {**api_key_header, _HEADER: "otari"}, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 400 + assert "no sandbox is configured" in response.json()["detail"]["error"]["message"] + + +# --- the deployment default --------------------------------------------------------------- + + +def test_a_provider_default_forwards_every_provider_declaration( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + monkeypatch.setenv("OTARI_CODE_EXECUTION_EXECUTOR", "provider") + + response, seen = _post_messages(client, api_key_header, _messages_body(_OPENAI, _DATED)) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution_20250825"} + + +def test_the_explicit_type_is_always_the_gateways_whatever_the_default_says( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + monkeypatch.setenv("OTARI_CODE_EXECUTION_EXECUTOR", "provider") + + response, seen = _post_messages( + client, api_key_header, _messages_body(_ANTHROPIC, {"type": "otari_code_execution"}) + ) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + assert seen.loop_extra is not None + assert "emit_native_code_execution" not in seen.loop_extra + + +def test_an_otari_default_claims_anthropics_declaration_for_anthropic( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + monkeypatch.setenv("OTARI_CODE_EXECUTION_EXECUTOR", "otari") + + response, seen = _post_messages(client, api_key_header, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + assert seen.loop_extra is not None + assert seen.loop_extra.get("emit_native_code_execution") is True + + +# --- the workspace pin --------------------------------------------------------------------- + + +def test_a_workspace_pin_overrides_the_deployment_default( + client: TestClient, + api_key_header: dict[str, str], + master_key_header: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + _pin_executor(client, master_key_header, "otari") + + response, seen = _post_messages(client, api_key_header, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + + +def test_a_header_that_disagrees_with_the_pin_is_refused( + client: TestClient, + api_key_header: dict[str, str], + master_key_header: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + _pin_executor(client, master_key_header, "otari") + + response, _ = _post_messages(client, {**api_key_header, _HEADER: "provider"}, _messages_body(_ANTHROPIC, _DATED)) + + assert response.status_code == 403 + assert "pins" in response.json()["detail"]["error"]["message"] or "decides" in ( + response.json()["detail"]["error"]["message"] + ) + + +def test_the_pin_does_not_refuse_a_header_over_the_explicit_otari_tool( + client: TestClient, + api_key_header: dict[str, str], + master_key_header: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``otari_code_execution`` names no provider tool, so the pin has nothing to say. + + Without the header the same body runs on the sandbox whatever the pin is; + refusing it only because the header spelled that out would 403 a request + that is otherwise served. + """ + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + _pin_executor(client, master_key_header, "provider") + body = _messages_body(_ANTHROPIC, {"type": "otari_code_execution"}) + + without, _ = _post_messages(client, api_key_header, body) + assert without.status_code == 200, without.text + + with_header, seen = _post_messages(client, {**api_key_header, _HEADER: "otari"}, body) + + assert with_header.status_code == 200, with_header.text + assert seen.loop_kwargs is not None + + +def test_a_header_that_agrees_with_the_pin_is_fine( + client: TestClient, + api_key_header: dict[str, str], + master_key_header: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + _pin_executor(client, master_key_header, "provider") + + response, seen = _post_messages(client, {**api_key_header, _HEADER: "provider"}, _messages_body(_OPENAI, _DATED)) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution_20250825"} + + +def test_a_provider_pin_keeps_a_disabled_veto_out_of_the_way( + client: TestClient, + api_key_header: dict[str, str], + master_key_header: dict[str, str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``enabled=False`` vetoes the sandbox; a declaration the provider runs is not the sandbox.""" + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + workspace_id = _default_workspace_id(client, master_key_header) + stored = client.put( + f"{API_ROOT}/workspaces/{workspace_id}/code-execution-policy", + json={"enabled": False, "executor": "provider"}, + headers=master_key_header, + ) + assert stored.status_code == 200, stored.text + + response, seen = _post_messages(client, api_key_header, _messages_body(_OPENAI, _DATED)) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_execution_20250825"} + + +def test_the_policy_refuses_an_executor_outside_the_vocabulary( + client: TestClient, master_key_header: dict[str, str] +) -> None: + workspace_id = _default_workspace_id(client, master_key_header) + response = client.put( + f"{API_ROOT}/workspaces/{workspace_id}/code-execution-policy", + json={"enabled": True, "executor": "anthropic"}, + headers=master_key_header, + ) + assert response.status_code == 422 + + +# --- two declarations in one request ---------------------------------------------------------- + + +def test_the_explicit_type_beside_a_claimed_keyword_is_folded_in( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """Same request said twice, so one sandbox and the hint the keyword could not carry.""" + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_messages( + client, + api_key_header, + _messages_body(_OPENAI, {"type": "otari_code_execution", "purpose_hint": "Show your working"}, _DATED), + ) + + assert response.status_code == 200, response.text + assert seen.backend_kwargs is not None + assert seen.backend_kwargs["purpose_hint"] == "Show your working" + assert seen.loop_extra is not None + assert seen.loop_extra.get("emit_native_code_execution") is True + + +def test_the_explicit_type_beside_a_keyword_the_provider_keeps_is_still_two_sandboxes( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, _ = _post_messages( + client, api_key_header, _messages_body(_ANTHROPIC, {"type": "otari_code_execution"}, _DATED) + ) + + assert response.status_code == 400 + assert "cannot be combined with a provider-native" in response.json()["detail"]["error"]["message"] + + +# --- Responses ---------------------------------------------------------------------------------- + + +def test_responses_leaves_openais_interpreter_with_openai( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_responses( + client, api_key_header, {"model": _OPENAI, "input": "compute", "tools": [_INTERPRETER]} + ) + + assert response.status_code == 200, response.text + assert seen.forwarded_tool_types == {"code_interpreter"} + + +def test_responses_answers_a_claimed_interpreter_in_openais_vocabulary( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_responses( + client, {**api_key_header, _HEADER: "otari"}, {"model": _OPENAI, "input": "compute", "tools": [_INTERPRETER]} + ) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + assert seen.loop_extra is not None + assert seen.loop_extra.get("emit_native_code_execution") is True + + +def test_responses_brings_anthropics_words_here_but_answers_plainly( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + """Anthropic's keyword is not native on Responses, and its blocks do not exist there.""" + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, seen = _post_responses(client, api_key_header, {"model": _OPENAI, "input": "compute", "tools": [_DATED]}) + + assert response.status_code == 200, response.text + assert seen.loop_kwargs is not None + assert seen.loop_extra is not None + assert "emit_native_code_execution" not in seen.loop_extra + + +def test_responses_header_outside_the_vocabulary_is_refused_in_its_own_envelope( + client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OTARI_SANDBOX_URL", _SANDBOX_URL) + + response, _ = _post_responses( + client, {**api_key_header, _HEADER: "nobody"}, {"model": _OPENAI, "input": "compute", "tools": [_INTERPRETER]} + ) + + assert response.status_code == 400 + assert _HEADER in response.json()["detail"] diff --git a/tests/integration/test_files_endpoint.py b/tests/integration/test_files_endpoint.py index d22e5907c8..19dbfc2c3b 100644 --- a/tests/integration/test_files_endpoint.py +++ b/tests/integration/test_files_endpoint.py @@ -359,6 +359,32 @@ def test_expired_file_returns_404( assert client.get(f"{API_ROOT}/files/{file_id}/content", headers=api_key_header).status_code == 404 +def test_an_expired_file_is_not_listed( + client: TestClient, + api_key_header: dict[str, str], + tmp_file_store: None, + db_session: Session, +) -> None: + """The listing hides what every other verb 404s, rather than waiting for the sweep.""" + live = client.post( + f"{API_ROOT}/files", headers=api_key_header, files={"file": ("live.txt", b"hi", "text/plain")} + ).json()["id"] + gone = client.post( + f"{API_ROOT}/files", headers=api_key_header, files={"file": ("gone.txt", b"hi", "text/plain")} + ).json()["id"] + + record = db_session.get(FileObject, gone) + assert record is not None + record.expires_at = datetime.now(UTC) - timedelta(hours=1) + db_session.commit() + + listed = client.get(f"{API_ROOT}/files", headers=api_key_header) + assert listed.status_code == 200, listed.text + ids = {row["id"] for row in listed.json()["data"]} + assert live in ids + assert gone not in ids + + def test_vision_describe_side_call_is_billed( client: TestClient, master_key_header: dict[str, str], @@ -462,3 +488,207 @@ def test_files_user_mismatch_ignored_when_lenient( listing = client.get(f"{API_ROOT}/files", headers=api_key_header) assert listing.status_code == 200 assert any(f["id"] == file_id for f in listing.json()["data"]) + + +_ANTHROPIC = {"anthropic-version": "2023-06-01"} + + +def test_anthropic_sdk_headers_get_anthropic_shapes( + client: TestClient, api_key_header: dict[str, str], tmp_file_store: None +) -> None: + """The Anthropic SDK sends ``anthropic-version`` on every call and reads ``FileMetadata``.""" + headers = {**api_key_header, **_ANTHROPIC} + up = client.post(f"{API_ROOT}/files", headers=headers, files={"file": ("a.pdf", b"%PDF-1.4", "application/pdf")}) + assert up.status_code == 200, up.text + meta = up.json() + assert meta["type"] == "file" + assert meta["size_bytes"] == len(b"%PDF-1.4") + assert meta["mime_type"] == "application/pdf" + assert meta["downloadable"] is True + assert meta["created_at"].endswith("Z") + assert "object" not in meta and "bytes" not in meta + + got = client.get(f"{API_ROOT}/files/{meta['id']}", headers=headers) + assert got.status_code == 200 + assert got.json()["size_bytes"] == len(b"%PDF-1.4") + + listed = client.get(f"{API_ROOT}/files", headers=headers) + assert listed.status_code == 200 + page = listed.json() + assert "object" not in page + assert page["has_more"] is False + assert page["first_id"] == page["last_id"] == meta["id"] + + # The same file, read with OpenAI's headers, is the OpenAI object. + assert client.get(f"{API_ROOT}/files/{meta['id']}", headers=api_key_header).json()["object"] == "file" + + deleted = client.delete(f"{API_ROOT}/files/{meta['id']}", headers=headers) + assert deleted.status_code == 200 + assert deleted.json() == {"id": meta["id"], "type": "file_deleted"} + + +def test_list_is_cursor_paged(client: TestClient, api_key_header: dict[str, str], tmp_file_store: None) -> None: + ids = [ + client.post( + f"{API_ROOT}/files", headers=api_key_header, files={"file": (f"{n}.txt", b"x", "text/plain")} + ).json()["id"] + for n in range(3) + ] + + first = client.get(f"{API_ROOT}/files", headers=api_key_header, params={"limit": 2}).json() + assert first["object"] == "list" + assert len(first["data"]) == 2 + assert first["has_more"] is True + assert first["first_id"] == first["data"][0]["id"] + assert first["last_id"] == first["data"][1]["id"] + + second = client.get( + f"{API_ROOT}/files", headers=api_key_header, params={"limit": 2, "after": first["last_id"]} + ).json() + assert len(second["data"]) == 1 + assert second["has_more"] is False + seen = [f["id"] for f in first["data"] + second["data"]] + assert sorted(seen) == sorted(ids) + assert len(set(seen)) == 3 + + # Anthropic's cursor name, ascending, walks the same set the other way. + asc = client.get( + f"{API_ROOT}/files", headers={**api_key_header, **_ANTHROPIC}, params={"limit": 3, "order": "asc"} + ).json() + assert [f["id"] for f in asc["data"]] == list(reversed(seen)) + tail = client.get( + f"{API_ROOT}/files", + headers={**api_key_header, **_ANTHROPIC}, + params={"after_id": asc["data"][0]["id"], "order": "asc"}, + ).json() + assert [f["id"] for f in tail["data"]] == [f["id"] for f in asc["data"][1:]] + + # A cursor that has since been deleted is still a position: the usual + # "list a page, delete each, list again from last_id" loop must not 404 + # on its second page. + assert client.delete(f"{API_ROOT}/files/{first['last_id']}", headers=api_key_header).status_code == 200 + after_deleted = client.get( + f"{API_ROOT}/files", headers=api_key_header, params={"limit": 2, "after": first["last_id"]} + ) + assert after_deleted.status_code == 200 + assert [f["id"] for f in after_deleted.json()["data"]] == [second["data"][0]["id"]] + + # A cursor the caller never owned answers like a direct read of it would. + assert client.get(f"{API_ROOT}/files", headers=api_key_header, params={"after": "file-nope"}).status_code == 404 + assert client.get(f"{API_ROOT}/files", headers=api_key_header, params={"limit": 0}).status_code == 422 + + +def test_sweep_reclaims_expired_and_deleted_files( + client: TestClient, + api_key_header: dict[str, str], + tmp_file_store: None, + tmp_path: Path, + db_session: Session, + test_config: Any, +) -> None: + """Expiry hides a file; the sweep takes its bytes and row, and a deleted file's row with them.""" + import asyncio + + from sqlalchemy.engine import make_url + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + from gateway.core.unit_of_work import UnitOfWork + from gateway.services.files import sweep_files + + def _upload(name: str) -> str: + resp = client.post( + f"{API_ROOT}/files", headers=api_key_header, files={"file": (name, b"payload", "text/plain")} + ) + assert resp.status_code == 200, resp.text + return str(resp.json()["id"]) + + expired, deleted, live = _upload("expired.txt"), _upload("deleted.txt"), _upload("live.txt") + # Every row here is an upload, so its blob ref is set; ``storage_ref`` is + # nullable only for a file a provider holds. + refs = { + row.id: str(row.storage_ref) + for row in db_session.query(FileObject).filter(FileObject.id.in_([expired, deleted, live])).all() + } + db_session.query(FileObject).filter(FileObject.id == expired).update( + {"expires_at": datetime.now(UTC) - timedelta(hours=1)} + ) + db_session.commit() + assert client.delete(f"{API_ROOT}/files/{deleted}", headers=api_key_header).status_code == 200 + assert (tmp_path / refs[expired]).exists() + + store = LocalDirFileStore(str(tmp_path)) + + async def _sweep() -> int: + engine = create_async_engine(make_url(test_config.database_url).set(drivername="postgresql+asyncpg")) + try: + async with async_sessionmaker(engine)() as db, UnitOfWork(db) as uow: + batch = await sweep_files(uow, store, batch_size=10) + return batch.reclaimed + finally: + await engine.dispose() + + assert asyncio.run(_sweep()) == 2 + db_session.expire_all() + remaining = {row.id for row in db_session.query(FileObject).all()} + assert expired not in remaining and deleted not in remaining and live in remaining + assert not (tmp_path / refs[expired]).exists() + assert (tmp_path / refs[live]).exists() + assert client.get(f"{API_ROOT}/files/{live}", headers=api_key_header).status_code == 200 + + +def test_sweep_pages_past_rows_whose_blob_will_not_delete( + client: TestClient, + api_key_header: dict[str, str], + tmp_file_store: None, + tmp_path: Path, + db_session: Session, + test_config: Any, +) -> None: + """A row whose blob keeps failing must not park at the head and hide the rows behind it.""" + import asyncio + + from sqlalchemy.engine import make_url + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + from gateway.core.unit_of_work import UnitOfWork + from gateway.services.files import sweep_files + + ids = [] + for name in ("stuck-1.txt", "stuck-2.txt", "fine.txt"): + resp = client.post( + f"{API_ROOT}/files", headers=api_key_header, files={"file": (name, b"payload", "text/plain")} + ) + assert resp.status_code == 200, resp.text + ids.append(str(resp.json()["id"])) + db_session.query(FileObject).filter(FileObject.id.in_(ids)).update( + {"expires_at": datetime.now(UTC) - timedelta(hours=1)} + ) + db_session.commit() + refs = {row.id: str(row.storage_ref) for row in db_session.query(FileObject).filter(FileObject.id.in_(ids)).all()} + stuck = {refs[ids[0]], refs[ids[1]]} + + class _StickyStore(LocalDirFileStore): + async def delete(self, storage_ref: str) -> None: + if storage_ref in stuck: + raise PermissionError(storage_ref) + await super().delete(storage_ref) + + store = _StickyStore(str(tmp_path)) + + async def _sweep_two_batches() -> list[tuple[int, int]]: + engine = create_async_engine(make_url(test_config.database_url).set(drivername="postgresql+asyncpg")) + try: + async with async_sessionmaker(engine)() as db, UnitOfWork(db) as uow: + first = await sweep_files(uow, store, batch_size=2) + second = await sweep_files(uow, store, batch_size=2, after=first.cursor) + return [(first.seen, first.reclaimed), (second.seen, second.reclaimed)] + finally: + await engine.dispose() + + # The first batch is the two stuck rows and reclaims nothing; the second, + # started past them, reaches the one that can go. + assert asyncio.run(_sweep_two_batches()) == [(2, 0), (1, 1)] + db_session.expire_all() + remaining = {row.id for row in db_session.query(FileObject).filter(FileObject.id.in_(ids)).all()} + assert remaining == {ids[0], ids[1]} + assert not (tmp_path / refs[ids[2]]).exists() diff --git a/tests/integration/test_hybrid_mode_chat.py b/tests/integration/test_hybrid_mode_chat.py index f69229b7f3..26f8596fd6 100644 --- a/tests/integration/test_hybrid_mode_chat.py +++ b/tests/integration/test_hybrid_mode_chat.py @@ -2358,6 +2358,8 @@ def __init__( image: str | None = None, allowed_tools: frozenset[str] | None = None, tally: Any = None, + files: Any = None, + files_base_url: str | None = None, ) -> None: type(self).last_purpose_hint = purpose_hint type(self).last_image = image diff --git a/tests/integration/test_messages_route_dispatch.py b/tests/integration/test_messages_route_dispatch.py index 97dd972584..5c3d05d4e1 100644 --- a/tests/integration/test_messages_route_dispatch.py +++ b/tests/integration/test_messages_route_dispatch.py @@ -868,14 +868,18 @@ def test_code_execution_combined_with_mcp_servers_returns_400( ) -@pytest.mark.parametrize("native_type", ["code_execution", "code_interpreter", "code_execution_20250825"]) def test_code_execution_combined_with_a_provider_native_tool_returns_400( client: TestClient, api_key_header: dict[str, str], monkeypatch: pytest.MonkeyPatch, - native_type: str, ) -> None: - """Two sandboxes in one request have no single home for the caller's state.""" + """Two sandboxes in one request have no single home for the caller's state. + + Only a declaration the provider keeps is a second sandbox: Anthropic's dated + keyword against an Anthropic model. A keyword the executor brings here is the + same request said twice and is folded in instead + (``test_code_execution_executor.py``). + """ monkeypatch.setenv("OTARI_SANDBOX_URL", "http://127.0.0.1:9999/sandbox") resp = client.post( f"{API_ROOT}/messages", @@ -883,7 +887,7 @@ def test_code_execution_combined_with_a_provider_native_tool_returns_400( "model": "anthropic:claude-3-5-sonnet-20241022", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100, - "tools": [{"type": "otari_code_execution"}, {"type": native_type}], + "tools": [{"type": "otari_code_execution"}, {"type": "code_execution_20250825"}], }, headers=api_key_header, ) @@ -1175,11 +1179,11 @@ def test_echoed_gateway_activity_is_removed_before_prompt_estimation( async def fake_normalize_messages(input_messages: Any, **kwargs: Any) -> Any: captured["normalized_messages"] = input_messages - return input_messages, SimpleNamespace(vision_usage=lambda: None) + return input_messages, SimpleNamespace(vision_usage=lambda: None, sandbox_inputs=[]) async def fake_resolve_request_context(**kwargs: Any) -> Any: captured.update(kwargs) - await kwargs["normalize_messages"]("user", None, "model", None, None) + await kwargs["normalize_messages"]("user", None, "model", None, None, None) raise HTTPException(status_code=418, detail="stop after admission inputs") with ( diff --git a/tests/integration/test_provider_file_download.py b/tests/integration/test_provider_file_download.py new file mode 100644 index 0000000000..d48a94dc74 --- /dev/null +++ b/tests/integration/test_provider_file_download.py @@ -0,0 +1,313 @@ +"""Integration tests for downloading a file a provider's own sandbox produced. + +Such a file is a ``file_objects`` row with no ``storage_ref``: Otari holds the +record saying whose it is, and streams the bytes from the provider on demand. +The provider call itself is faked here (the URL and headers it builds are unit +tested); what these cover is the route, the tenant predicate, and what a +listing says about a file whose size Otari does not know. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator, AsyncIterator +from pathlib import Path +from typing import Any, cast +from unittest.mock import patch + +import httpx +import pytest +from anthropic.types import CodeExecutionOutputBlock, CodeExecutionResultBlock, CodeExecutionToolResultBlock +from any_llm.types.messages import ( + ContentBlockStartEvent, + MessageDelta, + MessageDeltaEvent, + MessageDeltaUsage, + MessageResponse, + MessageStartEvent, + MessageStopEvent, + MessageStreamEvent, +) +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + +from gateway.core.config import API_ROOT +from gateway.models.tools import FileObject +from gateway.services.file_store import LocalDirFileStore + +CHART = b"\x89PNG\r\n\x1a\nfake chart bytes" + + +@pytest.fixture +def tmp_file_store(client: TestClient, tmp_path: Path) -> None: + cast(Any, client.app).state.file_store = LocalDirFileStore(str(tmp_path)) + + +@pytest.fixture +def provider_file( + client: TestClient, + api_key_header: dict[str, str], + db_session: Session, + tmp_file_store: None, +) -> str: + """A file id owned by the test key's user, held by Anthropic rather than locally. + + Uploaded first so the row carries the same user and workspace an upload + does, then turned into a provider-held row, which is what a native code + execution records. + """ + upload = client.post( + f"{API_ROOT}/files", + headers=api_key_header, + files={"file": ("bar_plot.png", b"placeholder", "image/png")}, + data={"purpose": "user_data"}, + ) + assert upload.status_code == 200, upload.text + file_id = upload.json()["id"] + db_session.query(FileObject).filter(FileObject.id == file_id).update( + { + "storage_ref": None, + "provider": "anthropic", + "provider_container_id": None, + "purpose": "code_execution_output", + "bytes": 0, + } + ) + db_session.commit() + return str(file_id) + + +def _serving(payload: bytes) -> Any: + async def _stream(record: FileObject, config: Any) -> AsyncGenerator[bytes, None]: + del record, config + yield payload + + return _stream + + +def _refusing(exc: BaseException) -> Any: + async def _stream(record: FileObject, config: Any) -> AsyncGenerator[bytes, None]: + del record, config + raise exc + yield b"" # pragma: no cover - unreachable, keeps this a generator + + return _stream + + +def test_a_provider_held_file_streams_through_the_gateway( + client: TestClient, + api_key_header: dict[str, str], + provider_file: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("gateway.api.routes.files.stream_provider_file", _serving(CHART)) + + resp = client.get(f"{API_ROOT}/files/{provider_file}/content", headers=api_key_header) + + assert resp.status_code == 200 + assert resp.content == CHART + assert resp.headers["content-type"].startswith("image/png") + assert "bar_plot.png" in resp.headers["content-disposition"] + + +def test_metadata_answers_for_a_file_otari_does_not_hold( + client: TestClient, + api_key_header: dict[str, str], + provider_file: str, +) -> None: + resp = client.get(f"{API_ROOT}/files/{provider_file}", headers=api_key_header) + + assert resp.status_code == 200 + body = resp.json() + assert body["purpose"] == "code_execution_output" + # The provider does not say how many bytes there are until they are read. + assert body["bytes"] == 0 + assert body["filename"] == "bar_plot.png" + + +def test_a_provider_that_refuses_the_file_is_a_502( + client: TestClient, + api_key_header: dict[str, str], + provider_file: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "gateway.api.routes.files.stream_provider_file", + _refusing( + httpx.HTTPStatusError("410", request=httpx.Request("GET", "https://x"), response=httpx.Response(410)) + ), + ) + + resp = client.get(f"{API_ROOT}/files/{provider_file}/content", headers=api_key_header) + + assert resp.status_code == 502 + # The provider's own message never reaches the caller. + assert "410" not in resp.text + + +def test_a_deployment_with_no_credential_for_the_provider_is_a_500( + client: TestClient, + api_key_header: dict[str, str], + provider_file: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "gateway.api.routes.files.stream_provider_file", + _refusing(LookupError("no credential configured for provider 'anthropic'")), + ) + + resp = client.get(f"{API_ROOT}/files/{provider_file}/content", headers=api_key_header) + + assert resp.status_code == 500 + assert "anthropic" not in resp.text + + +def test_another_users_provider_file_is_not_found( + client: TestClient, + master_key_header: dict[str, str], + provider_file: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The row is what makes the proxy safe: the provider would serve any id. + + A master-key request names the user it reads as, so asking as someone else + is the same 404 a missing file gets, and the provider is never called. + """ + called = False + + def _unexpected(record: FileObject, config: Any) -> AsyncGenerator[bytes, None]: + nonlocal called + called = True + stream: AsyncGenerator[bytes, None] = _serving(CHART)(record, config) + return stream + + monkeypatch.setattr("gateway.api.routes.files.stream_provider_file", _unexpected) + + resp = client.get( + f"{API_ROOT}/files/{provider_file}/content", + headers=master_key_header, + params={"user": "somebody-else"}, + ) + + assert resp.status_code == 404 + assert called is False + + +# --- recording what a provider-native run produced ------------------------------------ + + +def _provider_run_block(file_id: str) -> CodeExecutionToolResultBlock: + """Anthropic's own result block for a run that wrote one file.""" + return CodeExecutionToolResultBlock( + type="code_execution_tool_result", + tool_use_id="srvtoolu_01provider", + content=CodeExecutionResultBlock( + type="code_execution_result", + stdout="", + stderr="", + return_code=0, + content=[CodeExecutionOutputBlock(type="code_execution_output", file_id=file_id)], + ), + ) + + +def _provider_reply(*blocks: Any) -> MessageResponse: + return MessageResponse( + id="msg_test", + type="message", + role="assistant", + model="claude-sonnet-4-5", + content=list(blocks), + stop_reason=cast(Any, "end_turn"), + stop_sequence=None, + usage=cast(Any, {"input_tokens": 3, "output_tokens": 2}), + ) + + +async def _stream_of(*events: MessageStreamEvent) -> AsyncIterator[MessageStreamEvent]: + for event in events: + yield event + + +def _native_request(*, stream: bool = False) -> dict[str, Any]: + return { + "model": "anthropic:claude-sonnet-4-5", + "messages": [{"role": "user", "content": "plot it"}], + "max_tokens": 100, + "tools": [{"type": "code_execution_20250825", "name": "code_execution"}], + "stream": stream, + } + + +@pytest.fixture +def anthropic_credentialed(monkeypatch: pytest.MonkeyPatch) -> None: + """A deployment credentialed for Anthropic by the SDK's own variable, with the metadata call faked.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test") + + async def _named(provider: str, file_id: str, api_key: str, api_base: str | None) -> str | None: + del provider, file_id, api_key, api_base + return "bar_plot.png" + + monkeypatch.setattr("gateway.services.files.provider_files._fetch_filename", _named) + + +def test_a_file_a_provider_native_run_produced_is_recorded_and_served( + client: TestClient, + api_key_header: dict[str, str], + db_session: Session, + tmp_file_store: None, + anthropic_credentialed: None, +) -> None: + """The provider ran the code and kept the file; the row Otari records is what + lets ``/v1/files`` answer for the id the caller was handed.""" + + async def fake_amessages(**kwargs: Any) -> MessageResponse: + return _provider_reply(_provider_run_block("file_01provider")) + + with patch("gateway.api.routes.messages.amessages", new=fake_amessages): + resp = client.post(f"{API_ROOT}/messages", json=_native_request(), headers=api_key_header) + assert resp.status_code == 200, resp.text + + meta = client.get(f"{API_ROOT}/files/file_01provider", headers=api_key_header) + assert meta.status_code == 200, meta.text + assert (meta.json()["filename"], meta.json()["purpose"]) == ("bar_plot.png", "code_execution_output") + row = db_session.get(FileObject, "file_01provider") + assert row is not None + assert (row.storage_ref, row.provider, row.provider_instance) == (None, "anthropic", "anthropic") + + +def test_a_streamed_provider_native_run_records_its_files_too( + client: TestClient, + api_key_header: dict[str, str], + tmp_file_store: None, + anthropic_credentialed: None, +) -> None: + """Anthropic's SDK streams by default, so the stream path owes the same row.""" + + async def fake_amessages(**kwargs: Any) -> AsyncIterator[MessageStreamEvent]: + return _stream_of( + MessageStartEvent(type="message_start", message=cast(Any, _provider_reply())), + ContentBlockStartEvent( + type="content_block_start", index=0, content_block=_provider_run_block("file_01streamed") + ), + MessageDeltaEvent( + type="message_delta", + delta=MessageDelta(stop_reason=cast(Any, "end_turn"), stop_sequence=None), + usage=MessageDeltaUsage( + input_tokens=None, + output_tokens=1, + cache_creation_input_tokens=None, + cache_read_input_tokens=None, + server_tool_use=None, + ), + ), + MessageStopEvent(type="message_stop"), + ) + + with patch("gateway.api.routes.messages.amessages", new=fake_amessages): + resp = client.post(f"{API_ROOT}/messages", json=_native_request(stream=True), headers=api_key_header) + assert resp.status_code == 200, resp.text + assert "file_01streamed" in resp.text + + meta = client.get(f"{API_ROOT}/files/file_01streamed", headers=api_key_header) + assert meta.status_code == 200, meta.text + assert meta.json()["filename"] == "bar_plot.png" diff --git a/tests/unit/test_chat_request_helpers.py b/tests/unit/test_chat_request_helpers.py index dd0a667f53..bb7b24754c 100644 --- a/tests/unit/test_chat_request_helpers.py +++ b/tests/unit/test_chat_request_helpers.py @@ -7,10 +7,11 @@ extracted — they stay in `tools[]` and pass through to the upstream provider, which executes them server-side. -Web search has one opt-in exception: with `intercept=True` the provider-named -web-search keywords are claimed too, so a client that can only speak a -provider's vocabulary reaches a configured gateway backend. Code execution has -no such mode, and an OpenAI `function` named `web_search` is never claimed. +With `intercept=True` the provider-named keywords are claimed too, so a client +that can only speak a provider's vocabulary reaches a configured gateway +backend. For web search that is the `web_search_intercept` opt-in; for code +execution it is the executor decision (`tests/unit/test_code_executor.py`). An +OpenAI `function` named `web_search` or `code_execution` is never claimed. """ from __future__ import annotations diff --git a/tests/unit/test_code_executor.py b/tests/unit/test_code_executor.py new file mode 100644 index 0000000000..61dfa20087 --- /dev/null +++ b/tests/unit/test_code_executor.py @@ -0,0 +1,316 @@ +"""Who runs a provider-native code-execution declaration: the executor decision. + +Pure logic in ``gateway.api.routes._tools`` plus the settings that feed it. The +request-path wiring (claiming the keyword, the policy pin, the header) is +covered by ``tests/integration/test_code_execution_executor.py``. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from any_llm import LLMProvider + +from gateway.api.routes._normalize import sandbox_requested +from gateway.api.routes._tools import ( + CODE_EXECUTION_HEADER, + _extract_code_execution_tool, + code_execution_declaration_forms, + decide_code_executor, + first_provider_code_execution_tool, + native_code_execution_dialect, + parse_code_execution_header, + provider_runs_code_natively, + resolve_code_executor_preference, +) +from gateway.core.config import GatewayConfig +from gateway.services.tool_settings_service import field_choices, validate_value +from gateway.types.code_execution import CodeExecutor + +ANTHROPIC_DATED = {"type": "code_execution_20250825", "name": "code_execution"} +OPENAI_INTERPRETER = {"type": "code_interpreter", "container": {"type": "auto"}} +BARE = {"type": "code_execution"} + + +# --- the vocabulary ----------------------------------------------------------- + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("auto", CodeExecutor.AUTO), + ("OTARI", CodeExecutor.OTARI), + (" provider ", CodeExecutor.PROVIDER), + (CodeExecutor.OTARI, CodeExecutor.OTARI), + ("anthropic", None), + ("", None), + (None, None), + (3, None), + ], +) +def test_parse_accepts_the_three_values_in_any_case_and_nothing_else( + raw: object, expected: CodeExecutor | None +) -> None: + assert CodeExecutor.parse(raw) is expected + + +def test_header_absent_or_blank_means_no_request_preference() -> None: + assert parse_code_execution_header(None) is None + assert parse_code_execution_header(" ") is None + + +def test_header_value_is_parsed_case_insensitively() -> None: + assert parse_code_execution_header("Otari") is CodeExecutor.OTARI + + +def test_header_outside_the_vocabulary_is_a_caller_error() -> None: + with pytest.raises(ValueError, match=CODE_EXECUTION_HEADER): + parse_code_execution_header("anthropic") + + +# --- composing the three layers --------------------------------------------------- + + +def test_deployment_default_applies_when_nobody_closer_said_otherwise() -> None: + assert resolve_code_executor_preference(requested=None, workspace=None, deployment=CodeExecutor.AUTO) == ( + CodeExecutor.AUTO, + False, + ) + + +def test_request_header_wins_over_the_deployment_default() -> None: + preference, conflict = resolve_code_executor_preference( + requested=CodeExecutor.OTARI, workspace=None, deployment=CodeExecutor.PROVIDER + ) + assert preference is CodeExecutor.OTARI + assert conflict is False + + +def test_workspace_pin_wins_over_both_and_a_disagreeing_header_is_a_conflict() -> None: + preference, conflict = resolve_code_executor_preference( + requested=CodeExecutor.PROVIDER, workspace=CodeExecutor.OTARI, deployment=CodeExecutor.AUTO + ) + assert preference is CodeExecutor.OTARI + assert conflict is True + + +def test_a_header_that_agrees_with_the_pin_is_not_a_conflict() -> None: + preference, conflict = resolve_code_executor_preference( + requested=CodeExecutor.OTARI, workspace=CodeExecutor.OTARI, deployment=CodeExecutor.PROVIDER + ) + assert preference is CodeExecutor.OTARI + assert conflict is False + + +# --- turning a preference into a decision ----------------------------------------- + + +@pytest.mark.parametrize("native_available", [True, False]) +@pytest.mark.parametrize("sandbox_configured", [True, False]) +def test_an_explicit_preference_is_returned_as_asked(native_available: bool, sandbox_configured: bool) -> None: + for explicit in (CodeExecutor.OTARI, CodeExecutor.PROVIDER): + assert ( + decide_code_executor(explicit, sandbox_configured=sandbox_configured, native_available=native_available) + is explicit + ) + + +def test_auto_prefers_a_provider_that_runs_the_tool_natively() -> None: + assert ( + decide_code_executor(CodeExecutor.AUTO, sandbox_configured=True, native_available=True) + is CodeExecutor.PROVIDER + ) + + +def test_auto_brings_the_code_here_when_the_provider_cannot_run_it() -> None: + assert ( + decide_code_executor(CodeExecutor.AUTO, sandbox_configured=True, native_available=False) is CodeExecutor.OTARI + ) + + +def test_auto_leaves_the_provider_in_charge_when_there_is_no_sandbox() -> None: + """Nothing to bring the code to, so the declaration is forwarded as it always was.""" + assert ( + decide_code_executor(CodeExecutor.AUTO, sandbox_configured=False, native_available=False) + is CodeExecutor.PROVIDER + ) + + +# --- which declarations a provider runs natively ----------------------------------- + + +def test_anthropics_dated_keyword_is_native_on_messages_against_anthropic() -> None: + assert provider_runs_code_natively(ANTHROPIC_DATED, provider="anthropic", dialect="messages") is True + + +def test_openais_interpreter_is_native_on_responses_against_openai() -> None: + assert provider_runs_code_natively(OPENAI_INTERPRETER, provider="openai", dialect="responses") is True + + +@pytest.mark.parametrize( + ("entry", "provider", "dialect"), + [ + (ANTHROPIC_DATED, "mistral", "messages"), # the model swap the executor exists for + (ANTHROPIC_DATED, "anthropic", "chat"), # no native form on Chat Completions + (ANTHROPIC_DATED, "anthropic", "responses"), # Anthropic's words in OpenAI's format + (OPENAI_INTERPRETER, "openai", "chat"), + (OPENAI_INTERPRETER, "anthropic", "responses"), + (BARE, "anthropic", "messages"), # the bare short form is nobody's + (BARE, "openai", "responses"), + (ANTHROPIC_DATED, None, "messages"), # provider unknown + (None, "anthropic", "messages"), + ], +) +def test_everything_else_is_not_natively_served( + entry: dict[str, str] | None, provider: str | None, dialect: str +) -> None: + assert provider_runs_code_natively(entry, provider=provider, dialect=dialect) is False + + +def test_provider_name_is_matched_case_insensitively() -> None: + assert provider_runs_code_natively(ANTHROPIC_DATED, provider="Anthropic", dialect="messages") is True + + +# --- which native result shape a caller expects back ----------------------------------- + + +def test_dated_anthropic_keyword_expects_messages_blocks() -> None: + assert native_code_execution_dialect(ANTHROPIC_DATED) == "messages" + + +def test_openai_interpreter_expects_a_responses_item() -> None: + assert native_code_execution_dialect(OPENAI_INTERPRETER) == "responses" + + +@pytest.mark.parametrize("entry", [BARE, {"type": "otari_code_execution"}, None, {}]) +def test_other_declarations_expect_the_plain_result(entry: dict[str, str] | None) -> None: + assert native_code_execution_dialect(entry) is None + + +# --- finding and claiming the keyword -------------------------------------------------- + + +def test_the_first_provider_keyword_is_found_without_being_removed() -> None: + tools: list[dict[str, Any]] = [{"type": "function", "function": {"name": "f"}}, OPENAI_INTERPRETER, BARE] + assert first_provider_code_execution_tool(tools) is OPENAI_INTERPRETER + assert len(tools) == 3 + + +def test_a_function_named_code_execution_is_the_callers_own() -> None: + assert first_provider_code_execution_tool([{"type": "function", "function": {"name": "code_execution"}}]) is None + + +def test_intercept_claims_the_provider_keyword_and_leaves_the_rest() -> None: + user_tool = {"type": "function", "function": {"name": "get_weather"}} + entry, remaining = _extract_code_execution_tool([user_tool, ANTHROPIC_DATED], intercept=True) + assert entry == ANTHROPIC_DATED + assert remaining == [user_tool] + + +def test_intercept_off_still_leaves_the_provider_keyword_alone() -> None: + entry, remaining = _extract_code_execution_tool([ANTHROPIC_DATED]) + assert entry is None + assert remaining == [ANTHROPIC_DATED] + + +# --- the deployment setting ---------------------------------------------------------- + + +def test_unset_means_auto() -> None: + assert GatewayConfig().effective_code_executor() is CodeExecutor.AUTO + + +def test_the_config_field_is_normalized_and_validated() -> None: + assert GatewayConfig(code_execution_executor="Provider").effective_code_executor() is CodeExecutor.PROVIDER + with pytest.raises(ValueError, match="code_execution_executor"): + GatewayConfig(code_execution_executor="anthropic") + + +def test_the_env_var_fills_in_when_the_override_is_cleared(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OTARI_CODE_EXECUTION_EXECUTOR", "otari") + config = GatewayConfig() + config.code_execution_executor = None + assert config.effective_code_executor() is CodeExecutor.OTARI + + +def test_the_dashboard_setting_is_a_closed_vocabulary() -> None: + assert field_choices("code_execution_executor") == ["auto", "otari", "provider"] + assert field_choices("sandbox_url") is None + assert validate_value("code_execution_executor", "OTARI") == "otari" + assert validate_value("code_execution_executor", "") is None + with pytest.raises(ValueError, match="must be one of"): + validate_value("code_execution_executor", "anthropic") + + +def test_declaration_forms_include_the_provider_keywords_unless_the_provider_owns_them() -> None: + assert code_execution_declaration_forms(GatewayConfig()) == [ + "otari_code_execution", + "code_execution", + "code_interpreter", + "code_execution_", + ] + assert code_execution_declaration_forms(GatewayConfig(code_execution_executor="provider")) == [ + "otari_code_execution" + ] + + +# --- staging attachments follows the executor --------------------------------------------- + + +def _requested( + tools: list[dict[str, Any]], + *, + provider: str | None, + dialect: str = "messages", + header: str | None = None, + pin: CodeExecutor | None = None, + **config: Any, +) -> bool: + return sandbox_requested( + tools, + config=GatewayConfig(sandbox_url="http://sandbox:8080", **config), + provider=LLMProvider(provider) if provider else None, + dialect=dialect, + code_execution_header=header, + workspace_executor=pin, + ) + + +def test_the_explicit_type_always_stages() -> None: + assert _requested([{"type": "otari_code_execution"}], provider="anthropic") is True + + +def test_a_natively_served_declaration_does_not_stage() -> None: + assert _requested([ANTHROPIC_DATED], provider="anthropic") is False + + +def test_a_declaration_the_provider_cannot_run_stages() -> None: + assert _requested([ANTHROPIC_DATED], provider="mistral") is True + assert _requested([BARE], provider="anthropic") is True + + +def test_the_deployment_default_decides_staging_too() -> None: + assert _requested([ANTHROPIC_DATED], provider="anthropic", code_execution_executor="otari") is True + assert _requested([BARE], provider="mistral", code_execution_executor="provider") is False + + +def test_a_workspace_pin_decides_staging_over_the_header_and_the_default() -> None: + # The same three layers admission composes: a pin pulling a natively served + # declaration here stages, and one pushing it to the provider does not. + assert _requested([ANTHROPIC_DATED], provider="anthropic", pin=CodeExecutor.OTARI) is True + assert _requested([ANTHROPIC_DATED], provider="mistral", pin=CodeExecutor.PROVIDER) is False + assert _requested([BARE], provider="mistral", header="otari", pin=CodeExecutor.PROVIDER) is False + + +def test_no_sandbox_means_nothing_is_staged() -> None: + assert ( + sandbox_requested( + [BARE], + config=GatewayConfig(), + provider=LLMProvider("mistral"), + dialect="messages", + code_execution_header=None, + ) + is False + ) diff --git a/tests/unit/test_content_normalizer.py b/tests/unit/test_content_normalizer.py index 2addd326fd..072a920850 100644 --- a/tests/unit/test_content_normalizer.py +++ b/tests/unit/test_content_normalizer.py @@ -228,3 +228,213 @@ async def test_disabled_is_noop() -> None: ) assert out == _image_msg() assert not stats.touched + + +def _stored(file_id: str = "file-csv", filename: str = "data.csv", mime: str = "text/csv") -> FileObject: + return FileObject( + id=file_id, + user_id="u", + filename=filename, + mime_type=mime, + bytes=9, + purpose="user_data", + storage_ref=f"x/{file_id}", + ) + + +def _patch_store(monkeypatch: pytest.MonkeyPatch, record: FileObject, data: bytes, reads: list[str]) -> None: + async def fake_fetch(db, file_id, user_id, *, workspace_id=None): # type: ignore[no-untyped-def] + return record if file_id == record.id else None + + async def fake_read(file_store, rec): # type: ignore[no-untyped-def] + reads.append(rec.id) + return data + + monkeypatch.setattr(cn, "fetch_file", fake_fetch) + monkeypatch.setattr(cn, "read_file_bytes", fake_read) + + +@pytest.mark.asyncio +async def test_container_upload_staged_for_sandbox_without_reading_bytes(monkeypatch: pytest.MonkeyPatch) -> None: + reads: list[str] = [] + _patch_store(monkeypatch, _stored(), b"a,b\n1,2\n", reads) + msgs = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Plot this."}, + {"type": "container_upload", "file_id": "file-csv"}, + ], + } + ] + out, stats = await normalize_messages( + msgs, + config=GatewayConfig(), + caps=_TEXT_ONLY, + fmt="anthropic", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + sandbox_requested=True, + ) + # Staged for the sandbox, named for the model, and the blob never loaded here. + assert [s.file_id for s in stats.sandbox_inputs] == ["file-csv"] + assert stats.sandbox_inputs[0].filename == "data.csv" + marker = out[0]["content"][1] + assert marker["type"] == "text" + assert "data.csv" in marker["text"] + assert reads == [] + + +@pytest.mark.asyncio +async def test_container_upload_without_sandbox_is_read_as_document(monkeypatch: pytest.MonkeyPatch) -> None: + reads: list[str] = [] + _patch_store(monkeypatch, _stored(), b"a,b\n1,2\n", reads) + + async def fake_extract(data: bytes, mime: str, filename: str | None) -> ExtractionResult: + return ExtractionResult("| a | b |", True, "ok") + + monkeypatch.setattr(cn, "extract_text_from_file", fake_extract) + msgs = [{"role": "user", "content": [{"type": "container_upload", "file_id": "file-csv"}]}] + out, stats = await normalize_messages( + msgs, + config=GatewayConfig(), + caps=_TEXT_ONLY, + fmt="anthropic", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + ) + assert stats.sandbox_inputs == [] + assert stats.files_extracted == 1 + assert "| a | b |" in out[0]["content"][0]["text"] + + +@pytest.mark.asyncio +async def test_document_file_id_is_also_staged_when_sandbox_runs(monkeypatch: pytest.MonkeyPatch) -> None: + reads: list[str] = [] + _patch_store(monkeypatch, _stored("file-pdf", "report.pdf", "application/pdf"), b"%PDF", reads) + msgs = [ + {"role": "user", "content": [{"type": "document", "source": {"type": "file", "file_id": "file-pdf"}}]}, + {"role": "user", "content": [{"type": "document", "source": {"type": "file", "file_id": "file-pdf"}}]}, + ] + out, stats = await normalize_messages( + msgs, + config=GatewayConfig(), + caps=_NATIVE, + fmt="anthropic", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + sandbox_requested=True, + ) + # The model still gets the document (inlined for a native model), and the + # sandbox gets it once even though it was referenced twice. + assert out[0]["content"][0]["source"]["type"] == "base64" + assert [s.file_id for s in stats.sandbox_inputs] == ["file-pdf"] + + +@pytest.mark.asyncio +async def test_bare_responses_input_file_item_is_normalized(monkeypatch: pytest.MonkeyPatch) -> None: + reads: list[str] = [] + _patch_store(monkeypatch, _stored("file-txt", "notes.txt", "text/plain"), b"hello", reads) + + async def fake_extract(data: bytes, mime: str, filename: str | None) -> ExtractionResult: + return ExtractionResult(data.decode(), True, "ok") + + monkeypatch.setattr(cn, "extract_text_from_file", fake_extract) + items = [ + {"role": "user", "content": "Summarize."}, + {"type": "input_file", "file_id": "file-txt"}, + ] + out, stats = await normalize_messages( + items, + config=GatewayConfig(), + caps=_TEXT_ONLY, + fmt="responses", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + ) + assert stats.files_extracted == 1 + # Extracted text cannot sit bare in ``input``; it is wrapped in a user message. + assert out[1]["role"] == "user" + assert out[1]["content"][0]["type"] == "input_text" + assert "hello" in out[1]["content"][0]["text"] + + +@pytest.mark.asyncio +async def test_bare_responses_input_file_item_inlined_for_native(monkeypatch: pytest.MonkeyPatch) -> None: + reads: list[str] = [] + _patch_store(monkeypatch, _stored("file-pdf", "r.pdf", "application/pdf"), b"%PDF", reads) + items = [{"type": "input_file", "file_id": "file-pdf"}] + out, _ = await normalize_messages( + items, + config=GatewayConfig(), + caps=_NATIVE, + fmt="responses", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + ) + # Stays a bare item, now carrying inline data the provider can read. + assert out[0]["type"] == "input_file" + assert out[0]["file_data"].startswith("data:application/pdf;base64,") + + +@pytest.mark.parametrize( + ("filename", "taken", "expected"), + [ + ("data.csv", set(), "data.csv"), + ("reports/q3/data.csv", set(), "data.csv"), + ("..\\..\\etc\\passwd", set(), "passwd"), + ("../", set(), "file"), + ("data.csv", {"data.csv"}, "data-2.csv"), + ("data.csv", {"data.csv", "data-2.csv"}, "data-3.csv"), + ("Makefile", {"Makefile"}, "Makefile-2"), + (".env", {".env"}, ".env-2"), + ], +) +def test_sandbox_path_for(filename: str, taken: set[str], expected: str) -> None: + from gateway.services.file_service import sandbox_path_for + + assert sandbox_path_for(filename, taken) == expected + + +@pytest.mark.asyncio +async def test_two_uploads_named_alike_are_both_staged_and_the_model_learns_both_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _stored("file-a", "data.csv") + second = _stored("file-b", "data.csv") + + async def fake_fetch(db, file_id, user_id, *, workspace_id=None): # type: ignore[no-untyped-def] + return {"file-a": first, "file-b": second}.get(file_id) + + monkeypatch.setattr(cn, "fetch_file", fake_fetch) + msgs = [ + { + "role": "user", + "content": [ + {"type": "container_upload", "file_id": "file-a"}, + {"type": "container_upload", "file_id": "file-b"}, + # The same file again is staged once, under the name it already has. + {"type": "container_upload", "file_id": "file-a"}, + ], + } + ] + out, stats = await normalize_messages( + msgs, + config=GatewayConfig(), + caps=_TEXT_ONLY, + fmt="anthropic", + db=cast(Any, object()), + file_store=cast(Any, object()), + user_id="u", + sandbox_requested=True, + ) + assert [(s.file_id, s.filename) for s in stats.sandbox_inputs] == [("file-a", "data.csv"), ("file-b", "data-2.csv")] + markers = [block["text"] for block in out[0]["content"]] + assert "data.csv" in markers[0] + assert "data-2.csv" in markers[1] + assert "data.csv" in markers[2] and "data-2" not in markers[2] diff --git a/tests/unit/test_fsspec_file_store.py b/tests/unit/test_fsspec_file_store.py new file mode 100644 index 0000000000..3f4b5e39e5 --- /dev/null +++ b/tests/unit/test_fsspec_file_store.py @@ -0,0 +1,150 @@ +"""Unit tests for the fsspec-backed file store. + +Runs on fsspec's built-in ``memory://`` and ``file://`` filesystems, so the +suite needs no cloud implementation package and no network. What it proves is +the adapter's own contract (refs, streaming, cleanup, error translation); the +cloud implementations are fsspec's to keep working. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path + +import pytest + +from gateway.core.config import GatewayConfig +from gateway.services.file_store import FsspecFileStore, build_file_store + +# The backend is an optional extra; the store module itself imports it lazily. +fsspec = pytest.importorskip("fsspec") + + +async def _iter(chunks: list[bytes]) -> AsyncIterator[bytes]: + for chunk in chunks: + yield chunk + + +@pytest.fixture +def memory_root() -> str: + # The memory filesystem is process-global; give each test its own prefix + # and clear it afterwards so one test's blobs never show up in another. + root = "memory://otari-test" + fs = fsspec.filesystem("memory") + if fs.exists("otari-test"): + fs.rm("otari-test", recursive=True) + return root + + +@pytest.mark.asyncio +async def test_put_get_roundtrip(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + ref = await store.put("file-abcdef0123", b"hello bytes") + assert ref == "ab/file-abcdef0123" + assert await store.get(ref) == b"hello bytes" + + +@pytest.mark.asyncio +async def test_put_stream_and_get_stream_roundtrip(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + payload = b"x" * (2 * 1024 * 1024 + 5) + ref, size = await store.put_stream("file-streamtest01", _iter([payload[:1000], payload[1000:]])) + assert size == len(payload) + collected = bytearray() + async for chunk in store.get_stream(ref): + collected.extend(chunk) + assert bytes(collected) == payload + + +@pytest.mark.asyncio +async def test_put_stream_removes_partial_blob_on_failure(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + + async def _failing() -> AsyncIterator[bytes]: + yield b"partial" + raise RuntimeError("client went away") + + with pytest.raises(RuntimeError): + await store.put_stream("file-partial00001", _failing()) + assert not fsspec.filesystem("memory").exists("otari-test/pa/file-partial00001") + + +@pytest.mark.asyncio +async def test_put_stream_removes_partial_blob_on_cancellation(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + started = asyncio.Event() + + async def _slow() -> AsyncIterator[bytes]: + yield b"first" + started.set() + await asyncio.sleep(30) + yield b"never" + + task = asyncio.create_task(store.put_stream("file-cancel000001", _slow())) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert not fsspec.filesystem("memory").exists("otari-test/ca/file-cancel000001") + + +@pytest.mark.asyncio +async def test_missing_blob_is_file_not_found(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + with pytest.raises(FileNotFoundError): + await store.get("no/file-nope") + with pytest.raises(FileNotFoundError): + async for _ in store.get_stream("no/file-nope"): + pass + + +@pytest.mark.asyncio +async def test_delete_is_idempotent(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + ref = await store.put("file-deleteme0001", b"x") + await store.delete(ref) + await store.delete(ref) + with pytest.raises(FileNotFoundError): + await store.get(ref) + + +@pytest.mark.asyncio +async def test_rejects_refs_that_could_leave_the_root(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + for bad in ("../escape", "/absolute", "a//b", "a/./b", ""): + with pytest.raises(ValueError): + await store.get(bad) + + +@pytest.mark.asyncio +async def test_backend_client_errors_become_oserror(memory_root: str) -> None: + store = FsspecFileStore(memory_root) + + def _boom(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("some client's own exception class") + + store._fs.cat_file = _boom + with pytest.raises(OSError, match="fsspec operation failed"): + await store.get("ab/file-abcdef0123") + + +@pytest.mark.asyncio +async def test_local_file_protocol_writes_under_the_root(tmp_path: Path) -> None: + store = FsspecFileStore(f"file://{tmp_path}") + ref = await store.put("file-abcdef0123", b"on disk") + assert (tmp_path / "ab" / "file-abcdef0123").read_bytes() == b"on disk" + assert await store.get(ref) == b"on disk" + + +def test_build_file_store_fsspec_requires_url() -> None: + cfg = GatewayConfig(files_backend="fsspec") + with pytest.raises(ValueError, match="files_url"): + build_file_store(cfg) + + +def test_build_file_store_fsspec(tmp_path: Path) -> None: + cfg = GatewayConfig( + files_backend="fsspec", files_url=f"file://{tmp_path}", files_storage_options={"auto_mkdir": True} + ) + assert isinstance(build_file_store(cfg), FsspecFileStore) diff --git a/tests/unit/test_gateway_lifespan_shutdown.py b/tests/unit/test_gateway_lifespan_shutdown.py index 054c1fa0eb..f023813d73 100644 --- a/tests/unit/test_gateway_lifespan_shutdown.py +++ b/tests/unit/test_gateway_lifespan_shutdown.py @@ -231,3 +231,21 @@ async def test_the_reservation_sweeper_is_the_one_worker_a_setting_turns_off( assert "budget reservation sweep" not in names assert names == [worker.name for worker in _LIFESPAN_WORKERS if worker.name != "budget reservation sweep"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config", + [ + GatewayConfig(master_key="sk-test-master", files_sweep_interval_sec=0), + GatewayConfig(master_key="sk-test-master", files_enabled=False), + ], +) +async def test_the_file_sweeper_stops_with_files_or_its_interval( + monkeypatch: pytest.MonkeyPatch, config: GatewayConfig +) -> None: + """Disabling files, or the sweep alone, drops that one worker and no other.""" + names, _called = await _started_worker_names(config, monkeypatch) + + assert "file retention sweep" not in names + assert names == [worker.name for worker in _LIFESPAN_WORKERS if worker.name != "file retention sweep"] diff --git a/tests/unit/test_mcp_loop_messages.py b/tests/unit/test_mcp_loop_messages.py index 4cf3f55929..368f86049c 100644 --- a/tests/unit/test_mcp_loop_messages.py +++ b/tests/unit/test_mcp_loop_messages.py @@ -33,17 +33,20 @@ from gateway.services import mcp_loop_messages as messages_loop_module from gateway.services.mcp_client import MCPToolCallOutcome from gateway.services.mcp_loop_messages import ( + SERVER_TOOL_USE_ID_PREFIX, WEB_SEARCH_TOOL_USE_ID_PREFIX, MaxToolIterationsExceeded, anthropic_tool_loop, anthropic_tool_loop_stream, ) +from gateway.services.sandbox_backend import CodeExecution from gateway.services.tool_format import ( inject_purpose_hints_anthropic, openai_to_anthropic_tools, ) from gateway.services.web_retrieval_backend import WEB_RETRIEVAL_RESULT_MAX_BYTES from gateway.services.web_search_budget import WebSearchBudget +from gateway.types.code_execution import ResultBlock class _FakePool: @@ -2098,3 +2101,219 @@ async def fake_amessages(**kwargs: Any) -> AsyncIterator[MessageStreamEvent]: assert types[-2:] == ["message_delta", "message_stop"] # The search really ran. assert pool.calls == [("web_search", {"query": "python"})] + + +# --- native code-execution blocks ----------------------------------------------------- + + +def _exec_result(stdout: str = "42\n", stderr: str = "", return_code: int = 0) -> ResultBlock: + return ResultBlock.model_validate( + { + "type": "code_execution_tool_result", + "content": { + "type": "code_execution_result", + "stdout": stdout, + "stderr": stderr, + "return_code": return_code, + "content": [{"type": "code_execution_output", "file_id": "file_1", "filename": "chart.png"}], + }, + } + ) + + +class _FakeSandboxPool(_FakePool): + """A pool that owns ``code_execution`` and keeps executions like the real backend. + + ``take_executions`` is what marks it as the gateway's sandbox rather than an + MCP server that happens to expose the same tool name. + """ + + def __init__(self, *, result: ResultBlock | None = _exec_result(), fail: bool = False) -> None: + text = "[tool error] boom" if result is not None and result.content.return_code else "stdout:\n42" + super().__init__(tool_names=["code_execution"], results={"code_execution": text}) + self._result = result + self._fail = fail + self._executions: list[CodeExecution] = [] + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> str: + self.calls.append((name, arguments)) + code = str(arguments.get("code") or "") + if self._fail: + self._executions.append(CodeExecution(code=code, result=None)) + raise RuntimeError("sandbox down") + self._executions.append( + CodeExecution(code=code, result=self._result, file_ids={"chart.png": "file-stored-1"}) + ) + return self._results["code_execution"] + + def take_executions(self) -> list[CodeExecution]: + taken, self._executions = self._executions, [] + return taken + + +def _code_use(block_id: str = "tu_1", code: str = "print(6 * 7)") -> ToolUseBlock: + return _tool_use(block_id, "code_execution", {"code": code}) + + +@pytest.mark.asyncio +async def test_native_code_execution_pair_is_prepended_to_the_final_content(monkeypatch: pytest.MonkeyPatch) -> None: + responses = [ + _message_response(stop_reason="tool_use", content=[_code_use()]), + _message_response(stop_reason="end_turn", content=[_text_block("42")]), + ] + monkeypatch.setattr(messages_loop_module, "amessages", _fake_amessages_for(responses)) + + result = await anthropic_tool_loop( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, _FakeSandboxPool()), + max_iterations=5, + emit_native_code_execution=True, + ) + + assert [b.type for b in result.content] == ["server_tool_use", "code_execution_tool_result", "text"] + server_use, tool_result, _text = (cast(Any, block) for block in result.content) + assert server_use.name == "code_execution" + assert server_use.input == {"code": "print(6 * 7)"} + assert server_use.id.startswith(SERVER_TOOL_USE_ID_PREFIX) + assert tool_result.tool_use_id == server_use.id + assert tool_result.content.type == "code_execution_result" + assert tool_result.content.stdout == "42\n" + assert tool_result.content.return_code == 0 + # The stored id a caller can download, not the sandbox-internal ``file_1``. + assert [ref.file_id for ref in tool_result.content.content] == ["file-stored-1"] + + +@pytest.mark.asyncio +async def test_a_program_that_failed_is_still_reported_natively(monkeypatch: pytest.MonkeyPatch) -> None: + """Unlike a failed search, a non-zero exit is a result the vocabulary carries.""" + responses = [ + _message_response(stop_reason="tool_use", content=[_code_use(code="1/0")]), + _message_response(stop_reason="end_turn", content=[_text_block("oops")]), + ] + monkeypatch.setattr(messages_loop_module, "amessages", _fake_amessages_for(responses)) + + result = await anthropic_tool_loop( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, _FakeSandboxPool(result=_exec_result(stdout="", stderr="ZeroDivisionError", return_code=1))), + max_iterations=5, + emit_native_code_execution=True, + ) + + tool_result = cast(Any, result.content[1]) + assert tool_result.type == "code_execution_tool_result" + assert tool_result.content.return_code == 1 + assert tool_result.content.stderr == "ZeroDivisionError" + + +@pytest.mark.asyncio +async def test_an_unreachable_sandbox_is_reported_as_the_native_error_shape(monkeypatch: pytest.MonkeyPatch) -> None: + responses = [ + _message_response(stop_reason="tool_use", content=[_code_use()]), + _message_response(stop_reason="end_turn", content=[_text_block("sorry")]), + ] + monkeypatch.setattr(messages_loop_module, "amessages", _fake_amessages_for(responses)) + + result = await anthropic_tool_loop( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, _FakeSandboxPool(fail=True)), + max_iterations=5, + emit_native_code_execution=True, + ) + + tool_result = cast(Any, result.content[1]) + assert tool_result.content.model_dump() == {"type": "code_execution_tool_result_error", "error_code": "unavailable"} + + +@pytest.mark.asyncio +async def test_no_native_code_execution_blocks_without_the_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + """A caller who said ``otari_code_execution`` keeps the plain result it always had.""" + responses = [ + _message_response(stop_reason="tool_use", content=[_code_use()]), + _message_response(stop_reason="end_turn", content=[_text_block("42")]), + ] + monkeypatch.setattr(messages_loop_module, "amessages", _fake_amessages_for(responses)) + + result = await anthropic_tool_loop( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, _FakeSandboxPool()), + max_iterations=5, + ) + + assert [b.type for b in result.content] == ["text"] + + +@pytest.mark.asyncio +async def test_stream_announces_the_execution_as_native_blocks(monkeypatch: pytest.MonkeyPatch) -> None: + streams = iter( + [ + _async_iter( + _msg_start_event(), + _tool_use_block_start(0, "tu_1", "code_execution"), + _input_json_delta(0, '{"code": "print(1)"}'), + _content_block_stop(0), + _msg_delta_event("tool_use"), + _msg_stop_event(), + ), + _async_iter( + _msg_start_event(), + _text_block_start(0), + _text_delta(0, "1"), + _content_block_stop(0), + _msg_delta_event("end_turn"), + _msg_stop_event(), + ), + ] + ) + + async def fake_amessages(**kwargs: Any) -> AsyncIterator[MessageStreamEvent]: + return next(streams) + + monkeypatch.setattr(messages_loop_module, "amessages", fake_amessages) + pool = _FakeSandboxPool() + events = [ + event + async for event in anthropic_tool_loop_stream( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, pool), + max_iterations=5, + emit_native_code_execution=True, + ) + ] + + assert pool.calls == [("code_execution", {"code": "print(1)"})] + starts = [event.content_block for event in events if event.type == "content_block_start"] + assert [block.type for block in starts] == ["server_tool_use", "code_execution_tool_result", "text"] + assert cast(Any, starts[0]).input == {"code": "print(1)"} + # Renumbered continuously: the client sees one message. + assert [event.index for event in events if event.type == "content_block_start"] == [0, 1, 2] + + +@pytest.mark.asyncio +async def test_native_result_announces_a_stored_file_the_block_did_not_name(monkeypatch: pytest.MonkeyPatch) -> None: + """The ``code_execution_output`` entries come from what was stored, not from the block's own list.""" + + class _DiffPool(_FakeSandboxPool): + async def call_tool(self, name: str, arguments: dict[str, Any]) -> str: + self.calls.append((name, arguments)) + block = _exec_result() + block.content.content = [] # the backend named nothing; the workspace diff found out.txt + self._executions.append( + CodeExecution(code=str(arguments.get("code") or ""), result=block, file_ids={"out.txt": "file-9"}) + ) + return "stdout:\n42" + + responses = [ + _message_response(stop_reason="tool_use", content=[_code_use()]), + _message_response(stop_reason="end_turn", content=[_text_block("done")]), + ] + monkeypatch.setattr(messages_loop_module, "amessages", _fake_amessages_for(responses)) + + result = await anthropic_tool_loop( + completion_kwargs={"model": "fake", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 100}, + pool=cast(Any, _DiffPool()), + max_iterations=5, + emit_native_code_execution=True, + ) + + tool_result = cast(Any, result.content[1]) + assert [o.file_id for o in tool_result.content.content] == ["file-9"] diff --git a/tests/unit/test_mcp_loop_responses.py b/tests/unit/test_mcp_loop_responses.py index cc7c8475f6..dcd5d6076c 100644 --- a/tests/unit/test_mcp_loop_responses.py +++ b/tests/unit/test_mcp_loop_responses.py @@ -29,15 +29,18 @@ from gateway.services import mcp_loop_responses as responses_loop_module from gateway.services.mcp_loop_responses import ( + CODE_INTERPRETER_CALL_ID_PREFIX, MaxToolIterationsExceeded, responses_tool_loop, responses_tool_loop_stream, ) +from gateway.services.sandbox_backend import CodeExecution from gateway.services.tool_format import ( inject_purpose_hints_responses, openai_to_responses_tools, ) from gateway.services.web_search_budget import WebSearchBudget +from gateway.types.code_execution import ResultBlock class _FakePool: @@ -1097,3 +1100,242 @@ async def fake_aresponses(**kwargs: Any) -> AsyncIterator[ResponseStreamEvent]: completed = next(e for e in events if e.type == "response.completed") assert [getattr(item, "call_id", None) for item in completed.response.output] == ["call_foreign"] assert pool.calls == [] + + +# --- native code_interpreter_call items ------------------------------------------------- + + +def _exec_result(stdout: str = "42\n", stderr: str = "", return_code: int = 0) -> ResultBlock: + return ResultBlock.model_validate( + { + "type": "code_execution_tool_result", + "content": { + "type": "code_execution_result", + "stdout": stdout, + "stderr": stderr, + "return_code": return_code, + "content": [], + }, + } + ) + + +class _FakeSandboxPool(_FakePool): + """A pool that owns ``code_execution`` and keeps executions like the real backend.""" + + container_id = "otari_cntr_test" + + def __init__(self, *, result: ResultBlock | None = _exec_result()) -> None: + super().__init__(tool_names=["code_execution"], results={"code_execution": "stdout:\n42"}) + self._result = result + self._executions: list[CodeExecution] = [] + + async def call_tool(self, name: str, arguments: dict[str, Any]) -> str: + self.calls.append((name, arguments)) + self._executions.append(CodeExecution(code=str(arguments.get("code") or ""), result=self._result)) + return self._results["code_execution"] + + def take_executions(self) -> list[CodeExecution]: + taken, self._executions = self._executions, [] + return taken + + +@pytest.mark.asyncio +async def test_a_gateway_execution_is_announced_as_a_code_interpreter_call(monkeypatch: pytest.MonkeyPatch) -> None: + responses = iter( + [ + _response(output=[_function_call("call_1", "code_execution", '{"code": "print(6 * 7)"}')]), + _response(output=[], status="completed"), + ] + ) + + async def fake_aresponses(**kwargs: Any) -> Response: + return next(responses) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + out = await responses_tool_loop( + completion_kwargs={"model": "fake", "input_data": [{"role": "user", "content": "compute"}]}, + pool=cast(Any, _FakeSandboxPool()), + max_iterations=5, + emit_native_code_execution=True, + ) + + items = [item for item in (out.output or []) if getattr(item, "type", None) == "code_interpreter_call"] + assert len(items) == 1 + item = cast(Any, items[0]) + assert item.id.startswith(CODE_INTERPRETER_CALL_ID_PREFIX) + assert item.code == "print(6 * 7)" + assert item.container_id == "otari_cntr_test" + assert item.status == "completed" + assert [(output.type, output.logs) for output in item.outputs] == [("logs", "42\n")] + + +@pytest.mark.asyncio +async def test_a_mixed_batch_still_announces_the_gateway_execution(monkeypatch: pytest.MonkeyPatch) -> None: + # The round exits for the caller to dispatch its own tool, but the code the + # gateway ran is announced alongside, as it is on the all-owned path. + async def fake_aresponses(**kwargs: Any) -> Response: + return _response( + output=[ + _function_call("call_1", "code_execution", '{"code": "print(1)"}'), + _function_call("foreign_id", "user_tool", "{}"), + ], + ) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + pool = _FakeSandboxPool() + out = await responses_tool_loop( + completion_kwargs={"model": "fake", "input_data": "go"}, + pool=cast(Any, pool), + max_iterations=5, + emit_native_code_execution=True, + ) + + assert pool.calls == [("code_execution", {"code": "print(1)"})] + types = [getattr(item, "type", None) for item in (out.output or [])] + assert types == ["code_interpreter_call", "function_call"] + assert cast(Any, out.output[1]).call_id == "foreign_id" + + +@pytest.mark.asyncio +async def test_a_streamed_mixed_batch_announces_the_execution_and_the_terminal_lists_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + owned = _function_call("call_1", "code_execution", '{"code": "print(1)"}') + foreign = _function_call("call_foreign", "user_tool", "{}") + iter_streams = iter( + [ + _async_iter( + _output_item_added(0, owned), + _function_call_args_done(0, "fc_owned", "code_execution", '{"code": "print(1)"}'), + _output_item_added(1, foreign), + _function_call_args_done(1, "fc_foreign", "user_tool", "{}"), + _response_completed(output=[owned, foreign]), + ), + ] + ) + + async def fake_aresponses(**kwargs: Any) -> AsyncIterator[ResponseStreamEvent]: + return next(iter_streams) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + pool = _FakeSandboxPool() + events = [ + event + async for event in responses_tool_loop_stream( + completion_kwargs={"model": "fake", "input_data": "go"}, + pool=cast(Any, pool), + max_iterations=5, + emit_native_code_execution=True, + ) + ] + + assert pool.calls == [("code_execution", {"code": "print(1)"})] + announced = [e for e in events if getattr(getattr(e, "item", None), "type", None) == "code_interpreter_call"] + assert len(announced) == 2 # added and done, before the terminal event + completed = next(e for e in events if e.type == "response.completed") + # ``get_final_response()`` agrees with the stream: the run it announced is + # in the terminal output, the gateway's consumed call is not. + assert [getattr(item, "type", None) for item in completed.response.output] == [ + "code_interpreter_call", + "function_call", + ] + assert events.index(announced[-1]) < events.index(completed) + + +@pytest.mark.asyncio +async def test_a_failed_program_is_a_failed_interpreter_call(monkeypatch: pytest.MonkeyPatch) -> None: + responses = iter( + [ + _response(output=[_function_call("call_1", "code_execution", '{"code": "1/0"}')]), + _response(output=[], status="completed"), + ] + ) + + async def fake_aresponses(**kwargs: Any) -> Response: + return next(responses) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + out = await responses_tool_loop( + completion_kwargs={"model": "fake", "input_data": [{"role": "user", "content": "compute"}]}, + pool=cast(Any, _FakeSandboxPool(result=_exec_result(stdout="", stderr="boom", return_code=1))), + max_iterations=5, + emit_native_code_execution=True, + ) + + item = cast(Any, next(i for i in (out.output or []) if getattr(i, "type", None) == "code_interpreter_call")) + assert item.status == "failed" + assert item.outputs[0].logs == "boom" + + +@pytest.mark.asyncio +async def test_no_interpreter_call_without_the_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: + responses = iter( + [ + _response(output=[_function_call("call_1", "code_execution", '{"code": "print(1)"}')]), + _response(output=[], status="completed"), + ] + ) + + async def fake_aresponses(**kwargs: Any) -> Response: + return next(responses) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + out = await responses_tool_loop( + completion_kwargs={"model": "fake", "input_data": [{"role": "user", "content": "compute"}]}, + pool=cast(Any, _FakeSandboxPool()), + max_iterations=5, + ) + + assert [getattr(item, "type", None) for item in (out.output or [])] == [] + + +@pytest.mark.asyncio +async def test_stream_announces_the_execution_as_a_code_interpreter_call(monkeypatch: pytest.MonkeyPatch) -> None: + fc = _function_call("call_1", "code_execution", "") + iter_streams = iter( + [ + _async_iter( + _output_item_added(0, fc), + _function_call_args_done(0, "fc_item_1", "code_execution", '{"code": "print(1)"}'), + _output_item_done(0, _function_call("call_1", "code_execution", '{"code": "print(1)"}')), + _response_completed(), + ), + _async_iter( + _text_delta("msg_1", 0, "1"), + _response_completed(), + ), + ] + ) + + async def fake_aresponses(**kwargs: Any) -> AsyncIterator[ResponseStreamEvent]: + return next(iter_streams) + + monkeypatch.setattr(responses_loop_module, "aresponses", fake_aresponses) + + pool = _FakeSandboxPool() + events = [ + event + async for event in responses_tool_loop_stream( + completion_kwargs={"model": "fake", "input_data": "go"}, + pool=cast(Any, pool), + max_iterations=5, + emit_native_code_execution=True, + ) + ] + + assert pool.calls == [("code_execution", {"code": "print(1)"})] + items = [ + getattr(e, "item") + for e in events + if getattr(getattr(e, "item", None), "type", None) == "code_interpreter_call" + ] + # One added and one done event, both carrying the complete item. + assert len(items) == 2 + assert items[0].code == "print(1)" + assert items[0].status == "completed" diff --git a/tests/unit/test_messages_client_betas.py b/tests/unit/test_messages_client_betas.py new file mode 100644 index 0000000000..4d47f37703 --- /dev/null +++ b/tests/unit/test_messages_client_betas.py @@ -0,0 +1,72 @@ +"""Unit tests for which betas the Messages route forwards. + +Two never reach a provider: the gateway's own MCP client capability, and every +beta at all when the dispatched provider has no Anthropic Messages API, where +any-llm refuses the whole request and a model swap would otherwise stop working. +""" + +from __future__ import annotations + +from gateway.api.routes.messages import _serves_messages_natively, _split_client_betas +from gateway.services.mcp_loop_messages import MCP_CLIENT_BETA + +ANTHROPIC = "anthropic:claude-sonnet-4-6" +OPEN_MODEL = "nebius:openai/gpt-oss-120b" + + +def test_a_native_provider_keeps_its_betas() -> None: + kwargs = {"model": ANTHROPIC, "betas": ["code-execution-2025-08-25"]} + + forwarded, saw_mcp = _split_client_betas(kwargs) + + assert forwarded["betas"] == ["code-execution-2025-08-25"] + assert saw_mcp is False + + +def test_a_provider_without_a_messages_api_has_its_betas_dropped() -> None: + kwargs = {"model": OPEN_MODEL, "betas": ["code-execution-2025-08-25", "files-api-2025-04-14"]} + + forwarded, saw_mcp = _split_client_betas(kwargs) + + assert "betas" not in forwarded + assert saw_mcp is False + # The caller's own dict is left alone. + assert kwargs["betas"] == ["code-execution-2025-08-25", "files-api-2025-04-14"] + + +def test_the_mcp_capability_is_consumed_and_the_rest_forwarded() -> None: + kwargs = {"model": ANTHROPIC, "betas": [MCP_CLIENT_BETA, "code-execution-2025-08-25"]} + + forwarded, saw_mcp = _split_client_betas(kwargs) + + assert forwarded["betas"] == ["code-execution-2025-08-25"] + assert saw_mcp is True + + +def test_the_mcp_capability_alone_leaves_no_betas_key() -> None: + forwarded, saw_mcp = _split_client_betas({"model": ANTHROPIC, "betas": [MCP_CLIENT_BETA]}) + + assert "betas" not in forwarded + assert saw_mcp is True + + +def test_the_mcp_capability_is_still_reported_for_an_open_model() -> None: + forwarded, saw_mcp = _split_client_betas({"model": OPEN_MODEL, "betas": [MCP_CLIENT_BETA]}) + + assert "betas" not in forwarded + assert saw_mcp is True + + +def test_a_request_with_no_betas_is_passed_through_untouched() -> None: + kwargs = {"model": OPEN_MODEL} + + forwarded, saw_mcp = _split_client_betas(kwargs) + + assert forwarded is kwargs + assert saw_mcp is False + + +def test_an_unknown_selector_is_not_stripped_on_a_guess() -> None: + assert _serves_messages_natively("not-a-provider:some-model") + assert _serves_messages_natively(None) + assert _serves_messages_natively("") diff --git a/tests/unit/test_messages_minted_block_stripping.py b/tests/unit/test_messages_minted_block_stripping.py index c68e18ce10..66b92d17ec 100644 --- a/tests/unit/test_messages_minted_block_stripping.py +++ b/tests/unit/test_messages_minted_block_stripping.py @@ -15,7 +15,11 @@ from gateway.api.routes._pipeline import ToolContext from gateway.api.routes.messages import _strip_gateway_minted_blocks from gateway.core.config import GatewayConfig -from gateway.services.mcp_loop_messages import MCP_ACTIVITY_ID_PREFIX, WEB_SEARCH_TOOL_USE_ID_PREFIX +from gateway.services.mcp_loop_messages import ( + MCP_ACTIVITY_ID_PREFIX, + SERVER_TOOL_USE_ID_PREFIX, + WEB_SEARCH_TOOL_USE_ID_PREFIX, +) def test_strips_the_minted_pair_but_keeps_the_text() -> None: @@ -392,3 +396,106 @@ def test_a_pre_prefix_gateway_pair_is_still_stripped() -> None: assert _strip_gateway_minted_blocks(messages) == [ {"role": "assistant", "content": [{"type": "text", "text": "done"}]} ] + + +# --- gateway-minted code execution is folded, not dropped -------------------------------- + + +def _code_pair(tool_use_id: str = f"{SERVER_TOOL_USE_ID_PREFIX}code") -> list[dict[str, Any]]: + return [ + {"type": "server_tool_use", "id": tool_use_id, "name": "code_execution", "input": {"code": "print(6 * 7)"}}, + { + "type": "code_execution_tool_result", + "tool_use_id": tool_use_id, + "content": { + "type": "code_execution_result", + "stdout": "42\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + }, + ] + + +def test_a_gateway_code_execution_pair_becomes_text_the_model_can_still_read() -> None: + """The output lives nowhere else in the transcript, so it is kept rather than stripped.""" + messages: list[dict[str, Any]] = [ + {"role": "assistant", "content": [*_code_pair(), {"type": "text", "text": "It is 42."}]} + ] + + kept = _strip_gateway_minted_blocks(messages)[0]["content"] + + assert [block["type"] for block in kept] == ["text", "text"] + assert "print(6 * 7)" in kept[0]["text"] + assert "stdout:\n42" in kept[0]["text"] + assert kept[1] == {"type": "text", "text": "It is 42."} + + +def test_a_failed_gateway_execution_folds_its_error() -> None: + pair = _code_pair() + pair[1]["content"] = {"type": "code_execution_tool_result_error", "error_code": "unavailable"} + kept = _strip_gateway_minted_blocks([{"role": "assistant", "content": pair}])[0]["content"] + + assert len(kept) == 1 + assert "error: unavailable" in kept[0]["text"] + + +def test_anthropics_own_code_execution_pair_survives() -> None: + """A ``srvtoolu_`` pair describes a run Anthropic did; it is echoed to Anthropic untouched.""" + pair = _code_pair(tool_use_id="srvtoolu_theirs") + messages: list[dict[str, Any]] = [{"role": "assistant", "content": pair}] + + assert _strip_gateway_minted_blocks(messages) == messages + + +# ---- the Responses counterpart ------------------------------------------- + + +def test_a_gateway_interpreter_call_becomes_a_message_the_model_can_still_read() -> None: + from gateway.api.routes.responses import _strip_gateway_minted_items + from gateway.services.mcp_loop_responses import CODE_INTERPRETER_CALL_ID_PREFIX + + items: list[dict[str, Any]] = [ + {"role": "user", "content": "compute"}, + { + "type": "code_interpreter_call", + "id": f"{CODE_INTERPRETER_CALL_ID_PREFIX}abc", + "code": "print(6 * 7)", + "container_id": "otari_cntr_1", + "outputs": [{"type": "logs", "logs": "42\n"}], + "status": "completed", + }, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "It is 42."}]}, + ] + + out = _strip_gateway_minted_items(items) + + assert len(out) == 3 + folded = out[1] + assert folded["type"] == "message" and folded["role"] == "assistant" + text = folded["content"][0]["text"] + assert "print(6 * 7)" in text + assert "42" in text + assert "status" not in text + + +def test_a_failed_gateway_interpreter_call_folds_its_status() -> None: + from gateway.api.routes.responses import _strip_gateway_minted_items + from gateway.services.mcp_loop_responses import CODE_INTERPRETER_CALL_ID_PREFIX + + item = { + "type": "code_interpreter_call", + "id": f"{CODE_INTERPRETER_CALL_ID_PREFIX}x", + "code": "1/0", + "status": "failed", + } + (folded,) = _strip_gateway_minted_items([item]) + assert "status: failed" in folded["content"][0]["text"] + + +def test_openais_own_interpreter_call_survives_untouched() -> None: + from gateway.api.routes.responses import _strip_gateway_minted_items + + item = {"type": "code_interpreter_call", "id": "ci_123", "code": "x", "status": "completed"} + assert _strip_gateway_minted_items([item, {"type": "web_search_call", "id": "ws_1"}]) == [item] diff --git a/tests/unit/test_pipeline_vision_billing.py b/tests/unit/test_pipeline_vision_billing.py index 25eb70d455..60054bce07 100644 --- a/tests/unit/test_pipeline_vision_billing.py +++ b/tests/unit/test_pipeline_vision_billing.py @@ -102,6 +102,7 @@ async def _normalize_with_vision( model: str, instance: str | None, workspace_id: uuid.UUID | None, + workspace_executor: object = None, ) -> tuple[int, CompletionUsage | None]: return 5000, _VISION_USAGE diff --git a/tests/unit/test_provider_produced_files.py b/tests/unit/test_provider_produced_files.py new file mode 100644 index 0000000000..b00010242d --- /dev/null +++ b/tests/unit/test_provider_produced_files.py @@ -0,0 +1,167 @@ +"""Unit tests for files a provider's own sandbox produced. + +Covers the pure parts: reading the produced ids out of each vocabulary's +response, which provider Otari can fetch back from, and the request it makes. +The download itself is exercised over HTTP in +tests/integration/test_files_endpoint.py. +""" + +from __future__ import annotations + +import uuid +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +from gateway.models.tools import FileObject +from gateway.services.files.provider_files import ( + ANTHROPIC_FILES_BETA, + ProviderFile, + _request_for, + anthropic_produced_files, + produced_files_for, + responses_produced_files, + serves_files, +) + + +def _anthropic_reply(*outputs: list[dict[str, str]]) -> SimpleNamespace: + """A Messages response whose tool-result blocks carry ``outputs``.""" + return SimpleNamespace( + content=[ + SimpleNamespace( + type="code_execution_tool_result", + content=SimpleNamespace(content=[SimpleNamespace(**output) for output in group]), + ) + for group in outputs + ] + ) + + +def test_anthropic_outputs_are_read_whichever_variant_ran() -> None: + reply = _anthropic_reply( + [{"type": "code_execution_output", "file_id": "file_01python"}], + [{"type": "bash_code_execution_output", "file_id": "file_01bash"}], + ) + + assert [file.file_id for file in anthropic_produced_files(reply)] == ["file_01python", "file_01bash"] + + +def test_anthropic_reply_with_no_files_produces_nothing() -> None: + assert anthropic_produced_files(_anthropic_reply([])) == [] + assert anthropic_produced_files(SimpleNamespace(content=[SimpleNamespace(type="text", text="hi")])) == [] + + +def test_an_id_cited_twice_is_recorded_once() -> None: + reply = _anthropic_reply( + [{"type": "code_execution_output", "file_id": "file_01same"}], + [{"type": "code_execution_output", "file_id": "file_01same"}], + ) + + assert anthropic_produced_files(reply) == [ProviderFile(file_id="file_01same")] + + +def test_responses_citations_carry_the_container_and_the_name() -> None: + reply = SimpleNamespace( + output=[ + SimpleNamespace( + type="message", + content=[ + SimpleNamespace( + type="output_text", + annotations=[ + SimpleNamespace( + type="container_file_citation", + container_id="cntr_1", + file_id="cfile_1", + filename="bar_plot.png", + ), + SimpleNamespace(type="url_citation", url="https://example.com"), + ], + ) + ], + ) + ] + ) + + assert responses_produced_files(reply) == [ + ProviderFile(file_id="cfile_1", filename="bar_plot.png", container_id="cntr_1") + ] + + +def test_only_providers_otari_can_read_back_are_recorded() -> None: + assert serves_files("anthropic") + assert serves_files("openai") + assert not serves_files("nebius") + + +def test_anthropic_download_takes_the_id_alone() -> None: + record = SimpleNamespace( + id="file_01abc", + provider="anthropic", + provider_container_id=None, + workspace_id=uuid.uuid4(), + ) + + url, headers = _request_for(cast(FileObject, cast(Any, record)), "sk-ant-test", None) + + assert url == "https://api.anthropic.com/v1/files/file_01abc/content" + assert headers["x-api-key"] == "sk-ant-test" + assert headers["anthropic-beta"] == ANTHROPIC_FILES_BETA + + +def test_openai_download_is_keyed_on_the_container() -> None: + record = SimpleNamespace( + id="cfile_1", + provider="openai", + provider_container_id="cntr_1", + workspace_id=uuid.uuid4(), + ) + + url, headers = _request_for(cast(FileObject, cast(Any, record)), "sk-test", None) + + assert url == "https://api.openai.com/v1/containers/cntr_1/files/cfile_1/content" + assert headers == {"Authorization": "Bearer sk-test"} + + +def test_an_openai_file_with_no_container_cannot_be_read() -> None: + # The download is keyed on the container, so a row without one has no URL + # to build; refusing here is what keeps ``containers/None/...`` off the wire. + record = SimpleNamespace(id="cfile_1", provider="openai", provider_container_id=None, workspace_id=None) + + with pytest.raises(LookupError): + _request_for(cast(FileObject, cast(Any, record)), "sk-test", None) + + +def test_a_streamed_messages_result_block_names_its_files() -> None: + reply = _anthropic_reply([{"file_id": "file_01abc"}]) + event = SimpleNamespace(type="content_block_start", index=0, content_block=reply.content[0]) + + assert produced_files_for("messages", event) == [ProviderFile(file_id="file_01abc")] + # A delta carries no block, and a completed reply is read whole. + assert produced_files_for("messages", SimpleNamespace(type="content_block_delta", delta=None)) == [] + assert produced_files_for("messages", reply) == [ProviderFile(file_id="file_01abc")] + + +def test_a_streamed_responses_completion_names_its_files() -> None: + citation = SimpleNamespace( + type="container_file_citation", file_id="cfile_1", filename="bar_plot.png", container_id="cntr_1" + ) + response = SimpleNamespace(output=[SimpleNamespace(content=[SimpleNamespace(annotations=[citation])])]) + event = SimpleNamespace(type="response.completed", response=response) + + expected = [ProviderFile(file_id="cfile_1", filename="bar_plot.png", container_id="cntr_1")] + assert produced_files_for("responses", event) == expected + assert produced_files_for("responses", response) == expected + assert produced_files_for("responses", SimpleNamespace(type="response.output_text.delta")) == [] + # Chat Completions has no native code tool, so nothing is ever read off it. + assert produced_files_for("chat", response) == [] + + +def test_a_configured_api_base_is_where_the_download_goes() -> None: + record = SimpleNamespace(id="file_01abc", provider="anthropic", provider_container_id=None, workspace_id=None) + + url, _ = _request_for(cast(FileObject, cast(Any, record)), "sk-ant-test", "https://anthropic.internal/v1/") + + assert url == "https://anthropic.internal/v1/files/file_01abc/content" diff --git a/tests/unit/test_responses_produced_images.py b/tests/unit/test_responses_produced_images.py new file mode 100644 index 0000000000..2be5c36e06 --- /dev/null +++ b/tests/unit/test_responses_produced_images.py @@ -0,0 +1,84 @@ +"""Unit tests for how a gateway-run execution announces produced files on Responses. + +OpenAI's ``code_interpreter_call`` has no shape for a file id, only logs and an +``image`` URL, so a produced image is announced as the address Otari serves it +from and anything else is left to the files API. +""" + +from __future__ import annotations + +from gateway.services.mcp_loop_responses import _code_interpreter_call_item +from gateway.services.sandbox_backend import CodeExecution +from gateway.types.code_execution import CodeExecutionResult, ResultBlock + +BASE = "https://otari.example.com/api/v1/files" + + +def _execution(**file_ids: str) -> CodeExecution: + return CodeExecution( + code="print('hi')", + result=ResultBlock( + type="code_execution_result", + content=CodeExecutionResult(stdout="hi\n", stderr="", return_code=0), + ), + file_ids=dict(file_ids), + ) + + +def test_a_produced_image_is_announced_as_the_url_otari_serves_it_from() -> None: + item = _code_interpreter_call_item(_execution(**{"bar_plot.png": "file-abc"}), "otari_cntr_1", BASE) + + assert item.outputs is not None + kinds = [output.type for output in item.outputs] + assert kinds == ["logs", "image"] + image = item.outputs[1] + assert image.type == "image" + assert image.url == f"{BASE}/file-abc/content" + assert item.status == "completed" + + +def test_a_non_image_is_left_to_the_files_api() -> None: + item = _code_interpreter_call_item(_execution(**{"table.csv": "file-abc"}), "otari_cntr_1", BASE) + + assert item.outputs is not None + assert [output.type for output in item.outputs] == ["logs"] + + +def test_a_run_outside_a_request_announces_no_url() -> None: + """No base URL is the direct-use case (tests, a backend built by hand).""" + item = _code_interpreter_call_item(_execution(**{"bar_plot.png": "file-abc"}), "otari_cntr_1", None) + + assert item.outputs is not None + assert [output.type for output in item.outputs] == ["logs"] + + +def test_a_deployment_with_no_public_address_announces_a_root_relative_url() -> None: + item = _code_interpreter_call_item(_execution(**{"bar_plot.png": "file-abc"}), "otari_cntr_1", "/api/v1/files") + + assert item.outputs is not None + image = item.outputs[1] + assert image.type == "image" + assert image.url == "/api/v1/files/file-abc/content" + + +def test_an_image_is_announced_even_when_the_run_logged_nothing() -> None: + execution = CodeExecution( + code="savefig()", + result=ResultBlock( + type="code_execution_result", + content=CodeExecutionResult(stdout="", stderr="", return_code=0), + ), + file_ids={"bar_plot.png": "file-abc"}, + ) + + item = _code_interpreter_call_item(execution, "otari_cntr_1", BASE) + + assert item.outputs is not None + assert [output.type for output in item.outputs] == ["image"] + + +def test_a_run_the_backend_never_answered_announces_nothing() -> None: + item = _code_interpreter_call_item(CodeExecution(code="print(1)", result=None), "otari_cntr_1", BASE) + + assert item.outputs is None + assert item.status == "failed" diff --git a/tests/unit/test_sandbox_backend.py b/tests/unit/test_sandbox_backend.py index 50ece46a18..bf1c4e2688 100644 --- a/tests/unit/test_sandbox_backend.py +++ b/tests/unit/test_sandbox_backend.py @@ -927,3 +927,503 @@ async def test_exec_503_preserves_retry_hint(monkeypatch: pytest.MonkeyPatch) -> with pytest.raises(SandboxUnavailableError) as caught: await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "print(42)"}) assert caught.value.retry_after == "15" + + +class _FakeFiles: + """A stand-in for ``SandboxFileBridge``: inputs to seed, outputs it was handed.""" + + def __init__(self, inputs: list[Any], *, max_output_bytes: int = 1 << 20, max_output_files: int = 20) -> None: + self.inputs = inputs + self.max_output_bytes = max_output_bytes + self.max_output_files = max_output_files + self.stored: list[tuple[str, bytes]] = [] + # Streams the backend started and abandoned, as a store would see them. + self.abandoned: list[str] = [] + + async def read_input(self, staged: Any) -> bytes: + return b"a,b\n1,2\n" + + async def store_output(self, filename: str, chunks: Any) -> str | None: + data = bytearray() + try: + async for chunk in chunks: + data.extend(chunk) + except BaseException: + self.abandoned.append(filename) + raise + if not data: + return None + self.stored.append((filename, bytes(data))) + return f"file-{len(self.stored)}" + + +def _staged(file_id: str = "file-csv", filename: str = "data.csv") -> Any: + from gateway.services.file_service import StagedFile + + return StagedFile(file_id, filename, "text/csv", f"x/{file_id}") + + +@pytest.mark.asyncio +async def test_staged_inputs_are_seeded_before_the_first_call(monkeypatch: pytest.MonkeyPatch) -> None: + transport = _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/files"): httpx.Response(201, json={"path": "data.csv", "size": 8}), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([_staged()]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files): + pass + + put = next(r for r in transport.captured if r.method == "POST" and r.url.path == "/sessions/s1/files") + body = put.read() + assert b'filename="data.csv"' in body + assert b"a,b\n1,2\n" in body + assert b'name="path"' in body + + +@pytest.mark.asyncio +async def test_refused_seed_is_terminal_and_releases_the_session(monkeypatch: pytest.MonkeyPatch) -> None: + transport = _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/files"): httpx.Response(413, json={"error": "too large"}), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + with pytest.raises(SandboxNotReachableError, match="file-csv"): + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=_FakeFiles([_staged()])): + pass + assert ("DELETE", "/sessions/s1") in [(r.method, r.url.path) for r in transport.captured] + + +@pytest.mark.asyncio +async def test_produced_files_are_fetched_stored_and_named_with_file_ids(monkeypatch: pytest.MonkeyPatch) -> None: + result_block = { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": { + "type": "code_execution_result", + "stdout": "saved\n", + "stderr": "", + "return_code": 0, + "content": [{"type": "code_execution_output", "file_id": "sbx-1", "filename": "chart.png"}], + }, + } + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": result_block}), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"\x89PNG"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "plt.savefig('chart.png')"}) + + assert files.stored == [("chart.png", b"\x89PNG")] + assert "chart.png (file_id: file-1)" in result + + +@pytest.mark.asyncio +async def test_unfetchable_output_is_still_named_and_does_not_fail_the_run(monkeypatch: pytest.MonkeyPatch) -> None: + result_block = { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": { + "type": "code_execution_result", + "stdout": "ok\n", + "stderr": "", + "return_code": 0, + "content": [{"type": "code_execution_output", "file_id": "sbx-1", "filename": "out.csv"}], + }, + } + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": result_block}), + ("GET", "/sessions/s1/files"): httpx.Response(404, json={"error": "gone"}), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + assert files.stored == [] + assert "files: out.csv" in result + assert "file_id" not in result + + +@pytest.mark.asyncio +async def test_no_bridge_leaves_outputs_untouched(monkeypatch: pytest.MonkeyPatch) -> None: + result_block = { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": { + "type": "code_execution_result", + "stdout": "", + "stderr": "", + "return_code": 0, + "content": [{"type": "code_execution_output", "file_id": "sbx-1", "filename": "a.txt"}], + }, + } + transport = _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": result_block}), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + async with SandboxBackend(sandbox_url="http://sandbox:8080") as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + assert result == "files: a.txt" + assert all(r.url.path != "/sessions/s1/files" for r in transport.captured) + + +@pytest.mark.asyncio +async def test_executions_are_kept_in_order_and_drained_by_take(monkeypatch: pytest.MonkeyPatch) -> None: + """What a loop minting native result blocks reads: the code and the structured result.""" + result_block = { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": {"type": "code_execution_result", "stdout": "1\n", "stderr": "", "return_code": 0, "content": []}, + } + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": result_block}), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + + async with SandboxBackend(sandbox_url="http://sandbox:8080") as backend: + assert backend.container_id.startswith("otari_cntr_") + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "print(1)"}) + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "print(2)"}) + executions = backend.take_executions() + assert [execution.code for execution in executions] == ["print(1)", "print(2)"] + assert executions[0].result is not None + assert executions[0].result.content.stdout == "1\n" + # Drained: a later round cannot claim an earlier round's executions. + assert backend.take_executions() == [] + + +@pytest.mark.asyncio +async def test_an_exec_that_never_answered_is_kept_without_a_result(monkeypatch: pytest.MonkeyPatch) -> None: + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(500), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + + async with SandboxBackend(sandbox_url="http://sandbox:8080") as backend: + with pytest.raises(SandboxNotReachableError): + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "print(1)"}) + executions = backend.take_executions() + + assert len(executions) == 1 + assert executions[0].code == "print(1)" + assert executions[0].result is None + + +@pytest.mark.asyncio +async def test_an_execution_carries_the_stored_ids_of_the_files_it_produced(monkeypatch: pytest.MonkeyPatch) -> None: + """What a native block announces: the ``/v1/files`` id, never the sandbox's own.""" + result_block = { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": { + "type": "code_execution_result", + "stdout": "", + "stderr": "", + "return_code": 0, + "content": [{"type": "code_execution_output", "file_id": "sbx-1", "filename": "chart.png"}], + }, + } + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": result_block}), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"\x89PNG"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=_FakeFiles([])) as backend: + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "plt.savefig('chart.png')"}) + execution = backend.take_executions()[0] + + assert execution.file_ids == {"chart.png": "file-1"} + + +def _result_block_naming(filename: str) -> dict[str, Any]: + return { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": { + "type": "code_execution_result", + "stdout": "", + "stderr": "", + "return_code": 0, + "content": [{"type": "code_execution_output", "file_id": "sbx-1", "filename": filename}], + }, + } + + +@pytest.mark.asyncio +async def test_an_output_declared_over_the_cap_is_refused_before_it_is_read(monkeypatch: pytest.MonkeyPatch) -> None: + class _CountingStream(httpx.AsyncByteStream): + reads = 0 + + async def __aiter__(self) -> Any: + _CountingStream.reads += 1 + yield b"x" * 64 + + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _result_block_naming("big.bin")}), + ("GET", "/sessions/s1/files"): httpx.Response( + 200, headers={"content-length": "64"}, stream=_CountingStream() + ), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([], max_output_bytes=16) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + assert files.stored == [] + assert _CountingStream.reads == 0 + assert "files: big.bin" in result + + +@pytest.mark.asyncio +async def test_an_output_that_grows_past_the_cap_is_abandoned_mid_stream(monkeypatch: pytest.MonkeyPatch) -> None: + class _EndlessStream(httpx.AsyncByteStream): + chunks = 0 + + async def __aiter__(self) -> Any: + while True: + _EndlessStream.chunks += 1 + yield b"x" * 8 + + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _result_block_naming("big.bin")}), + # No Content-Length: the cap has to hold on the bytes as they arrive. + ("GET", "/sessions/s1/files"): httpx.Response(200, stream=_EndlessStream()), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([], max_output_bytes=32) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + assert files.stored == [] + # Read just past the cap and no further: 32 bytes is four chunks, the fifth trips it. + assert _EndlessStream.chunks == 5 + assert "file_id" not in result + + +def _empty_result_block(stdout: str = "saved\n") -> dict[str, Any]: + """A result block that names no files, as the reference container returns.""" + return { + "type": "code_execution_tool_result", + "tool_use_id": "t1", + "content": {"type": "code_execution_result", "stdout": stdout, "stderr": "", "return_code": 0, "content": []}, + } + + +def _listing(*entries: tuple[str, int, float]) -> dict[str, Any]: + return {"files": [{"path": p, "size_bytes": s, "mime_type": None, "modified_at": m} for p, s, m in entries]} + + +_Handlers = dict[tuple[str, str], httpx.Response | list[httpx.Response]] + + +class _SequenceTransport(httpx.AsyncBaseTransport): + """Like ``_MockTransport``, but a handler may be a list answered in order.""" + + def __init__(self, handlers: _Handlers) -> None: + self._handlers = handlers + self.captured: list[httpx.Request] = [] + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.captured.append(request) + handler = self._handlers.get((request.method, request.url.path)) + if handler is None: + return httpx.Response(404, json={"error": "no handler"}) + if isinstance(handler, list): + return handler.pop(0) if len(handler) > 1 else handler[0] + return handler + + +def _patched_sequence_client(handlers: _Handlers, monkeypatch: pytest.MonkeyPatch) -> _SequenceTransport: + transport = _SequenceTransport(handlers) + original_init = httpx.AsyncClient.__init__ + + def patched_init(self: httpx.AsyncClient, *args: Any, **kwargs: Any) -> None: + kwargs["transport"] = transport + original_init(self, *args, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "__init__", patched_init) + return transport + + +@pytest.mark.asyncio +async def test_a_file_the_block_does_not_name_is_found_by_the_workspace_diff(monkeypatch: pytest.MonkeyPatch) -> None: + transport = _patched_sequence_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("POST", "/sessions/s1/files"): httpx.Response(201, json={"path": "data.csv", "size": 8}), + # Listed once after seeding (the input only), once after the call (the output too). + ("GET", "/sessions/s1/files/list"): [ + httpx.Response(200, json=_listing(("data.csv", 8, 1.0))), + httpx.Response(200, json=_listing(("data.csv", 8, 1.0), ("out.txt", 5, 2.0))), + ], + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _empty_result_block()}), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"hello"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([_staged()]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "open('out.txt','w').write('hello')"}) + execution = backend.take_executions()[0] + + # Only the new file was fetched: the unchanged seeded input was not re-read. + fetched = [ + r.url.params.get("path") for r in transport.captured if r.method == "GET" and r.url.path.endswith("/files") + ] + assert fetched == ["out.txt"] + assert files.stored == [("out.txt", b"hello")] + assert "files: out.txt (file_id: file-1)" in result + assert execution.file_ids == {"out.txt": "file-1"} + + +@pytest.mark.asyncio +async def test_the_diff_moves_forward_so_a_later_call_collects_only_its_own_files( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patched_sequence_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("GET", "/sessions/s1/files/list"): [ + httpx.Response(200, json=_listing()), + httpx.Response(200, json=_listing(("a.txt", 1, 1.0))), + # a.txt rewritten (new stamp) and b.txt new: both are this call's. + httpx.Response(200, json=_listing(("a.txt", 2, 3.0), ("b.txt", 1, 3.0))), + ], + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _empty_result_block()}), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"x"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "one"}) + await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "two"}) + first, second = backend.take_executions() + + assert list(first.file_ids) == ["a.txt"] + assert sorted(second.file_ids) == ["a.txt", "b.txt"] + + +@pytest.mark.asyncio +async def test_a_backend_without_list_files_still_collects_what_the_block_names( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patched_async_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + # No /files/list handler: the mock answers 404, as a backend without the operation would. + ("POST", "/sessions/s1/exec"): httpx.Response( + 200, json={"result_block": _result_block_naming("chart.png")} + ), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"\x89PNG"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([]) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + assert files.stored == [("chart.png", b"\x89PNG")] + assert "chart.png (file_id: file-1)" in result + + +@pytest.mark.asyncio +async def test_a_call_stores_at_most_the_configured_number_of_files(monkeypatch: pytest.MonkeyPatch) -> None: + _patched_sequence_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("GET", "/sessions/s1/files/list"): [ + httpx.Response(200, json=_listing()), + httpx.Response(200, json=_listing(("a.txt", 1, 1.0), ("b.txt", 1, 1.0), ("c.txt", 1, 1.0))), + ], + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _empty_result_block()}), + ("GET", "/sessions/s1/files"): httpx.Response(200, content=b"x"), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([], max_output_files=2) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + assert [name for name, _ in files.stored] == ["a.txt", "b.txt"] + # The third is still named, so the model and the caller know it exists. + assert "c.txt" in result + assert "c.txt (file_id" not in result + + +@pytest.mark.asyncio +async def test_a_call_stores_at_most_the_configured_bytes_across_its_files(monkeypatch: pytest.MonkeyPatch) -> None: + class _TwentyBytes(httpx.AsyncByteStream): + async def __aiter__(self) -> Any: + yield b"x" * 10 + yield b"y" * 10 + + _patched_sequence_client( + { + ("POST", "/sessions"): httpx.Response(200, json={"session_id": "s1"}), + ("GET", "/sessions/s1/files/list"): [ + httpx.Response(200, json=_listing()), + httpx.Response(200, json=_listing(("a.bin", 20, 1.0), ("b.bin", 20, 1.0))), + ], + ("POST", "/sessions/s1/exec"): httpx.Response(200, json={"result_block": _empty_result_block()}), + # No Content-Length: the budget has to hold on the bytes as they arrive. + ("GET", "/sessions/s1/files"): httpx.Response(200, stream=_TwentyBytes()), + ("DELETE", "/sessions/s1"): httpx.Response(204), + }, + monkeypatch, + ) + files = _FakeFiles([], max_output_bytes=30) + async with SandboxBackend(sandbox_url="http://sandbox:8080", files=files) as backend: + result = await backend.call_tool(CODE_EXECUTION_TOOL_NAME, {"code": "x"}) + + # The first file fits (20 of 30); the second runs past what is left and is + # abandoned mid-stream, which the store sees as a failed stream to clean up. + assert files.stored == [("a.bin", b"x" * 10 + b"y" * 10)] + assert files.abandoned == ["b.bin"] + assert "a.bin (file_id: file-1)" in result + assert "b.bin (file_id" not in result diff --git a/tests/unit/test_sandbox_file_bridge.py b/tests/unit/test_sandbox_file_bridge.py new file mode 100644 index 0000000000..67e37f6186 --- /dev/null +++ b/tests/unit/test_sandbox_file_bridge.py @@ -0,0 +1,143 @@ +"""The bridge between the ``/v1/files`` store and a sandbox session. + +Covers what the sandbox backend's own tests stub out: that a produced file is +streamed into the store, that an empty one leaves nothing behind, and that a +row which fails to land takes its blob with it. +""" + +from __future__ import annotations + +import uuid +from collections.abc import AsyncIterator +from typing import Any, cast + +import pytest + +from gateway.core.config import GatewayConfig +from gateway.core.unit_of_work import UnitOfWork +from gateway.services import file_service +from gateway.services.files import SandboxFileBridge + + +class _MemoryStore: + def __init__(self) -> None: + self.blobs: dict[str, bytes] = {} + + async def put(self, file_id: str, data: bytes) -> str: + self.blobs[file_id] = data + return file_id + + async def get(self, storage_ref: str) -> bytes: + return self.blobs[storage_ref] + + async def put_stream(self, file_id: str, chunks: AsyncIterator[bytes]) -> tuple[str, int]: + data = bytearray() + async for chunk in chunks: + data.extend(chunk) + self.blobs[file_id] = bytes(data) + return file_id, len(data) + + async def get_stream(self, storage_ref: str) -> Any: + yield self.blobs[storage_ref] + + async def delete(self, storage_ref: str) -> None: + self.blobs.pop(storage_ref, None) + + +class _FakeDb: + def __init__(self) -> None: + self.added: list[Any] = [] + + def add(self, record: Any) -> None: + self.added.append(record) + + async def flush(self) -> None: + return None + + +class _FakeUnitOfWork: + """Enough of a Unit of Work for ``session_for``: the session and an open block.""" + + def __init__(self, db: Any) -> None: + self._session = db + self._depth = 0 + + async def __aenter__(self) -> _FakeUnitOfWork: + self._depth += 1 + return self + + async def __aexit__(self, *exc: object) -> None: + self._depth -= 1 + + +class _FailingUnitOfWork(_FakeUnitOfWork): + """A Unit of Work whose commit fails the way a connect timeout does: a bare ``TimeoutError``.""" + + async def __aexit__(self, *exc: object) -> None: + raise TimeoutError("connect timed out") + + +class _CommittingUnitOfWork(_FakeUnitOfWork): + pass + + +async def _chunks(*parts: bytes) -> AsyncIterator[bytes]: + for part in parts: + yield part + + +def _bridge(store: _MemoryStore, uow: Any = None, **config: Any) -> SandboxFileBridge: + return SandboxFileBridge( + file_store=store, + config=GatewayConfig(**config), + uow=cast(UnitOfWork, uow if uow is not None else _CommittingUnitOfWork(_FakeDb())), + user_id="u1", + workspace_id=uuid.uuid4(), + inputs=[], + ) + + +@pytest.mark.asyncio +async def test_store_output_streams_the_file_in_and_writes_its_row() -> None: + store = _MemoryStore() + db = _FakeDb() + + file_id = await _bridge(store, _CommittingUnitOfWork(db)).store_output("chart.png", _chunks(b"\x89PNG", b"...")) + + assert file_id is not None and file_id.startswith("file-") + assert store.blobs == {file_id: b"\x89PNG..."} + (record,) = db.added + assert (record.id, record.filename, record.bytes, record.purpose) == ( + file_id, + "chart.png", + 7, + file_service.CODE_EXECUTION_OUTPUT_PURPOSE, + ) + + +@pytest.mark.asyncio +async def test_an_empty_output_leaves_no_blob_and_no_row() -> None: + store = _MemoryStore() + db = _FakeDb() + + assert await _bridge(store, _CommittingUnitOfWork(db)).store_output("empty.txt", _chunks()) is None + assert store.blobs == {} + assert db.added == [] + + +@pytest.mark.asyncio +async def test_a_row_that_fails_to_land_takes_its_blob_with_it() -> None: + store = _MemoryStore() + + with pytest.raises(TimeoutError): + await _bridge(store, _FailingUnitOfWork(_FakeDb())).store_output("out.csv", _chunks(b"a,b\n")) + # Nothing references the bytes any more, and the sweep only sees rows, so + # leaving them would be a leak nothing reclaims. + assert store.blobs == {} + + +def test_the_output_budget_never_exceeds_the_upload_cap() -> None: + store = _MemoryStore() + assert _bridge(store, files_output_max_bytes=1 << 30, files_max_bytes=1 << 20).max_output_bytes == 1 << 20 + assert _bridge(store, files_output_max_bytes=1 << 10, files_max_bytes=1 << 20).max_output_bytes == 1 << 10 + assert _bridge(store, files_output_max_files=3).max_output_files == 3 diff --git a/tests/unit/test_setting_names.py b/tests/unit/test_setting_names.py index 45d698412b..e94791249e 100644 --- a/tests/unit/test_setting_names.py +++ b/tests/unit/test_setting_names.py @@ -27,6 +27,7 @@ "budget_reservation_ttl_sec", "budget_strategy", "capture_agent_telemetry", + "code_execution_executor", "cors_allow_origins", "dashboard_login_rate_limit_per_minute", "dashboard_session_ttl_hours", @@ -50,10 +51,15 @@ "files_enabled", "files_local_dir", "files_max_bytes", + "files_output_max_bytes", + "files_output_max_files", "files_retention_hours", "files_s3_bucket", "files_s3_endpoint_url", "files_s3_region", + "files_storage_options", + "files_sweep_interval_sec", + "files_url", "guardrails_url", "host", "invitation_expiry_hours", diff --git a/tests/unit/test_tool_settings_endpoint.py b/tests/unit/test_tool_settings_endpoint.py index 30da7428c0..9245a39204 100644 --- a/tests/unit/test_tool_settings_endpoint.py +++ b/tests/unit/test_tool_settings_endpoint.py @@ -340,3 +340,22 @@ def test_guardrail_catalog_is_an_operator_read(tmp_path: Path) -> None: assert catalog in operator assert catalog not in reader assert profiles in reader + + +def test_the_executor_is_an_operator_setting_with_a_closed_vocabulary(tmp_path: Path) -> None: + with _client(tmp_path) as client: + before = _fields(client.get(f"{API_ROOT}/tool-settings", headers=AUTH).json()) + assert before["code_execution_executor"]["service"] == "sandbox" + assert before["code_execution_executor"]["choices"] == ["auto", "otari", "provider"] + assert before["code_execution_executor"]["value"] is None + assert before["sandbox_url"]["choices"] is None + + patched = client.patch(f"{API_ROOT}/tool-settings", json={"code_execution_executor": "Otari"}, headers=AUTH) + assert patched.status_code == 200, patched.text + refused = client.patch( + f"{API_ROOT}/tool-settings", json={"code_execution_executor": "anthropic"}, headers=AUTH + ) + assert refused.status_code == 422 + after = _fields(client.get(f"{API_ROOT}/tool-settings", headers=AUTH).json()) + + assert after["code_execution_executor"]["value"] == "otari" diff --git a/tests/unit/test_tools_endpoint.py b/tests/unit/test_tools_endpoint.py index f329d1c112..5de4187686 100644 --- a/tests/unit/test_tools_endpoint.py +++ b/tests/unit/test_tools_endpoint.py @@ -152,3 +152,13 @@ def test_not_registered_in_hybrid_mode(monkeypatch: pytest.MonkeyPatch) -> None: response = client.get(f"{API_ROOT}/tools", headers={"Authorization": "Bearer platform-user-token"}) assert response.status_code == 404, response.text + + +def test_a_configured_sandbox_advertises_the_provider_keywords_it_may_claim(tmp_path: Path) -> None: + with _client(tmp_path, sandbox_url="http://sandbox:8080") as client: + auto = _tools(client)["otari_code_execution"]["accepted_types"] + with _client(tmp_path, sandbox_url="http://sandbox:8080", code_execution_executor="provider") as client: + provider_only = _tools(client)["otari_code_execution"]["accepted_types"] + + assert auto == ["otari_code_execution", "code_execution", "code_interpreter", "code_execution_"] + assert provider_only == ["otari_code_execution"] diff --git a/uv.lock b/uv.lock index a7bfbd98a9..241d92f693 100644 --- a/uv.lock +++ b/uv.lock @@ -1063,6 +1063,9 @@ dependencies = [ ] [package.optional-dependencies] +fsspec = [ + { name = "fsspec" }, +] ocr = [ { name = "numpy" }, { name = "rapidocr-onnxruntime" }, @@ -1103,6 +1106,7 @@ requires-dist = [ { name = "cryptography", specifier = ">=50.0.0" }, { name = "dnspython", specifier = ">=2.7.0" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "fsspec", marker = "extra == 'fsspec'", specifier = ">=2024.6.0" }, { name = "genai-prices", specifier = ">=0.1.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "idna", specifier = ">=3.10" }, @@ -1126,7 +1130,7 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, { name = "webauthn", specifier = ">=2.5.0" }, ] -provides-extras = ["ocr", "s3"] +provides-extras = ["fsspec", "ocr", "s3"] [package.metadata.requires-dev] dev = [ diff --git a/web/src/client/schema.ts b/web/src/client/schema.ts index 34b540b373..150715313b 100644 --- a/web/src/client/schema.ts +++ b/web/src/client/schema.ts @@ -1101,12 +1101,17 @@ export interface paths { * * ``workspace_id`` narrows a master-key listing to one workspace; a keyed * request is already confined to its key's own and cannot widen or move it. + * + * Pages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names + * the last file of the previous page, and ``has_more`` says whether to ask + * again. A cursor that has since been deleted or has expired is still a + * position; one the caller never owned is a 404. */ get: operations["files-list_files"]; put?: never; /** * Create File - * @description OpenAI-compatible file upload endpoint. + * @description Upload a file. Answers in the OpenAI or Anthropic file shape, following the caller's headers. */ post: operations["files-create_file"]; delete?: never; @@ -6649,6 +6654,16 @@ export interface components { */ outcome: "pass" | "fail" | "error"; }; + /** + * CodeExecutor + * @description Who runs the code a request's code-execution tool asks for. + * + * The one vocabulary shared by the deployment setting, the workspace policy, + * the per-request header and the platform's resolve payload, so a value read + * from any of them means the same thing at admission. + * @enum {string} + */ + CodeExecutor: "auto" | "otari" | "provider"; /** * ConfigField * @description One effective config value surfaced to the dashboard's config viewer. @@ -8220,7 +8235,7 @@ export interface components { ManagedTool: { /** * Accepted Types - * @description Every `tools[].type` this deployment currently routes to the tool. Always includes the canonical `otari_*` type; for web search it also includes the provider-named keywords when interception is enabled. + * @description Every `tools[].type` this deployment currently routes to the tool. Always includes the canonical `otari_*` type; for web search it also includes the provider-named keywords when interception is enabled, and for code execution the provider-named keywords unless the deployment's executor is `provider`. */ accepted_types: string[]; /** @@ -11547,6 +11562,8 @@ export interface components { * @description One editable tool/guardrail field surfaced to the dashboard. */ ToolSettingField: { + /** Choices */ + choices?: string[] | null; /** Description */ description?: string | null; /** Key */ @@ -11830,6 +11847,8 @@ export interface components { * } */ UpdateToolSettingsRequest: { + /** Code Execution Executor */ + code_execution_executor?: string | null; /** Guardrails Url */ guardrails_url?: string | null; /** Sandbox Purpose Hint */ @@ -12615,6 +12634,7 @@ export interface components { enabled: boolean; /** Exec Timeout S */ exec_timeout_s: number | null; + executor: components["schemas"]["CodeExecutor"] | null; /** Image */ image: string | null; /** Max Iterations */ @@ -12655,6 +12675,8 @@ export interface components { * @description Ceiling on one execution's runtime in seconds; only ever lowers the effective limit, so at most 60 */ exec_timeout_s?: number | null; + /** @description Who runs a provider-native code-execution declaration for this workspace: 'auto' (the provider when it runs the tool natively for the model, else this gateway's sandbox), 'otari' or 'provider'. Pins over the deployment default and over the request's X-Otari-Code-Execution header; null leaves both in charge */ + executor?: components["schemas"]["CodeExecutor"] | null; /** * Image * @description Sandbox image this workspace's code runs in. Must be one the operator curated into sandbox_allowed_session_images (or the deployment's own sandbox_session_image); null uses the deployment's @@ -14621,6 +14643,10 @@ export interface operations { user?: string | null; purpose?: string | null; workspace_id?: string | null; + limit?: number; + after?: string | null; + after_id?: string | null; + order?: "asc" | "desc"; }; header?: never; path?: never; diff --git a/web/src/features/tools/ToolSettingRows.tsx b/web/src/features/tools/ToolSettingRows.tsx index 2f50a43b03..9fb13d7d66 100644 --- a/web/src/features/tools/ToolSettingRows.tsx +++ b/web/src/features/tools/ToolSettingRows.tsx @@ -55,6 +55,11 @@ export interface FieldCopy { placeholder: string /** Mono, for a value a machine reads. Off for a sentence a model reads. */ isMachineReadable?: boolean + /** + * What each value of a closed-vocabulary field is called, keyed by the value + * the backend lists in `choices`. A value without an entry shows as itself. + */ + choiceLabels?: Record } function useDraft(committed: string) { @@ -247,6 +252,63 @@ function BoolRow({ ) } +// A `str` field the backend closes to a fixed vocabulary (`choices`). A select +// rather than a text box, because the write refuses anything outside the list +// and a field that can only fail on save is worse than one that cannot be +// mistyped. "Default" is a clear, like the tri-state boolean beside it. +function ChoiceRow({ + field, + copy, + commit, + disabled, + defaultLabel, + note, +}: { + field: ToolSettingField + copy: FieldCopy + commit: CommitField + disabled: boolean + defaultLabel: string + note?: React.ReactNode +}) { + const save = useAutosave() + const errorId = useId() + const configKey = keyCaption(copy, field) + const current = + typeof field.value === "string" && field.value ? field.value : "default" + + return ( + + void save.run(() => + commit(field.key, next === "default" ? null : next), + ) + } + options={[ + { value: "default", label: defaultLabel }, + ...(field.choices ?? []).map((choice) => ({ + value: choice, + label: copy.choiceLabels?.[choice] ?? choice, + })), + ]} + disabled={disabled || save.isSaving} + /> + } + /> + ) +} + // The URL row carries Test, which probes the *typed* value so an operator can // check an endpoint before leaving the field. The result is pinned to the URL // it was asked about, so a late answer never lands beside a different one. @@ -338,12 +400,15 @@ export function ToolSettingRow({ disabled, readOnly, defaultLabel = "Default", + note, }: { field: ToolSettingField copy: FieldCopy commit: CommitField disabled: boolean readOnly: boolean + /** Shown under the control, for a condition the help text cannot know. */ + note?: React.ReactNode /** Names what the backend does when nothing is set ("Default (on)"). */ defaultLabel?: string }) { @@ -357,7 +422,7 @@ export function ToolSettingRow({ ? "On" : field.value === false ? "Off" - : String(field.value) + : (copy.choiceLabels?.[String(field.value)] ?? String(field.value)) return ( ) } + if (field.choices && field.choices.length > 0) { + return ( + + ) + } return ( ) diff --git a/web/src/features/tools/ToolsGuardrailsPage.test.tsx b/web/src/features/tools/ToolsGuardrailsPage.test.tsx index cb93316d9d..b566e9a250 100644 --- a/web/src/features/tools/ToolsGuardrailsPage.test.tsx +++ b/web/src/features/tools/ToolsGuardrailsPage.test.tsx @@ -14,7 +14,7 @@ import { CONTROL_LANE } from "@/design-system/layout/SettingRow" import { ToolsGuardrailsPage } from "@/features/tools/ToolsGuardrailsPage" import { API_ROOT } from "@/shared/api/client" import { organizationContext } from "@/tests/fixtures" -import { pickOption } from "@/tests/select" +import { pickOption, selectTrigger } from "@/tests/select" const FIELDS: ToolSettingField[] = [ { @@ -73,6 +73,14 @@ const FIELDS: ToolSettingField[] = [ value: null, description: "Purpose hint.", }, + { + key: "code_execution_executor", + service: "sandbox", + type: "str", + value: null, + description: "Who runs a provider-native code-execution declaration.", + choices: ["auto", "otari", "provider"], + }, { key: "guardrails_url", service: "guardrails", @@ -219,6 +227,10 @@ const MAX_RESULTS = named("Max results", "web_search_max_results") const EXTRACT = named("Extract page content", "web_search_extract") const INTERCEPT = named("Intercept provider web search", "web_search_intercept") const SANDBOX_URL = named("Backend URL", "sandbox_url") +const EXECUTOR = named( + "Who runs provider code tools", + "code_execution_executor", +) const GUARDRAILS_URL = named("Backend URL", "guardrails_url") /** The last PATCH body the page sent, parsed. */ @@ -523,6 +535,26 @@ describe("ToolsGuardrailsPage", () => { ) }) + it("offers a closed-vocabulary setting as a select and saves the chosen value", async () => { + const fetchMock = mockApi() + const user = userEvent.setup() + renderWithClient() + await screen.findByLabelText(SANDBOX_URL) + + expect(selectTrigger(EXECUTOR)).toHaveTextContent("Default (auto)") + // No sandbox URL in the fixture, so the row says the setting is inert. + expect( + screen.getByText(/Takes effect once a Backend URL is set/), + ).toBeInTheDocument() + await pickOption(user, EXECUTOR, "Always here, on this sandbox") + + await waitFor(() => + expect(lastPatch(fetchMock)).toEqual({ + code_execution_executor: "otari", + }), + ) + }) + it("surfaces a failed boolean save inline (not silently)", async () => { mockApi({ patchStatus: 422, diff --git a/web/src/features/tools/ToolsGuardrailsPage.tsx b/web/src/features/tools/ToolsGuardrailsPage.tsx index 51934b19ef..bb93db7599 100644 --- a/web/src/features/tools/ToolsGuardrailsPage.tsx +++ b/web/src/features/tools/ToolsGuardrailsPage.tsx @@ -97,6 +97,17 @@ const FIELD_COPY: Record = { help: "Sent to the backend when a tool entry has none of its own.", placeholder: "Run untrusted analysis code", }, + code_execution_executor: { + label: "Who runs provider code tools", + help: "For a request that declares a provider's own code tool (Anthropic code_execution, OpenAI code_interpreter). Auto keeps it with a provider that runs it natively and brings it here otherwise.", + placeholder: "", + defaultLabel: "Default (auto)", + choiceLabels: { + auto: "Auto: provider when native, else here", + otari: "Always here, on this sandbox", + provider: "Always the provider", + }, + }, guardrails_url: { label: "Backend URL", help: "Used when a request does not pass a guardrail URL of its own.", @@ -105,6 +116,9 @@ const FIELD_COPY: Record = { }, } +const EXECUTOR_NEEDS_BACKEND = + "Takes effect once a Backend URL is set above. Until then provider code tools are always forwarded." + function copyFor(field: ToolSettingField): FieldCopy & { defaultLabel?: string } { @@ -225,16 +239,18 @@ const SERVICES: ServiceSpec[] = [ groups: [ { title: "Backend", - blurb: "The sandbox that runs generated code for otari_code_execution.", + blurb: + "The sandbox that runs generated code. Uploaded files a request references are seeded into it, and files the code writes come back through the files API.", docsAnchor: "code-execution", keys: ["sandbox_url", "sandbox_session_image"], isPriced: true, }, { title: "Behavior", - blurb: "What the gateway sends the sandbox when a request does not.", - docsAnchor: "code-execution", - keys: ["sandbox_purpose_hint"], + blurb: + "Who runs a provider's own code tool, and what the gateway sends the sandbox when a request does not.", + docsAnchor: "code-execution-executor", + keys: ["code_execution_executor", "sandbox_purpose_hint"], catchAll: true, }, ], @@ -453,6 +469,15 @@ export function ToolsGuardrailsPage({ only }: { only?: ToolServiceName } = {}) { } disabled={disabled} readOnly={!isOperator} + // The executor only decides anything once there is a + // sandbox to bring code to; without one every provider + // declaration is forwarded whatever this says. + note={ + field.key === "code_execution_executor" && + !urlField?.value + ? EXECUTOR_NEEDS_BACKEND + : undefined + } /> ) })} diff --git a/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.test.tsx b/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.test.tsx index 7f65ecb99e..edcc6e04fb 100644 --- a/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.test.tsx +++ b/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.test.tsx @@ -20,6 +20,7 @@ import { pickOption, selectTrigger } from "@/tests/select" const ALPHA = "11111111-1111-1111-1111-111111111111" const STANCE = "Code execution for this workspace" const IMAGE = "Sandbox image for this workspace" +const EXECUTOR = "Who runs provider code tools for this workspace" function mockApi({ memberships = [{ workspace_id: ALPHA, name: "Alpha", role: "admin" }], @@ -124,6 +125,7 @@ describe("WorkspaceCodeExecutionPolicyCard", () => { exec_timeout_s: null, image: null, tools: null, + executor: null, }) expect(screen.queryByRole("button", { name: "Save" })).toBeNull() }) @@ -145,6 +147,67 @@ describe("WorkspaceCodeExecutionPolicyCard", () => { expect(await putBody(calls)).toMatchObject({ max_iterations: 4 }) }) + it("offers the executor pin as a choice over the deployment default", async () => { + mockApi({ + policy: workspaceCodeExecutionPolicy({ + workspace_id: ALPHA, + configured: true, + enabled: true, + executor: "otari", + }), + }) + await renderLoaded() + + expect(selectTrigger(EXECUTOR)).toHaveTextContent( + "Always here, on this sandbox", + ) + await userEvent.setup().click(selectTrigger(EXECUTOR)) + expect( + screen.getAllByRole("option").map((option) => option.textContent), + ).toEqual([ + "Deployment default", + "Auto: provider when native, else here", + "Always here, on this sandbox", + "Always the provider", + ]) + }) + + it("saves the executor pin", async () => { + const calls = mockApi({ + policy: workspaceCodeExecutionPolicy({ + workspace_id: ALPHA, + configured: true, + enabled: true, + }), + }) + const user = userEvent.setup() + await renderLoaded() + + await pickOption(user, EXECUTOR, "Always the provider") + expect(await putBody(calls)).toMatchObject({ + enabled: true, + executor: "provider", + }) + }) + + it("clears the executor pin back to the deployment default as null, not an empty string", async () => { + const calls = mockApi({ + policy: workspaceCodeExecutionPolicy({ + workspace_id: ALPHA, + configured: true, + enabled: true, + executor: "provider", + }), + }) + const user = userEvent.setup() + await renderLoaded() + expect(selectTrigger(EXECUTOR)).toHaveTextContent("Always the provider") + + await pickOption(user, EXECUTOR, "Deployment default") + // The service refuses "" (it is outside the vocabulary); null is the clear. + expect(await putBody(calls)).toMatchObject({ executor: null }) + }) + it("offers only the images the operator approved, plus the deployment default", async () => { mockApi({ policy: workspaceCodeExecutionPolicy({ diff --git a/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.tsx b/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.tsx index 6bf50c37f0..edab3384a7 100644 --- a/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.tsx +++ b/web/src/features/tools/WorkspaceCodeExecutionPolicyCard.tsx @@ -43,6 +43,15 @@ type Stance = "default" | "allowed" | "blocked" // The sentinel for "no workspace image", which is a real choice and not an // absent one: the workspace runs whatever the deployment runs. const DEPLOYMENT_IMAGE = "" +// The sentinel for "no workspace pin" on who runs a provider's code tool: the +// deployment default, and the request's own header, decide. +const DEPLOYMENT_EXECUTOR = "" +const EXECUTOR_OPTIONS = [ + { value: DEPLOYMENT_EXECUTOR, label: "Deployment default" }, + { value: "auto", label: "Auto: provider when native, else here" }, + { value: "otari", label: "Always here, on this sandbox" }, + { value: "provider", label: "Always the provider" }, +] // The server's own ceilings (`workspace_code_execution_policy_service`): a value // above either could never take effect, so it is refused rather than stored. @@ -75,6 +84,7 @@ export function WorkspaceCodeExecutionPolicyCard({ const stanceSave = useAutosave() const imageSave = useAutosave() const toolsSave = useAutosave() + const executorSave = useAutosave() // One writer for the group: a PUT replaces the whole policy, so two rows // saving at once would each carry the other's pre-save value. const write = usePolicyWriter({ @@ -87,6 +97,7 @@ export function WorkspaceCodeExecutionPolicyCard({ exec_timeout_s: stored.exec_timeout_s, image: stored.image, tools: stored.tools, + executor: stored.executor, }), put: (body: UpdateWorkspaceCodeExecutionPolicyRequest) => setPolicy.mutateAsync({ @@ -268,6 +279,31 @@ export function WorkspaceCodeExecutionPolicyCard({ disabled={narrowingDisabled} /> + + void executorSave.run(() => + commitField({ + executor: + next === DEPLOYMENT_EXECUTOR + ? null + : (next as UpdateWorkspaceCodeExecutionPolicyRequest["executor"]), + }), + ) + } + options={EXECUTOR_OPTIONS} + disabled={narrowingDisabled || executorSave.isSaving} + /> + } + /> +