feat(files): serve Anthropic's GA Files API shape and refuse the files beta header

demo/code-exec/plot_with_anthropic_sdk.py

@@ -66,8 +66,8 @@ def produced_file_ids(message: Any) -> list[str]:
 
 def download(file_id: str, model: str) -> pathlib.Path:
     """Fetch a produced file from Otari, wherever its bytes actually live."""
-    meta = otari.beta.files.retrieve_metadata(file_id)
-    body = otari.beta.files.download(file_id)
+    meta = otari.files.retrieve_metadata(file_id)
+    body = otari.files.download(file_id)
 
     OUT_DIR.mkdir(parents=True, exist_ok=True)
     path = OUT_DIR / f"{model.replace(':', '-').replace('/', '-')}-{meta.filename}"

docs/files.md

@@ -65,10 +65,18 @@ The five routes (`POST`/`GET /v1/files`, `GET`/`DELETE /v1/files/{id}`,
 `GET /v1/files/{id}/content`) share their paths and verbs with both vendors'
 Files APIs, so either official SDK works against Otari with only its base URL
 changed. The response shape follows the caller: a request carrying Anthropic's
-`anthropic-version` header, which its SDK sends on every call, gets Anthropic's
-`FileMetadata` (`type`, `size_bytes`, `mime_type`, `downloadable`, an RFC 3339
-`created_at`); everything else gets the OpenAI file object (`object`, `bytes`,
-`purpose`, an epoch `created_at`).
+`anthropic-version` header, which its SDK sends on every call, gets the
+`FileMetadata` of Anthropic's GA Files API (`type`, `size_bytes`, `mime_type`,
+`downloadable`, and RFC 3339 `created_at` and `expires_at`, with `expires_at`
+`null` for a file kept indefinitely); everything else gets the OpenAI file
+object (`object`, `bytes`, `purpose`, an epoch `created_at`).
+
+Otari serves Anthropic's GA shapes only. A request whose `anthropic-beta`
+header includes `files-api-2025-04-14` gets a 400, because that beta answers in
+different shapes. Anthropic's Python SDK before 1.2.0, and earlier releases of
+its other SDKs, send that header from `client.beta.files`, so call
+`client.files` instead (see Anthropic's
+[migration notes](https://platform.claude.com/docs/en/build-with-claude/files#migrate-from-files-api-2025-04-14)).
 
 Mind the base URL: Anthropic's SDK appends `/v1` itself, so it takes
 `http://localhost:8000/api`, while an OpenAI-compatible client takes
@@ -78,13 +86,21 @@ Mind the base URL: Anthropic's SDK appends `/v1` itself, so it takes
 ```python
 from anthropic import Anthropic
 client = Anthropic(base_url="http://localhost:8000/api", api_key="<your-api-key>")
-meta = client.beta.files.upload(file=("report.pdf", open("report.pdf", "rb"), "application/pdf"))
-client.beta.files.download(meta.id)  # Otari serves every stored file's bytes back
+meta = client.files.upload(file=("report.pdf", open("report.pdf", "rb"), "application/pdf"))
+client.files.download(meta.id)  # Otari serves every stored file's bytes back
 ```
 
-Listings are cursor-paged: `limit` (default 100, at most 1000), `after`
-(OpenAI) or `after_id` (Anthropic) naming the last file of the previous page,
-`order` (`desc` by default), and `has_more`, `first_id`, `last_id` on the page.
+Listings are cursor-paged, and each flavor pages with its own vendor's cursor.
+Both take `limit` (default 100, at most 1000).
+
+- OpenAI: `after` names the last file of the previous page, `order` is `desc`
+  by default, and the page carries `has_more`, `first_id` and `last_id`.
+- Anthropic: the page is `{data, next_page}`, and `next_page` goes back as
+  `page` to get the next one. To read up to 100 known files in one page, name
+  them with `ids[]`, which cannot be combined with `page` or `limit`; a file you
+  cannot see is left out. `after_id` and `before_id` get a 400.
+
+Unlike Anthropic, Otari leaves an expired file out of a listing.
 
 ## Files and code execution
 

docs/public/openapi.json

@@ -20923,7 +20923,7 @@
     },
     "/api/v1/files": {
       "get": {
-        "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nPages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names\nthe last file of the previous page, and ``has_more`` says whether to ask\nagain. A cursor that has since been deleted or has expired is still a\nposition; one the caller never owned is a 404.",
+        "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nEach flavor pages with its own cursor.\nOpenAI's ``after`` names the last file of the previous page, and ``has_more`` says whether to ask again.\nAnthropic's ``next_page`` is passed back as ``page``, and ``ids[]`` reads up to 100 named files in one page.\nA cursor whose file has since been deleted or has expired is still a position.\nAn ``after`` the caller never owned is a 404, and a ``page`` token this gateway did not issue is a 400.",
         "operationId": "files-list_files",
         "parameters": [
           {
@@ -21005,7 +21005,21 @@
           },
           {
             "in": "query",
-            "name": "after_id",
+            "name": "order",
+            "required": false,
+            "schema": {
+              "default": "desc",
+              "enum": [
+                "asc",
+                "desc"
+              ],
+              "title": "Order",
+              "type": "string"
+            }
+          },
+          {
+            "in": "query",
+            "name": "page",
             "required": false,
             "schema": {
               "anyOf": [
@@ -21016,21 +21030,26 @@
                   "type": "null"
                 }
               ],
-              "title": "After Id"
+              "title": "Page"
             }
           },
           {
             "in": "query",
-            "name": "order",
+            "name": "ids[]",
             "required": false,
             "schema": {
-              "default": "desc",
-              "enum": [
-                "asc",
-                "desc"
+              "anyOf": [
+                {
+                  "items": {
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                {
+                  "type": "null"
+                }
               ],
-              "title": "Order",
-              "type": "string"
+              "title": "Ids[]"
             }
           }
         ],

docs/public/otari.postman_collection.json

@@ -1763,7 +1763,7 @@
         {
           "name": "List Files",
           "request": {
-            "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nPages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names\nthe last file of the previous page, and ``has_more`` says whether to ask\nagain. A cursor that has since been deleted or has expired is still a\nposition; one the caller never owned is a 404.",
+            "description": "List the authenticated user's uploaded files in the request's workspace.\n\n``workspace_id`` narrows a master-key listing to one workspace; a keyed\nrequest is already confined to its key's own and cannot widen or move it.\n\nEach flavor pages with its own cursor.\nOpenAI's ``after`` names the last file of the previous page, and ``has_more`` says whether to ask again.\nAnthropic's ``next_page`` is passed back as ``page``, and ``ids[]`` reads up to 100 named files in one page.\nA cursor whose file has since been deleted or has expired is still a position.\nAn ``after`` the caller never owned is a 404, and a ``page`` token this gateway did not issue is a 400.",
             "header": [],
             "method": "GET",
             "url": {
@@ -1809,17 +1809,23 @@
                 {
                   "description": "",
                   "disabled": true,
-                  "key": "after_id",
+                  "key": "order",
                   "value": ""
                 },
                 {
                   "description": "",
                   "disabled": true,
-                  "key": "order",
+                  "key": "page",
+                  "value": ""
+                },
+                {
+                  "description": "",
+                  "disabled": true,
+                  "key": "ids[]",
                   "value": ""
                 }
               ],
-              "raw": "{{baseUrl}}/api/v1/files?user=&purpose=&workspace_id=&limit=&after=&after_id=&order="
+              "raw": "{{baseUrl}}/api/v1/files?user=&purpose=&workspace_id=&limit=&after=&order=&page=&ids[]="
             }
           }
         },

src/gateway/api/routes/files.py

@@ -18,8 +18,10 @@
 shape follows the caller: a request carrying Anthropic's ``anthropic-version``
 header (which its SDK sends on every call) gets ``FileMetadata``, everything
 else gets the OpenAI file object.
+The Anthropic flavor is its GA shape only, so a request for the Files API beta is a 400.
 """
 
+import base64
 import uuid
 from collections.abc import AsyncGenerator, AsyncIterator
 from datetime import UTC, datetime
@@ -44,7 +46,20 @@
 from gateway.services.files.provider_files import stream_provider_file
 from gateway.services.workspace_scope import default_workspace_id
 
-router = APIRouter(tags=["files"])
+_FILES_BETA = "files-api-2025-04-14"
+
+
+async def _refuse_files_beta(raw_request: Request) -> None:
+    """Refuse Anthropic's Files API beta, whose shapes differ from the GA shapes served here."""
+    for header_value in raw_request.headers.getlist("anthropic-beta"):
+        if _FILES_BETA in (beta.strip() for beta in header_value.split(",")):
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"The {_FILES_BETA} beta is not supported: send Files API requests without it in anthropic-beta",
+            )
+
+
+router = APIRouter(tags=["files"], dependencies=[Depends(_refuse_files_beta)])
 
 # OpenAI's documented file purposes plus a generic default. We don't enforce the
 # enum (forward-compat), but normalise the empty case to "user_data".
@@ -54,13 +69,62 @@
 # OpenAI's 10000 because a page is one query and one JSON body.
 _DEFAULT_LIST_LIMIT = 100
 _MAX_LIST_LIMIT = 1000
+_MAX_LIST_IDS = 100
+
+_PAGE_TOKEN_PREFIX = "page_"
 
 
 def _anthropic_shape(raw_request: Request) -> bool:
     """Whether the caller speaks Anthropic's Files API rather than OpenAI's."""
-    return "anthropic-version" in raw_request.headers or any(
-        beta.strip().startswith("files-api") for beta in raw_request.headers.get("anthropic-beta", "").split(",")
-    )
+    return "anthropic-version" in raw_request.headers
+
+
+def _page_token(file_id: str) -> str:
+    """The opaque Anthropic ``next_page`` token that resumes a listing after ``file_id``."""
+    return _PAGE_TOKEN_PREFIX + base64.urlsafe_b64encode(file_id.encode()).decode().rstrip("=")
+
+
+def _could_name_a_file(value: str) -> bool:
+    # Every file ID is printable ASCII, and PostgreSQL rejects a NUL in a text parameter.
+    return value.isascii() and value.isprintable()
+
+
+def _invalid_page_token() -> HTTPException:
+    return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid page token")
+
+
+def _page_token_file_id(token: str) -> str:
+    """The file ID a ``page`` token resumes after, raising a 400 for a token this gateway did not issue."""
+    if not token.startswith(_PAGE_TOKEN_PREFIX):
+        raise _invalid_page_token()
+    encoded = token.removeprefix(_PAGE_TOKEN_PREFIX)
+    try:
+        file_id = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)).decode()
+    except ValueError as exc:
+        raise _invalid_page_token() from exc
+    if not _could_name_a_file(file_id):
+        raise _invalid_page_token()
+    return file_id
+
+
+def _check_anthropic_list_params(raw_request: Request, page: str | None, ids: list[str] | None) -> None:
+    """Refuse the parameter combinations Anthropic's GA listing refuses, given de-duplicated ``ids``."""
+    params = raw_request.query_params
+    if "after_id" in params or "before_id" in params:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="after_id and before_id are not supported: pass next_page back as page instead",
+        )
+    if ids is None:
+        return
+    if page is not None or "limit" in params:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail="ids[] cannot be combined with page or limit"
+        )
+    if len(ids) > _MAX_LIST_IDS:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST, detail=f"ids[] takes at most {_MAX_LIST_IDS} file IDs"
+        )
 
 
 def _serialize(record: FileObject, raw_request: Request) -> dict[str, Any]:
@@ -255,22 +319,31 @@ async def list_files(
     workspace_id: uuid.UUID | None = None,
     limit: Annotated[int, Query(ge=1, le=_MAX_LIST_LIMIT)] = _DEFAULT_LIST_LIMIT,
     after: str | None = None,
-    after_id: str | None = None,
     order: Literal["asc", "desc"] = "desc",
+    page: str | None = None,
+    ids: Annotated[list[str] | None, Query(alias="ids[]")] = None,
 ) -> dict[str, Any]:
     """List the authenticated user's uploaded files in the request's workspace.
 
     ``workspace_id`` narrows a master-key listing to one workspace; a keyed
     request is already confined to its key's own and cannot widen or move it.
 
-    Pages are cursor-based: ``after`` (OpenAI) or ``after_id`` (Anthropic) names
-    the last file of the previous page, and ``has_more`` says whether to ask
-    again. A cursor that has since been deleted or has expired is still a
-    position; one the caller never owned is a 404.
+    Each flavor pages with its own cursor.
+    OpenAI's ``after`` names the last file of the previous page, and ``has_more`` says whether to ask again.
+    Anthropic's ``next_page`` is passed back as ``page``, and ``ids[]`` reads up to 100 named files in one page.
+    A cursor whose file has since been deleted or has expired is still a position.
+    An ``after`` the caller never owned is a 404, and a ``page`` token this gateway did not issue is a 400.
     """
     if not config.files_enabled:
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File uploads are disabled")
 
+    anthropic = _anthropic_shape(raw_request)
+    if anthropic:
+        named_ids = None if ids is None else list(dict.fromkeys(ids))
+        _check_anthropic_list_params(raw_request, page, named_ids)
+        cursor_id = _page_token_file_id(page) if page is not None else None
+    else:
+        named_ids, cursor_id = None, after
     user_id = _resolve_user(auth_result, user, config)
     # The key's own workspace wins over anything the caller sent, rather than
     # 400ing on a mismatch: the parameter is a master-key narrowing, and a keyed
@@ -289,17 +362,20 @@ async def list_files(
         stmt = stmt.where(FileObject.workspace_id == scope)
     if purpose is not None:
         stmt = stmt.where(FileObject.purpose == purpose)
+    if named_ids is not None:
+        stmt = stmt.where(FileObject.id.in_([file_id for file_id in named_ids if _could_name_a_file(file_id)]))
 
-    cursor_id = after or after_id
     if cursor_id is not None:
         # A position, not a file: the row is read with the tenant predicates
         # only, so a cursor that was deleted or expired between two pages (the
         # usual "list, delete each, list again" loop) still says where the next
-        # page starts. Another user's id stays a 404.
+        # page starts. Another user's ID is refused.
         cursor_conditions = [FileObject.id == cursor_id, FileObject.user_id == user_id]
         if scope is not None:
             cursor_conditions.append(FileObject.workspace_id == scope)
         cursor = (await db.execute(select(FileObject).where(*cursor_conditions))).scalar_one_or_none()
+        if cursor is None and anthropic:
+            raise _invalid_page_token()
         if cursor is None:
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="File not found")
         # (created_at, id) is the sort key, so the page after the cursor is
@@ -325,16 +401,17 @@ async def list_files(
     records = list((await db.execute(stmt.limit(limit + 1))).scalars().all())
     has_more = len(records) > limit
     records = records[:limit]
+    data = [_serialize(r, raw_request) for r in records]
 
-    page: dict[str, Any] = {
-        "data": [_serialize(r, raw_request) for r in records],
+    if anthropic:
+        return {"data": data, "next_page": _page_token(records[-1].id) if has_more else None}
+    return {
+        "object": "list",
+        "data": data,
         "has_more": has_more,
         "first_id": records[0].id if records else None,
         "last_id": records[-1].id if records else None,
     }
-    if not _anthropic_shape(raw_request):
-        page = {"object": "list", **page}
-    return page
 
 
 @router.get("/files/{file_id}")

src/gateway/models/tools.py

@@ -37,6 +37,15 @@ def _epoch_seconds(value: datetime | None) -> int | None:
     return int(value.timestamp())
 
 
+def _rfc3339(value: datetime | None) -> str | None:
+    """Return an RFC 3339 timestamp from a stored datetime, reading a naive value as UTC."""
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=UTC)
+    return value.isoformat().replace("+00:00", "Z")
+
+
 class SearchToolCredential(Base):
     """A ``POST /v1/search`` tool configured at runtime through the dashboard.
 
@@ -161,24 +170,19 @@ def to_dict(self) -> dict[str, Any]:
         }
 
     def to_anthropic_dict(self) -> dict[str, Any]:
-        """Convert to the Anthropic Files API ``FileMetadata`` shape.
+        """Convert to the ``FileMetadata`` shape of Anthropic's GA Files API.
 
-        Anthropic's SDK reads ``size_bytes`` and ``mime_type`` where OpenAI's
-        reads ``bytes`` and nothing, and takes ``created_at`` as an RFC 3339
-        string rather than an epoch. ``downloadable`` is always true here: the
-        gateway serves every stored file's bytes back, unlike Anthropic, which
-        withholds user uploads.
+        ``expires_at`` is always present and ``None`` for a file kept indefinitely.
+        ``downloadable`` is always true, because the gateway serves every stored file's bytes back.
         """
-        created_at = self.created_at
-        if created_at.tzinfo is None:
-            created_at = created_at.replace(tzinfo=UTC)
         return {
             "id": self.id,
             "type": "file",
             "filename": self.filename,
             "mime_type": self.mime_type,
             "size_bytes": self.bytes,
-            "created_at": created_at.isoformat().replace("+00:00", "Z"),
+            "created_at": _rfc3339(self.created_at),
+            "expires_at": _rfc3339(self.expires_at),
             "downloadable": True,
         }