Add bounded image-only Agent Browser view frames (#84)

Author: AetherAI3

CHANGELOG.md

@@ -6,6 +6,13 @@ All notable changes to Agent Browser are recorded here. The format follows
 
 ## [Unreleased]
 
+### Added
+
+- An observer-authorized, image-only `POST /browser/view-frame` endpoint and matching Python/Node
+  client methods for a future authenticated RC relay. Capture is limited to one frame per second
+  per owned session, omits page text and form values, and does not consume model vision steps.
+  The local noVNC port remains loopback-only and must not be tunneled.
+
 ## [0.2.2] - 2026-09-04
 
 ### Added

clients/node/src/index.d.ts

@@ -39,6 +39,7 @@ export type ErrorCode =
   | 'SESSION_NOT_FOUND'
   | 'SESSION_EXPIRED'
   | 'VISION_BUDGET_EXHAUSTED'
+  | 'VIEW_FRAME_RATE_LIMITED'
   | 'INVALID_URL'
   | 'DESTINATION_BLOCKED'
   | 'INVALID_INTERACTION'
@@ -114,6 +115,15 @@ export interface SnapshotResponse {
   vision_steps_remaining: number
 }
 
+export interface ViewFrameResponse {
+  api_version: 'v1'
+  status: 'view_frame'
+  session_id: string
+  screenshot_base64: string
+  viewport: Viewport
+  captured_at: string
+}
+
 export interface InteractResponse {
   api_version: 'v1'
   status: 'interacted'
@@ -181,6 +191,7 @@ export declare class Session {
 
   navigate(url: string, options?: RequestOptions): Promise<NavigateResponse>
   snapshot(options?: RequestOptions): Promise<SnapshotResponse>
+  viewFrame(options?: RequestOptions): Promise<ViewFrameResponse>
   click(
     target: { selector: string; x?: never; y?: never } | { x: number; y: number; selector?: never },
     options?: RequestOptions,

clients/node/src/index.js

@@ -116,6 +116,11 @@ export class Session {
     return this.browser._post('/browser/snapshot', { session_id: this.id }, 'observer', options)
   }
 
+  /** Capture pixels only for a local relay; does not consume model vision budget. */
+  viewFrame(options = {}) {
+    return this.browser._post('/browser/view-frame', { session_id: this.id }, 'observer', options)
+  }
+
   /**
    * Click a selector or an x/y point. Exactly one of the two is allowed by the server.
    * @param {{ selector?: string, x?: number, y?: number }} target

clients/node/test/client.test.js

@@ -94,6 +94,16 @@ test('routes reads to the observer token and writes to the controller token', as
   assert.equal(calls[1].headers.authorization, 'Bearer ctl')
 })
 
+test('view frames use observer authority and the image-only route', async () => {
+  const { browser, calls } = client([{ body: CREATED }, { body: { status: 'view_frame' } }], {
+    observerToken: 'obs',
+    controllerToken: 'ctl',
+  })
+  await (await browser.createSession()).viewFrame()
+  assert.ok(calls[1].url.endsWith('/browser/view-frame'))
+  assert.equal(calls[1].headers.authorization, 'Bearer obs')
+})
+
 test('falls back to the only token supplied', async () => {
   const { browser, calls } = client([{ body: { status: 'ok' } }], { controllerToken: 'ctl' })
   await browser.health()

clients/python/src/aether_browser/_client.py

@@ -157,6 +157,12 @@ def snapshot(self, *, timeout: float | None = None) -> dict[str, Any]:
         """Capture bounded page state plus a base64 PNG. Consumes exactly one vision step."""
         return self.browser._post("/browser/snapshot", {"session_id": self.id}, "observer", timeout)
 
+    def view_frame(self, *, timeout: float | None = None) -> dict[str, Any]:
+        """Capture pixels only for a local relay; does not consume model vision budget."""
+        return self.browser._post(
+            "/browser/view-frame", {"session_id": self.id}, "observer", timeout
+        )
+
     def click(
         self,
         *,

clients/python/tests/test_client.py

@@ -131,6 +131,16 @@ def test_snapshot_is_a_read(self) -> None:
         self.assertTrue(calls[1]["url"].endswith("/browser/snapshot"))
         self.assertEqual(calls[1]["headers"]["authorization"], "Bearer obs")
 
+    def test_view_frame_is_an_observer_read(self) -> None:
+        browser, calls = client(
+            [{"body": CREATED}, {"body": {"status": "view_frame"}}],
+            observer_token=OBSERVER,
+            controller_token=CONTROLLER,
+        )
+        browser.create_session().view_frame()
+        self.assertTrue(calls[1]["url"].endswith("/browser/view-frame"))
+        self.assertEqual(calls[1]["headers"]["authorization"], "Bearer obs")
+
     def test_falls_back_to_the_only_token_supplied(self) -> None:
         browser, calls = client([{"body": {"status": "ok"}}], controller_token=CONTROLLER)
         browser.health()

docs/API.md

@@ -42,6 +42,7 @@ Release acceptance instead consumes the immutable image ID from the preceding ex
 | `POST /browser/session/create` | no | yes |
 | `POST /browser/navigate` | no | yes |
 | `POST /browser/snapshot` | yes | yes |
+| `POST /browser/view-frame` | yes | yes |
 | `POST /browser/interact` | no | yes |
 | `POST /browser/session/end` | no | yes |
 
@@ -54,6 +55,7 @@ Release acceptance instead consumes the immutable image ID from the preceding ex
 - Accessibility snapshot: 500 flattened nodes.
 - Encoded PNG screenshot: 14,000,000 base64 characters.
 - Vision budget: 1–100 snapshots, default 25.
+- Image-only view capture: at most one frame per second per session; 429 includes `Retry-After: 1`.
 - Coordinates: 0–4,095; scroll delta: -10,000–10,000.
 - Capacity retry guidance: 1–300 seconds.
 
@@ -69,6 +71,8 @@ Release acceptance instead consumes the immutable image ID from the preceding ex
 
 `POST /browser/snapshot` atomically consumes one vision step and increments the session sequence. It returns bounded structured state, a base64 PNG, viewport metadata, counters, and capture time.
 
+`POST /browser/view-frame` accepts the same owned session ID and returns only a bounded base64 PNG, viewport, and capture time. It does not extract page text, accessibility, URL, or form values and does not consume the model vision budget. Pixels can still show what the human typed; a relay must keep frames out of model context and logs. It is a loopback API for an authenticated relay; it does not make the unauthenticated noVNC port safe to expose. The relay must enforce the remote owner/session grant separately.
+
 ## Interaction
 
 Only `click`, `type`, `scroll`, and `press` exist. Click/type require either a selector or an x/y pair, never both. Scroll accepts nonzero bounded deltas. Press accepts only the enumerated keys and combinations; clipboard shortcuts are not allowlisted. Typed text is preserved byte-for-byte after JSON decoding, including leading and trailing whitespace. There is no arbitrary JavaScript, CDP, upload, clipboard, download, extension, shell, filesystem, credential, or cookie field.

src/agent_browser/main.py

@@ -39,6 +39,7 @@
     NavigateResponse,
     SnapshotRequest,
     SnapshotResponse,
+    ViewFrameResponse,
 )
 from agent_browser.runtime import (
     BrowserLaunchError,
@@ -56,6 +57,7 @@
     SessionExpiredError,
     SessionManager,
     SessionNotFoundError,
+    ViewFrameRateLimitedError,
     VisionBudgetExhaustedError,
 )
 
@@ -575,6 +577,17 @@ async def snapshot(payload: SnapshotRequest, request: Request) -> SnapshotRespon
             vision_steps_remaining=result.vision_steps_remaining,
         )
 
+    @application.post("/browser/view-frame", response_model=ViewFrameResponse)
+    async def view_frame(payload: SnapshotRequest, request: Request) -> ViewFrameResponse:
+        await require(request, RequiredAuthority.OBSERVER)
+        result = await session_manager.capture_view_frame(payload.session_id)
+        return ViewFrameResponse(
+            session_id=result.session_id,
+            screenshot_base64=result.frame.screenshot_base64,
+            viewport=result.frame.viewport,
+            captured_at=result.captured_at,
+        )
+
     @application.post("/browser/interact", response_model=InteractResponse)
     async def interact(payload: InteractRequest, request: Request) -> InteractResponse:
         await require(request, RequiredAuthority.CONTROLLER)
@@ -617,6 +630,8 @@ def _known_fault(error: Exception) -> _ApiFault | None:
         return _ApiFault(ErrorCode.SESSION_EXPIRED, 410)
     if isinstance(error, VisionBudgetExhaustedError):
         return _ApiFault(ErrorCode.VISION_BUDGET_EXHAUSTED, 409)
+    if isinstance(error, ViewFrameRateLimitedError):
+        return _ApiFault(ErrorCode.VIEW_FRAME_RATE_LIMITED, 429, retry_after_seconds=1)
     if isinstance(error, InvalidBrowserInteractionError):
         return _ApiFault(ErrorCode.INVALID_INTERACTION, 400)
     if isinstance(error, (BrowserLaunchError, BrowserNotReadyError)):
@@ -661,6 +676,7 @@ def _external_security_fault(error: BaseException) -> _ApiFault | None:
     ErrorCode.SESSION_NOT_FOUND: "Session was not found.",
     ErrorCode.SESSION_EXPIRED: "Session expired.",
     ErrorCode.VISION_BUDGET_EXHAUSTED: "The snapshot budget is exhausted.",
+    ErrorCode.VIEW_FRAME_RATE_LIMITED: "View frame capture is rate limited.",
     ErrorCode.INVALID_URL: "The navigation URL is invalid.",
     ErrorCode.DESTINATION_BLOCKED: "The navigation destination is blocked.",
     ErrorCode.INVALID_INTERACTION: "The request or interaction is invalid.",

src/agent_browser/models.py

@@ -74,6 +74,7 @@ class ErrorCode(StrEnum):
     SESSION_NOT_FOUND = "SESSION_NOT_FOUND"
     SESSION_EXPIRED = "SESSION_EXPIRED"
     VISION_BUDGET_EXHAUSTED = "VISION_BUDGET_EXHAUSTED"
+    VIEW_FRAME_RATE_LIMITED = "VIEW_FRAME_RATE_LIMITED"
     INVALID_URL = "INVALID_URL"
     DESTINATION_BLOCKED = "DESTINATION_BLOCKED"
     INVALID_INTERACTION = "INVALID_INTERACTION"
@@ -215,6 +216,22 @@ class SnapshotResponse(ClosedModel):
     _captured_at_is_utc = field_validator("captured_at")(_validate_utc)
 
 
+class ViewFrameResponse(ClosedModel):
+    """Pixel-only response for a local, authenticated viewer relay."""
+
+    api_version: ApiVersion = API_VERSION
+    status: Literal["view_frame"] = "view_frame"
+    session_id: UUID
+    screenshot_base64: Annotated[
+        str,
+        StringConstraints(min_length=1, max_length=MAX_SCREENSHOT_BASE64_CHARS),
+    ]
+    viewport: Viewport
+    captured_at: datetime
+
+    _captured_at_is_utc = field_validator("captured_at")(_validate_utc)
+
+
 class InteractionTarget(ClosedModel):
     selector: BoundedSelector | None = None
     x: int | None = Field(default=None, ge=0, le=4095)

src/agent_browser/runtime.py

@@ -152,6 +152,14 @@ class BrowserSnapshot:
     viewport: Viewport
 
 
+@dataclass(frozen=True, slots=True)
+class BrowserViewFrame:
+    """Image-only local viewing frame; no page text or accessibility tree."""
+
+    screenshot_base64: str
+    viewport: Viewport
+
+
 class BrowserAdapter(Protocol):
     """The complete browser capability surface owned by a session."""
 
@@ -168,6 +176,9 @@ async def navigate(self, url: str) -> BrowserPageState:
     async def snapshot(self) -> BrowserSnapshot:
         """Capture bounded structured state and a PNG screenshot."""
 
+    async def capture_view_frame(self) -> BrowserViewFrame:
+        """Capture an image-only frame without consuming model vision budget."""
+
     async def click(
         self,
         *,
@@ -460,16 +471,7 @@ async def snapshot(self) -> BrowserSnapshot:
         try:
             async with asyncio.timeout(self._action_timeout):
                 state = await self._extract_page_state(page)
-                screenshot = await page.screenshot(
-                    type="png",
-                    full_page=False,
-                    animations="disabled",
-                    caret="hide",
-                    timeout=int(self._action_timeout * 1000),
-                )
-                encoded = base64.b64encode(bytes(screenshot)).decode("ascii")
-                if len(encoded) > MAX_SCREENSHOT_BASE64_CHARS:
-                    raise BrowserOperationError("The browser snapshot exceeded its size limit.")
+                encoded = await self._capture_png(page)
                 return BrowserSnapshot(
                     page=state,
                     screenshot_base64=encoded,
@@ -478,6 +480,31 @@ async def snapshot(self) -> BrowserSnapshot:
         except BaseException as error:
             self._raise_operation_error(error, "Snapshot failed.")
 
+    async def capture_view_frame(self) -> BrowserViewFrame:
+        """Read pixels only; the RC relay never needs DOM or form values."""
+        page = self._require_page()
+        try:
+            async with asyncio.timeout(self._action_timeout):
+                return BrowserViewFrame(
+                    screenshot_base64=await self._capture_png(page),
+                    viewport=self._viewport,
+                )
+        except BaseException as error:
+            self._raise_operation_error(error, "View capture failed.")
+
+    async def _capture_png(self, page: Any) -> str:
+        screenshot = await page.screenshot(
+            type="png",
+            full_page=False,
+            animations="disabled",
+            caret="hide",
+            timeout=int(self._action_timeout * 1000),
+        )
+        encoded = base64.b64encode(bytes(screenshot)).decode("ascii")
+        if len(encoded) > MAX_SCREENSHOT_BASE64_CHARS:
+            raise BrowserOperationError("The browser frame exceeded its size limit.")
+        return encoded
+
     async def click(
         self,
         *,