From b0593b290ee527f05e436fe2ec9fb3fb44b75ac4 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:53:05 +0100 Subject: [PATCH 1/2] feat(browser): show an agent cursor where the agent moves and clicks The Electron kernel reports each agent point (move, then click) before the action. The Browser overlay maps it through page zoom and the responsive scale, pings on click, dims when idle, and hides while the user holds control. During a tab recording the guest page draws the same mark so the video includes it. Closes #2210 --- .../preview/__tests__/preview-feature.test.ts | 1 + .../browser-automation-kernel-races.test.ts | 4 + .../browser-automation-kernel.test.ts | 54 +++++-- .../__tests__/browser-executor-parity.test.ts | 1 + .../src/features/preview/automation/index.ts | 11 +- .../src/features/preview/automation/kernel.ts | 94 +++++++++-- .../features/preview/contracts/guest-input.ts | 13 ++ .../preload/__tests__/agent-cursor.test.ts | 62 +++++++ .../features/preview/preload/agent-cursor.ts | 153 ++++++++++++++++++ .../features/preview/preload/guest-input.ts | 10 ++ apps/desktop/src/main/preload.ts | 9 ++ .../automation/BrowserAutomationHost.tsx | 6 + .../__tests__/BrowserAutomationHost.test.tsx | 1 + .../browserAutomationRecorder.test.ts | 23 ++- .../automation/browserAutomationRecorder.ts | 17 +- .../automation/browserAutomationStore.ts | 35 +++- .../preview/surfaces/BrowserActingOverlay.tsx | 125 ++++++++++++-- .../__tests__/BrowserActingOverlay.test.tsx | 92 +++++++++++ .../surfaces/__tests__/PreviewPanel.test.tsx | 6 +- apps/web/src/index.css | 15 ++ apps/web/src/transport/desktop-bridge.d.ts | 5 + packages/contracts/src/index.ts | 2 + .../src/models/browser-automation.ts | 29 +++- 23 files changed, 720 insertions(+), 48 deletions(-) create mode 100644 apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts create mode 100644 apps/desktop/src/features/preview/preload/agent-cursor.ts create mode 100644 apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx diff --git a/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts b/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts index 712e769fb..a314dd11e 100644 --- a/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts +++ b/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts @@ -125,6 +125,7 @@ describe("Preview feature public interface", () => { "preview:automation.release-agent-control", "preview:automation.describe-target", "preview:automation.media-source", + "preview:automation.recording-cursor", "preview:open-guest-devtools", "preview:get-perf-counters", ]; diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts index b98a87d6b..a90d126f6 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts @@ -160,6 +160,10 @@ class FakeWebContents extends NodeEvents.EventEmitter { return this.destroyed; } + getZoomFactor(): number { + return 1; + } + getURL(): string { return this.url; } diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts index 4091051de..4c2b2e3af 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { BROWSER_AUTOMATION_CONTRACT_VERSION, type BrowserAutomationRequest } from "@mcode/contracts"; import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, PreviewGuestInputSuppressor, } from "../../contracts/guest-input.js"; @@ -159,6 +160,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { url = "https://example.test/"; title = "Example"; focused = true; + zoomFactor = 1; focus = vi.fn(() => { this.focused = true; }); @@ -166,6 +168,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { isDestroyed() { return this.destroyed; } getURL() { return this.url; } getTitle() { return this.title; } + getZoomFactor() { return this.zoomFactor; } isLoading() { return false; } isFocused() { return this.focused; } async executeJavaScript(source: string) { @@ -650,9 +653,12 @@ describe("BrowserAutomationKernel", () => { await expect(click("opaque-replacement", "semantic-replacement")).resolves.toMatchObject({ ok: true }); }); - it("publishes a resolved semantic target pointer before agent input", async () => { - const states: unknown[] = []; - const unsubscribe = kernel.subscribe(7, (state) => states.push(state)); + it("reports a move then a click at the target's centre before pressing the mouse", async () => { + const pointerEvents: Array<{ event: unknown; commandsBefore: number }> = []; + const unsubscribe = kernel.subscribePointer(7, (pointerEvent) => { + pointerEvents.push({ event: pointerEvent, commandsBefore: currentWebContents!.debugger.commands.length }); + }); + currentWebContents!.zoomFactor = 1.5; currentWebContents!.semanticElements.set("semantic-button", { attached: true, visible: true, x: 25, y: 40 }); await kernel.execute(event(), payload(request("click", { target: { semanticId: "semantic-button" }, @@ -660,15 +666,41 @@ describe("BrowserAutomationKernel", () => { clickCount: 1, timeoutMs: 1_000, }, { requestId: "semantic-pointer" }))); - expect(states).toContainEqual(expect.objectContaining({ - controller: "agent", - pointer: { x: 25, y: 40 }, - })); - expect(currentWebContents!.send).toHaveBeenCalledWith( - "mcode:browser-agent-input", - expect.objectContaining({ kind: "pointer", count: 1 }), - ); unsubscribe(); + + expect(pointerEvents.map(({ event }) => event)).toEqual([ + { tabId: "tab", phase: "move", x: 25, y: 40, zoomFactor: 1.5, sequence: 1 }, + { tabId: "tab", phase: "click", x: 25, y: 40, zoomFactor: 1.5, sequence: 2 }, + ]); + const firstPress = currentWebContents!.debugger.commands.findIndex(({ method, params }) => + method === "Input.dispatchMouseEvent" && (params as { type?: string }).type === "mousePressed"); + expect(firstPress).toBeGreaterThanOrEqual(0); + expect(pointerEvents.every(({ commandsBefore }) => commandsBefore <= firstPress)).toBe(true); + }); + + it("forwards agent pointer events to the page only while its recording cursor is on", async () => { + currentWebContents!.semanticElements.set("recorded-button", { attached: true, visible: true, x: 30, y: 60 }); + const click = (requestId: string) => kernel.execute(event(), payload(request("click", { + target: { semanticId: "recorded-button" }, + button: "left", + clickCount: 1, + timeoutMs: 1_000, + }, { requestId }))); + const pointerSends = () => currentWebContents!.send.mock.calls.filter(([channel]) => channel === PREVIEW_GUEST_AGENT_POINTER_CHANNEL); + + await click("before-recording"); + expect(pointerSends()).toEqual([]); + + expect(kernel.setRecordingCursor(event(), { threadId: "thread", tabId: "tab", enabled: true })).toBe(true); + await click("while-recording"); + expect(pointerSends().map(([, pointer]) => pointer)).toEqual([ + { phase: "move", x: 30, y: 60 }, + { phase: "click", x: 30, y: 60 }, + ]); + + expect(kernel.setRecordingCursor(event(), { threadId: "thread", tabId: "tab", enabled: false })).toBe(true); + await click("after-recording"); + expect(pointerSends()).toHaveLength(2); }); it("scrolls an off-screen semantic target into view before clicking", async () => { diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts index 521e80e01..b8d39d77d 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts @@ -103,6 +103,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { stop = vi.fn(); send = vi.fn(); isDestroyed(): boolean { return this.destroyed; } + getZoomFactor(): number { return 1; } getURL(): string { return this.url; } getTitle(): string { return "Executor fixture"; } isLoading(): boolean { return false; } diff --git a/apps/desktop/src/features/preview/automation/index.ts b/apps/desktop/src/features/preview/automation/index.ts index 2f5b4b73e..386779132 100644 --- a/apps/desktop/src/features/preview/automation/index.ts +++ b/apps/desktop/src/features/preview/automation/index.ts @@ -22,16 +22,23 @@ export function registerBrowserAutomationHandlers(platform: NodeJS.Platform): vo ipcMain.handle("preview:automation.release-agent-control", (event, target: unknown) => automationKernel.releaseAgentControl(event, target)); ipcMain.handle("preview:automation.describe-target", (event, target: unknown) => automationKernel.describeTarget(event, target)); ipcMain.handle("preview:automation.media-source", (event, target: unknown) => automationKernel.getMediaSourceId(event, target)); + ipcMain.handle("preview:automation.recording-cursor", (event, target: unknown) => automationKernel.setRecordingCursor(event, target)); ipcMain.handle("preview:open-guest-devtools", (event, target: unknown) => automationKernel.openDevTools(event, target)); ipcMain.on("preview:automation.subscribe", (event: IpcMainEvent) => { const win = BrowserWindow.fromWebContents(event.sender); if (!win || win.isDestroyed()) return; if (subscribedRenderers.has(event.sender)) return; subscribedRenderers.add(event.sender); - const unsubscribe = automationKernel.subscribe(win.id, (state) => { + const unsubscribeController = automationKernel.subscribe(win.id, (state) => { if (!event.sender.isDestroyed()) event.sender.send("preview:automation.controller", state); }); - event.sender.once("destroyed", unsubscribe); + const unsubscribePointer = automationKernel.subscribePointer(win.id, (pointer) => { + if (!event.sender.isDestroyed()) event.sender.send("preview:automation.pointer", pointer); + }); + event.sender.once("destroyed", () => { + unsubscribeController(); + unsubscribePointer(); + }); }); ipcMain.on(BROWSER_AUTOMATION_HEARTBEAT_SUBSCRIBE_CHANNEL, (event: IpcMainEvent) => { heartbeatPulse.subscribe(event.sender); diff --git a/apps/desktop/src/features/preview/automation/kernel.ts b/apps/desktop/src/features/preview/automation/kernel.ts index a49716357..1dc4b0f73 100644 --- a/apps/desktop/src/features/preview/automation/kernel.ts +++ b/apps/desktop/src/features/preview/automation/kernel.ts @@ -24,11 +24,16 @@ import { type BrowserAutomationHostDispatchTarget, type BrowserAutomationHostDispatch, type BrowserAutomationControllerState, + type BrowserAutomationPointerEvent, } from "@mcode/contracts"; import { findAdoptedWebContentsForWindow } from "../surfaces/registry.js"; import { resolveActivePreviewWebContentsForWindow } from "../surfaces/active-web-contents.js"; import { loadPreviewGuestUrl } from "../navigation/guest-navigation.js"; -import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL } from "../contracts/guest-input.js"; +import { + PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, + type PreviewGuestAgentPointerMessage, +} from "../contracts/guest-input.js"; import { getSession, getThreadTabSet } from "../state/window-session.js"; import { BrowserAutomationCancelledError, @@ -616,6 +621,10 @@ export class BrowserAutomationKernel { private readonly targetGenerations = new Map(); private readonly cancellations = new Map void }>(); private readonly subscribers = new Map void>>(); + private readonly pointerSubscribers = new Map void>>(); + /** Target keys whose page is being recorded, so the guest must draw the agent cursor itself. */ + private readonly recordingCursorTargets = new Set(); + private pointerSequence = 0; private readonly rendererOperationLeases = new Map(); /** Validates and executes one request from a trusted renderer. */ @@ -974,6 +983,46 @@ export class BrowserAutomationKernel { }; } + /** Subscribes a renderer window to agent cursor points, reported before each pointer action. */ + subscribePointer(windowId: number, callback: (event: BrowserAutomationPointerEvent) => void): () => void { + const listeners = this.pointerSubscribers.get(windowId) ?? new Set(); + listeners.add(callback); + this.pointerSubscribers.set(windowId, listeners); + return () => { + listeners.delete(callback); + if (listeners.size === 0) this.pointerSubscribers.delete(windowId); + }; + } + + /** + * Turns the in-page agent cursor on or off for one tab while the renderer records it. + * Turning it off never requires a live tab, so a recording that outlives its tab still cleans up. + */ + setRecordingCursor(event: IpcMainInvokeEvent, input: unknown): boolean { + const value = asRecord(input); + let threadId: string; + let tabId: string; + try { + threadId = assertShortId(value.threadId, "threadId"); + tabId = assertShortId(value.tabId, "tabId"); + } catch { + return false; + } + if (typeof value.enabled !== "boolean") return false; + if (!value.enabled) { + const win = BrowserWindow.fromWebContents(event.sender); + if (!win) return false; + this.recordingCursorTargets.delete(targetKey(win.id, threadId, tabId)); + return true; + } + try { + this.recordingCursorTargets.add(this.resolveCurrentTarget(event, threadId, tabId).state.key); + return true; + } catch { + return false; + } + } + /** Releases target listeners, debugger sessions, queues, and buffers for one window. */ disposeWindow(windowId: number): void { for (const [key, state] of this.targets) { @@ -989,6 +1038,10 @@ export class BrowserAutomationKernel { if (key.startsWith(`[${windowId},`)) this.threadControls.delete(key); } this.subscribers.delete(windowId); + this.pointerSubscribers.delete(windowId); + for (const key of this.recordingCursorTargets) { + if (key.startsWith(`[${windowId},`)) this.recordingCursorTargets.delete(key); + } } /** Returns internal bounded counters for tests and performance diagnostics. */ @@ -1187,7 +1240,6 @@ export class BrowserAutomationKernel { state: TargetState, event: BrowserControlEvent, request?: BrowserAutomationRequest, - pointer?: { x: number; y: number }, ): boolean { const key = JSON.stringify([state.windowId, state.threadId]); const previous = this.threadControls.get(key) ?? createBrowserThreadControl(); @@ -1196,7 +1248,7 @@ export class BrowserAutomationKernel { this.threadControls.set(key, next); if (event.type === "agent-effect") { state.controlEpoch = next.epochByTab.get(state.tabId) ?? 0; - this.emitController(state, "agent", request, pointer); + this.emitController(state, "agent", request); } else { this.publishThreadControl(state, next); } @@ -1236,19 +1288,40 @@ export class BrowserAutomationKernel { } } - private admitAgentEffect(state: TargetState, request: BrowserAutomationRequest, pointer?: { x: number; y: number }): void { + private admitAgentEffect(state: TargetState, request: BrowserAutomationRequest): void { this.applyControlEvent(state, { type: "agent-effect", tabId: state.tabId, expectedEpoch: request.expectedControlEpoch, - }, request, pointer); + }, request); + } + + /** Reports where the agent is about to act; runs after admission so a held tab never shows a cursor. */ + private emitPointer(state: TargetState, phase: BrowserAutomationPointerEvent["phase"], point: { x: number; y: number }): void { + this.pointerSequence += 1; + const event: BrowserAutomationPointerEvent = { + tabId: state.tabId, + phase, + x: point.x, + y: point.y, + zoomFactor: state.webContents.getZoomFactor(), + sequence: this.pointerSequence, + }; + for (const listener of this.pointerSubscribers.get(state.windowId) ?? []) listener(event); + if (!this.recordingCursorTargets.has(state.key) || state.webContents.isDestroyed()) return; + const message: PreviewGuestAgentPointerMessage = { phase, x: point.x, y: point.y }; + state.webContents.send(PREVIEW_GUEST_AGENT_POINTER_CHANNEL, message); + } + + private emitPointerMoveThenClick(state: TargetState, point: { x: number; y: number }): void { + this.emitPointer(state, "move", point); + this.emitPointer(state, "click", point); } private emitController( state: TargetState, controller: "none" | "human" | "agent", request?: BrowserAutomationRequest, - pointer?: { x: number; y: number }, ): void { const payload: BrowserAutomationControllerState = { tabId: state.tabId, @@ -1262,7 +1335,6 @@ export class BrowserAutomationKernel { : {}), } : {}), - ...(pointer ? { pointer } : {}), }; state.controller = payload; for (const listener of this.subscribers.get(state.windowId) ?? []) listener(payload); @@ -1601,7 +1673,8 @@ export class BrowserAutomationKernel { ): Promise { const { state, webContents } = resolved; const point = await this.resolvePoint(resolved, request.args.target); - this.admitAgentEffect(state, request, point); + this.admitAgentEffect(state, request); + this.emitPointerMoveThenClick(state, point); await this.withSyntheticInput(state, async () => { await this.dispatchClick(state, webContents, point, request, signal, markEffect); this.throwIfAborted(signal); @@ -1667,7 +1740,8 @@ export class BrowserAutomationKernel { ): Promise { if (!request.args.target) return; const point = await this.resolvePoint(resolved, request.args.target); - this.admitAgentEffect(resolved.state, request, point); + this.admitAgentEffect(resolved.state, request); + this.emitPointerMoveThenClick(resolved.state, point); await this.withSyntheticInput(resolved.state, () => this.clickPoint(resolved.state, point, markEffect)); this.throwIfAborted(signal); } @@ -1714,6 +1788,8 @@ export class BrowserAutomationKernel { ): Promise { const { state, webContents } = resolved; const point = request.args.target ? await this.resolvePoint(resolved, request.args.target) : { x: 1, y: 1 }; + // An untargeted scroll has no point the user would recognise, so the cursor stays put. + if (request.args.target) this.emitPointer(state, "move", point); await this.withSyntheticInput(state, async () => { await this.ensureDebugger(state); await this.dispatchGuestInput( diff --git a/apps/desktop/src/features/preview/contracts/guest-input.ts b/apps/desktop/src/features/preview/contracts/guest-input.ts index c28a73164..8ff58f64c 100644 --- a/apps/desktop/src/features/preview/contracts/guest-input.ts +++ b/apps/desktop/src/features/preview/contracts/guest-input.ts @@ -16,6 +16,19 @@ export const PREVIEW_GUEST_CLIPBOARD_TRUST_CHANNEL = "mcode:browser-clipboard-tr /** Guest preload channel that reserves trusted events generated by agent CDP input. */ export const PREVIEW_GUEST_AGENT_INPUT_CHANNEL = "mcode:browser-agent-input"; +/** + * Guest preload channel that draws the agent cursor inside the page. The host only sends it while + * the tab is being recorded, because a tab capture sees the page and not the overlay above it. + */ +export const PREVIEW_GUEST_AGENT_POINTER_CHANNEL = "mcode:browser-agent-pointer"; + +/** Agent cursor point in viewport CSS pixels, sent to the guest preload before the agent acts. */ +export interface PreviewGuestAgentPointerMessage { + readonly phase: "move" | "click"; + readonly x: number; + readonly y: number; +} + /** Trusted guest interaction categories that transfer control from an agent to the human. */ export const PREVIEW_GUEST_HUMAN_INPUT_KINDS = [ "keyboard", diff --git a/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts b/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts new file mode 100644 index 000000000..df294c8dc --- /dev/null +++ b/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment jsdom +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createGuestAgentCursor, GUEST_AGENT_CURSOR_IDLE_MS } from "../agent-cursor.js"; + +const mark = (): HTMLElement | null => document.querySelector("mcode-agent-cursor"); + +describe("guest agent cursor", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + document.documentElement.innerHTML = ""; + }); + + it("draws an inert mark at the agent's viewport point", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "move", x: 120, y: 48 }); + + const element = mark(); + expect(element?.getAttribute("aria-hidden")).toBe("true"); + expect(element?.style.pointerEvents).toBe("none"); + expect(element?.style.transform).toBe("translate(120px, 48px)"); + }); + + it("pings once per click, not per move", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "move", x: 10, y: 10 }); + expect(cursor.pingCount()).toBe(0); + cursor.show({ phase: "click", x: 10, y: 10 }); + cursor.show({ phase: "move", x: 30, y: 30 }); + cursor.show({ phase: "click", x: 30, y: 30 }); + + expect(cursor.pingCount()).toBe(2); + }); + + it("removes the mark once the agent goes idle so nothing keeps animating", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "click", x: 10, y: 10 }); + vi.advanceTimersByTime(GUEST_AGENT_CURSOR_IDLE_MS - 1); + cursor.show({ phase: "move", x: 20, y: 20 }); + vi.advanceTimersByTime(GUEST_AGENT_CURSOR_IDLE_MS - 1); + expect(mark()).not.toBeNull(); + + vi.advanceTimersByTime(1_000); + expect(mark()).toBeNull(); + }); + + it("ignores malformed host messages", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "drag", x: 1, y: 1 }); + cursor.show({ phase: "move", x: Number.NaN, y: 1 }); + cursor.show(null); + + expect(mark()).toBeNull(); + }); +}); diff --git a/apps/desktop/src/features/preview/preload/agent-cursor.ts b/apps/desktop/src/features/preview/preload/agent-cursor.ts new file mode 100644 index 000000000..0cc58b0fd --- /dev/null +++ b/apps/desktop/src/features/preview/preload/agent-cursor.ts @@ -0,0 +1,153 @@ +import type { PreviewGuestAgentPointerMessage } from "../contracts/guest-input.js"; + +/** Time after the last agent point before the in-page mark fades out and is removed. */ +export const GUEST_AGENT_CURSOR_IDLE_MS = 700; + +const FADE_MS = 150; +const MAX_VIEWPORT_COORDINATE = 100_000; +// Matches --amber-500, the dark-theme --primary, so recordings show the same mark as the host. +const MARK_COLOR = "oklch(72% 0.170 75)"; +// The pointer tip sits at (4, 4) in the 24-unit icon box; shifting by it puts the tip on the point. +const TIP_OFFSET_PX = 3.3; + +const SHADOW_STYLE = ` + svg { + position: absolute; + left: ${-TIP_OFFSET_PX}px; + top: ${-TIP_OFFSET_PX}px; + width: 20px; + height: 20px; + fill: ${MARK_COLOR}; + stroke: ${MARK_COLOR}; + stroke-width: 2; + stroke-linejoin: round; + filter: drop-shadow(0 0 4px rgb(0 0 0 / 0.45)); + } + .ping { + position: absolute; + left: -10px; + top: -10px; + width: 20px; + height: 20px; + border-radius: 50%; + border: 2px solid ${MARK_COLOR}; + box-sizing: border-box; + animation: ping 450ms ease-out forwards; + } + @keyframes ping { + from { opacity: 0.9; transform: scale(0.4); } + to { opacity: 0; transform: scale(1.8); } + } + @media (prefers-reduced-motion: reduce) { + .ping { animation: none; opacity: 0; } + } +`; + +const POINTER_PATH = + "M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"; + +/** In-page agent cursor drawn by the guest preload so tab recordings include it. */ +export interface GuestAgentCursor { + /** Moves the mark to one host-reported point; malformed messages are ignored. */ + show(message: unknown): void; + /** Number of click pings drawn since creation. */ + pingCount(): number; +} + +interface MountedMark { + readonly host: HTMLElement; + readonly root: ShadowRoot; +} + +/** + * Creates the in-page agent cursor. The mark is inert: it never takes input, is hidden from + * assistive technology, and is removed once the agent goes idle so no animation keeps running. + */ +export function createGuestAgentCursor(document: Document): GuestAgentCursor { + let mounted: MountedMark | null = null; + let idleTimer: ReturnType | null = null; + let removeTimer: ReturnType | null = null; + let pings = 0; + + const unmount = () => { + mounted?.host.remove(); + mounted = null; + }; + + const scheduleIdle = () => { + if (idleTimer) clearTimeout(idleTimer); + if (removeTimer) clearTimeout(removeTimer); + removeTimer = null; + idleTimer = setTimeout(() => { + idleTimer = null; + mounted?.host.style.setProperty("opacity", "0", "important"); + removeTimer = setTimeout(unmount, FADE_MS); + }, GUEST_AGENT_CURSOR_IDLE_MS); + }; + + return { + show(message) { + const pointer = parsePointerMessage(message); + if (!pointer) return; + const mark = mounted?.host.isConnected ? mounted : mountMark(document); + mounted = mark; + mark.host.style.setProperty("opacity", "1", "important"); + mark.host.style.setProperty("transform", `translate(${pointer.x}px, ${pointer.y}px)`, "important"); + if (pointer.phase === "click") { + pings += 1; + drawPing(mark.root); + } + scheduleIdle(); + }, + pingCount: () => pings, + }; +} + +function mountMark(document: Document): MountedMark { + const host = document.createElement("mcode-agent-cursor"); + host.setAttribute("aria-hidden", "true"); + // Inline !important keeps page stylesheets from moving, hiding, or making the mark clickable. + for (const [property, value] of [ + ["all", "initial"], + ["position", "fixed"], + ["left", "0"], + ["top", "0"], + ["width", "0"], + ["height", "0"], + ["pointer-events", "none"], + ["z-index", "2147483647"], + ["transition", `transform ${FADE_MS}ms ease-out, opacity ${FADE_MS}ms ease-out`], + ] as const) { + host.style.setProperty(property, value, "important"); + } + const root = host.attachShadow({ mode: "closed" }); + const style = document.createElement("style"); + style.textContent = SHADOW_STYLE; + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("viewBox", "0 0 24 24"); + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", POINTER_PATH); + svg.append(path); + root.append(style, svg); + document.documentElement.append(host); + return { host, root }; +} + +function drawPing(root: ShadowRoot): void { + root.querySelector(".ping")?.remove(); + const ping = root.ownerDocument.createElement("span"); + ping.className = "ping"; + root.append(ping); +} + +function parsePointerMessage(message: unknown): PreviewGuestAgentPointerMessage | null { + if (!message || typeof message !== "object") return null; + const { phase, x, y } = message as Partial>; + if (phase !== "move" && phase !== "click") return null; + if (!isViewportCoordinate(x) || !isViewportCoordinate(y)) return null; + return { phase, x, y }; +} + +function isViewportCoordinate(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= MAX_VIEWPORT_COORDINATE; +} diff --git a/apps/desktop/src/features/preview/preload/guest-input.ts b/apps/desktop/src/features/preview/preload/guest-input.ts index d688fcfc8..15b9a503a 100644 --- a/apps/desktop/src/features/preview/preload/guest-input.ts +++ b/apps/desktop/src/features/preview/preload/guest-input.ts @@ -1,12 +1,14 @@ import { ipcRenderer } from "electron"; import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, PREVIEW_GUEST_CLIPBOARD_TRUST_CHANNEL, PREVIEW_GUEST_HUMAN_INPUT_CHANNEL, PREVIEW_GUEST_SCROLL_CHANNEL, PreviewGuestInputSuppressor, toPreviewGuestHumanInputMessage, } from "../contracts/guest-input.js"; +import { createGuestAgentCursor, type GuestAgentCursor } from "./agent-cursor.js"; const suppressor = new PreviewGuestInputSuppressor(); @@ -22,6 +24,14 @@ ipcRenderer.on(PREVIEW_GUEST_AGENT_INPUT_CHANNEL, (_event, input: unknown) => { } }); +// Created on the first point: the host only sends points while recording, so most pages never pay for it. +let agentCursor: GuestAgentCursor | null = null; + +ipcRenderer.on(PREVIEW_GUEST_AGENT_POINTER_CHANNEL, (_event, pointer: unknown) => { + agentCursor ??= createGuestAgentCursor(document); + agentCursor.show(pointer); +}); + for (const eventType of ["keydown", "pointerdown", "touchstart", "wheel"] as const) { window.addEventListener( eventType, diff --git a/apps/desktop/src/main/preload.ts b/apps/desktop/src/main/preload.ts index 95facb9df..d6d9623ca 100644 --- a/apps/desktop/src/main/preload.ts +++ b/apps/desktop/src/main/preload.ts @@ -517,12 +517,21 @@ contextBridge.exposeInMainWorld("desktopBridge", { }): Promise { return ipcRenderer.invoke("preview:automation.media-source", target); }, + setRecordingCursor(target: { threadId: string; tabId: string; enabled: boolean }): Promise { + return ipcRenderer.invoke("preview:automation.recording-cursor", target); + }, onControllerChanged(callback: (state: unknown) => void): () => void { const listener = (_event: unknown, state: unknown) => callback(state); ipcRenderer.send("preview:automation.subscribe"); ipcRenderer.on("preview:automation.controller", listener); return () => ipcRenderer.removeListener("preview:automation.controller", listener); }, + onPointer(callback: (pointer: unknown) => void): () => void { + const listener = (_event: unknown, pointer: unknown) => callback(pointer); + ipcRenderer.send("preview:automation.subscribe"); + ipcRenderer.on("preview:automation.pointer", listener); + return () => ipcRenderer.removeListener("preview:automation.pointer", listener); + }, onHostHeartbeat(callback: () => void): () => void { const listener = () => callback(); ipcRenderer.on("preview:automation.heartbeat", listener); diff --git a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx index f194fbf6e..2f7c1512e 100644 --- a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx +++ b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx @@ -1511,6 +1511,12 @@ export function BrowserAutomationHost() { sessionDriverRef.current?.invalidateTargetObservations(workspaceId, threadId, tabId); }), []); + useEffect(() => { + const bridge = window.desktopBridge?.preview?.automation; + if (!bridge) return; + return bridge.onPointer((pointer) => useBrowserAutomationStore.getState().setPointer(pointer)); + }, []); + useEffect(() => { const bridge = window.desktopBridge?.preview?.automation; if (!bridge) return; diff --git a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx index 9b53fc101..fd224dfbf 100644 --- a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx +++ b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx @@ -273,6 +273,7 @@ describe("BrowserAutomationHost", () => { if (controllerChanged === callback) controllerChanged = null; }; }, + onPointer: () => () => {}, onHostHeartbeat(callback: () => void) { hostHeartbeat = callback; return () => { diff --git a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts index e85669abe..05c66a416 100644 --- a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts +++ b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts @@ -69,7 +69,9 @@ describe("BrowserAutomationRecorder", () => { const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream; const getUserMedia = vi.fn(); const getMediaSourceId = vi.fn(); - const bridge = { getMediaSourceId } as unknown as PreviewAutomationBridge; + const setRecordingCursor = vi.fn(); + const bridge = { getMediaSourceId, setRecordingCursor } as unknown as PreviewAutomationBridge; + const cursorStates = () => setRecordingCursor.mock.calls.map(([target]) => (target as { enabled: boolean }).enabled); beforeEach(() => { vi.clearAllMocks(); @@ -206,6 +208,25 @@ describe("BrowserAutomationRecorder", () => { } }); + it("draws the page cursor only while a recording runs, however the recording ends", async () => { + const recorder = new BrowserAutomationRecorder(); + await recorder.start(dispatch("recordingStart", 1), bridge); + expect(setRecordingCursor).toHaveBeenLastCalledWith({ threadId: "thread", tabId: "tab", enabled: true }); + FakeMediaRecorder.instances[0]!.emit(400 * 1_024); + await Promise.resolve(); + expect(cursorStates()).toEqual([true, false]); + await recorder.stop(dispatch("recordingStop", 2)); + expect(cursorStates()).toEqual([true, false]); + + vi.useFakeTimers(); + FakeMediaRecorder.stopEmits = false; + await recorder.start(dispatch("recordingStart", 3), bridge); + const stopping = recorder.stop(dispatch("recordingStop", 4)); + await vi.advanceTimersByTimeAsync(1_000); + await stopping; + expect(cursorStates()).toEqual([true, false, true, false]); + }); + it("fails a never-resolving stop within the cleanup timeout and stops tracks", async () => { vi.useFakeTimers(); FakeMediaRecorder.stopEmits = false; diff --git a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts index 5380dab53..6f90bc179 100644 --- a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts +++ b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts @@ -42,6 +42,8 @@ interface RecordingSession { stopped: Promise; stopTimer: number; error: string | null; + /** Turns the in-page agent cursor off; tab capture cannot see the host overlay, so the page draws it. */ + readonly releaseCursor: () => void; } interface RecordingSessionWithResolver extends RecordingSession { @@ -197,10 +199,13 @@ export class BrowserAutomationRecorder { private createSession( dispatch: BrowserAutomationHostDispatch, + bridge: PreviewAutomationBridge, key: string, recorder: MediaRecorder, stream: MediaStream, ): RecordingSessionWithResolver { + const { threadId, tabId } = dispatch.target; + let cursorReleased = false; let resolveStopped!: () => void; const stopped = new Promise((resolve) => { resolveStopped = resolve; @@ -219,6 +224,11 @@ export class BrowserAutomationRecorder { stopped, stopTimer: 0, error: null, + releaseCursor: () => { + if (cursorReleased) return; + cursorReleased = true; + void bridge.setRecordingCursor({ threadId, tabId, enabled: false }); + }, resolveStopped, }; } @@ -239,6 +249,7 @@ export class BrowserAutomationRecorder { session.retainedBytes += event.data.size; }; recorder.onstop = () => { + session.releaseCursor(); window.clearTimeout(session.stopTimer); this.stopStream(stream); this.clearPendingAcquisition(key, acquisition); @@ -246,6 +257,7 @@ export class BrowserAutomationRecorder { }; recorder.onerror = () => { session.error = "The browser recording encoder failed"; + session.releaseCursor(); window.clearTimeout(session.stopTimer); if (recorder.state !== "inactive") recorder.stop(); this.stopStream(stream); @@ -316,9 +328,10 @@ export class BrowserAutomationRecorder { } const recorder = this.createRecorder(dispatch, stream.value, key, acquisition); if (!recorder.ok) return recorder.response; - const session = this.configureRecorder(this.createSession(dispatch, key, recorder.value, stream.value), key, acquisition); + const session = this.configureRecorder(this.createSession(dispatch, bridge, key, recorder.value, stream.value), key, acquisition); const startFailure = this.startRecorder(dispatch, session, key, acquisition); if (startFailure) return startFailure; + void bridge.setRecordingCursor({ threadId: dispatch.target.threadId, tabId: dispatch.target.tabId, enabled: true }); const requestedDuration = dispatch.request.operation === "recordingStart" ? dispatch.request.args.maxDurationMs : MAX_RECORDING_DURATION_MS; @@ -356,6 +369,7 @@ export class BrowserAutomationRecorder { ]); window.clearTimeout(stopTimeout); this.sessions.delete(key); + session.releaseCursor(); if (stopTimedOut) return failure(dispatch, "The browser recording encoder did not stop"); if (session.error) return failure(dispatch, session.error); const retained = new Uint8Array(await new Blob(session.chunks, { type: "video/webm" }).arrayBuffer()); @@ -398,6 +412,7 @@ export class BrowserAutomationRecorder { const session = this.sessions.get(key); if (!session) return; this.sessions.delete(session.targetKey); + session.releaseCursor(); window.clearTimeout(session.stopTimer); if (session.recorder.state !== "inactive") session.recorder.stop(); for (const track of session.stream.getTracks()) track.stop(); diff --git a/apps/web/src/features/preview/automation/browserAutomationStore.ts b/apps/web/src/features/preview/automation/browserAutomationStore.ts index f8fcd67d3..e9249cb3a 100644 --- a/apps/web/src/features/preview/automation/browserAutomationStore.ts +++ b/apps/web/src/features/preview/automation/browserAutomationStore.ts @@ -1,6 +1,7 @@ import type { BrowserAutomationControllerState, BrowserAutomationHostDispatch, + BrowserAutomationPointerEvent, } from "@mcode/contracts"; import { BROWSER_AUTOMATION_MAX_INSPECT_TABS, BROWSER_AUTOMATION_MAX_PENDING_REQUESTS } from "@mcode/contracts"; import { create } from "zustand"; @@ -48,6 +49,8 @@ interface BrowserAutomationState { readonly liveTargets: ReadonlyMap; readonly lifecycleTabs: ReadonlyMap; readonly controllers: ReadonlyMap; + /** Latest agent cursor point per target, in the page's viewport CSS pixels. */ + readonly pointers: ReadonlyMap; readonly activeRequests: ReadonlyMap; readonly pendingAgentOpens: ReadonlyMap; /** Targets whose agent has a Browser effect running or finished within {@link BROWSER_ACTING_LINGER_MS}. */ @@ -66,6 +69,7 @@ interface BrowserAutomationState { releaseThreadTargets: (workspaceId: string, threadId: string) => void; releaseWorkspaceTargets: (workspaceId: string) => void; setController: (state: BrowserAutomationControllerState) => void; + setPointer: (pointer: BrowserAutomationPointerEvent) => void; setControllerForTarget: ( workspaceId: string, threadId: string, @@ -202,10 +206,10 @@ export function isBrowserAutomationAgentControlled( ) !== null; } -/** Resolve a controller event only when its tab id maps to one exact live target. */ -export function resolveBrowserAutomationControllerTarget( +/** Resolve a controller or pointer event only when its tab id maps to one exact live target. */ +export function resolveBrowserAutomationControllerTarget( targets: Iterable, - controller: BrowserAutomationControllerState, + controller: TEvent, ): BrowserAutomationLiveTarget | null { const candidates = [...targets].filter((candidate) => candidate.tabId === controller.tabId); return candidates.length === 1 ? candidates[0]! : null; @@ -255,6 +259,7 @@ export const useBrowserAutomationStore = create((set, ge liveTargets: new Map(), lifecycleTabs: new Map(), controllers: new Map(), + pointers: new Map(), activeRequests: new Map(), pendingAgentOpens: new Map(), actingTargets: new Set(), @@ -317,6 +322,8 @@ export const useBrowserAutomationStore = create((set, ge liveTargets.delete(key); const controllers = new Map(state.controllers); controllers.delete(key); + const pointers = new Map(state.pointers); + pointers.delete(key); const viewportCoordinators = new Map(state.viewportCoordinators); viewportCoordinators.delete(key); const viewportState = coordinator?.snapshot(); @@ -328,7 +335,7 @@ export const useBrowserAutomationStore = create((set, ge viewportByTarget.set(key, viewportState.confirmed); } } - return { liveTargets, controllers, viewportByTarget, viewportStateByTarget, viewportCoordinators }; + return { liveTargets, controllers, pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators }; }); }, unregisterTarget: (workspaceId, threadId, tabId) => { @@ -341,6 +348,8 @@ export const useBrowserAutomationStore = create((set, ge liveTargets.delete(key); const controllers = new Map(state.controllers); controllers.delete(key); + const pointers = new Map(state.pointers); + pointers.delete(key); const viewportByTarget = new Map(state.viewportByTarget); viewportByTarget.delete(key); const viewportStateByTarget = new Map(state.viewportStateByTarget); @@ -358,6 +367,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -390,6 +400,7 @@ export const useBrowserAutomationStore = create((set, ge set((state) => { const liveTargets = new Map(state.liveTargets); const controllers = new Map(state.controllers); + const pointers = new Map(state.pointers); const viewportByTarget = new Map(state.viewportByTarget); const viewportStateByTarget = new Map(state.viewportStateByTarget); const viewportCoordinators = new Map(state.viewportCoordinators); @@ -399,6 +410,7 @@ export const useBrowserAutomationStore = create((set, ge const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); liveTargets.delete(key); controllers.delete(key); + pointers.delete(key); viewportByTarget.delete(key); viewportStateByTarget.delete(key); viewportCoordinators.delete(key); @@ -409,6 +421,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -429,6 +442,7 @@ export const useBrowserAutomationStore = create((set, ge set((state) => { const liveTargets = new Map(state.liveTargets); const controllers = new Map(state.controllers); + const pointers = new Map(state.pointers); const viewportByTarget = new Map(state.viewportByTarget); const viewportStateByTarget = new Map(state.viewportStateByTarget); const viewportCoordinators = new Map(state.viewportCoordinators); @@ -438,6 +452,7 @@ export const useBrowserAutomationStore = create((set, ge const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); liveTargets.delete(key); controllers.delete(key); + pointers.delete(key); viewportByTarget.delete(key); viewportStateByTarget.delete(key); viewportCoordinators.delete(key); @@ -448,6 +463,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -460,6 +476,17 @@ export const useBrowserAutomationStore = create((set, ge if (!target) return; get().setControllerForTarget(target.workspaceId, target.threadId, target.tabId, controller); }, + setPointer: (pointer) => { + const target = resolveBrowserAutomationControllerTarget(get().liveTargets.values(), pointer); + if (!target) return; + set((state) => { + const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); + const current = state.pointers.get(key); + // IPC delivers in order today; the guard keeps a late duplicate from rewinding the mark. + if (current && current.sequence >= pointer.sequence) return state; + return { pointers: new Map(state.pointers).set(key, pointer) }; + }); + }, setControllerForTarget: (workspaceId, threadId, tabId, controller) => set((state) => { const key = browserAutomationTargetKey(workspaceId, threadId, tabId); diff --git a/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx b/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx index b6f9a13af..d70792ece 100644 --- a/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx +++ b/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx @@ -1,4 +1,6 @@ +import type { BrowserAutomationPointerEvent } from "@mcode/contracts"; import { MousePointer2 } from "lucide-react"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; import { browserAutomationTargetKey, useBrowserAutomationStore } from "../automation/browserAutomationStore"; import { useBrowserActing } from "../state/useBrowserActing"; @@ -7,6 +9,114 @@ const EDGE_BOX_SHADOW = [ "inset 0 0 28px color-mix(in oklab, var(--primary) 35%, transparent)", ].join(", "); +/** How long the agent cursor stays fully opaque after the agent's last point before it dims. */ +export const AGENT_CURSOR_ACTIVE_MS = 700; + +const AGENT_CURSOR_IDLE_OPACITY = 0.35; +// The pointer tip sits at (4, 4) in the 24-unit icon box; at 20px that is 3.3px in from the corner. +const AGENT_CURSOR_TIP_OFFSET_PX = 3.3; + +/** Where the hosted page is drawn inside the overlay: its top-left corner and its drawn-to-layout size ratio. */ +export interface AgentCursorPagePlacement { + readonly left: number; + readonly top: number; + readonly scale: number; +} + +/** + * Maps an agent point from page CSS pixels to overlay pixels. Page zoom enlarges CSS pixels + * inside the page, and a responsive viewport draws the whole page smaller or larger. + */ +export function mapAgentCursorPoint( + point: Pick, + page: AgentCursorPagePlacement, +): { readonly x: number; readonly y: number } { + return { + x: page.left + point.x * point.zoomFactor * page.scale, + y: page.top + point.y * point.zoomFactor * page.scale, + }; +} + +/** Measures the page against `origin`, an element covering the overlay, whose surface sits beside the overlay. */ +function measurePagePlacement(origin: HTMLElement, threadId: string, tabId: string): AgentCursorPagePlacement { + const surface = origin.closest('[data-testid="browser-acting-overlay"]')?.parentElement; + const page = [...(surface?.querySelectorAll('[data-testid="preview-webview"]') ?? [])] + .find((candidate) => candidate.dataset.threadId === threadId && candidate.dataset.tabId === tabId); + if (!page) return { left: 0, top: 0, scale: 1 }; + const overlayRect = origin.getBoundingClientRect(); + const pageRect = page.getBoundingClientRect(); + // offsetWidth ignores CSS transforms, so the ratio is the responsive viewport's scale. + const scale = page.offsetWidth > 0 ? pageRect.width / page.offsetWidth : 1; + return { left: pageRect.left - overlayRect.left, top: pageRect.top - overlayRect.top, scale }; +} + +interface AgentCursorProps { + readonly threadId: string; + readonly tabId: string; + readonly pointer: BrowserAutomationPointerEvent; +} + +/** Presentation-only mark at the agent's latest point; it never takes input or moves the OS pointer. */ +function AgentCursor({ threadId, tabId, pointer }: AgentCursorProps) { + const originRef = useRef(null); + const [position, setPosition] = useState<{ readonly x: number; readonly y: number } | null>(null); + const [idleSequence, setIdleSequence] = useState(null); + + useLayoutEffect(() => { + const origin = originRef.current; + if (!origin) return; + setPosition(mapAgentCursorPoint(pointer, measurePagePlacement(origin, threadId, tabId))); + }, [pointer, threadId, tabId]); + + useEffect(() => { + const timer = window.setTimeout(() => setIdleSequence(pointer.sequence), AGENT_CURSOR_ACTIVE_MS); + return () => window.clearTimeout(timer); + }, [pointer.sequence]); + + return ( +
+ {position && } +
+ ); +} + +function AgentCursorMark({ + position, + active, + pointer, +}: { + readonly position: { readonly x: number; readonly y: number }; + readonly active: boolean; + readonly pointer: BrowserAutomationPointerEvent; +}) { + return ( +
+ {pointer.phase === "click" && ( + + )} + +
+ ); +} + interface BrowserActingOverlayProps { readonly workspaceId: string; readonly threadId: string; @@ -15,13 +125,13 @@ interface BrowserActingOverlayProps { } /** - * Click-through edge wash and pointer shown over the page while the agent acts on the active + * Click-through edge wash and agent cursor shown over the page while the agent acts on the active * tab. It renders above the hosted page, which paints from the detached surface host root. */ export function BrowserActingOverlay({ workspaceId, threadId, tabId, coveredRight }: BrowserActingOverlayProps) { const acting = useBrowserActing(workspaceId, threadId, tabId); const pointer = useBrowserAutomationStore( - (state) => state.controllers.get(browserAutomationTargetKey(workspaceId, threadId, tabId))?.pointer ?? null, + (state) => state.pointers.get(browserAutomationTargetKey(workspaceId, threadId, tabId)) ?? null, ); if (!acting) return null; return ( @@ -36,16 +146,7 @@ export function BrowserActingOverlay({ workspaceId, threadId, tabId, coveredRigh Agent controls Browser - + {pointer && } ); } diff --git a/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx b/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx new file mode 100644 index 000000000..178f14aaa --- /dev/null +++ b/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx @@ -0,0 +1,92 @@ +import { act, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { BrowserAutomationControllerState, BrowserAutomationPointerEvent } from "@mcode/contracts"; +import { browserAutomationTargetKey, useBrowserAutomationStore } from "../../automation/browserAutomationStore"; +import { AGENT_CURSOR_ACTIVE_MS, BrowserActingOverlay, mapAgentCursorPoint } from "../BrowserActingOverlay"; + +const WORKSPACE = "workspace"; +const THREAD = "thread"; +const TAB = "tab"; +const KEY = browserAutomationTargetKey(WORKSPACE, THREAD, TAB); + +function pointer(phase: BrowserAutomationPointerEvent["phase"], sequence: number, zoomFactor = 1): BrowserAutomationPointerEvent { + return { tabId: TAB, phase, x: 100, y: 40, zoomFactor, sequence }; +} + +function setAgentState(controller: BrowserAutomationControllerState["controller"], point: BrowserAutomationPointerEvent | null) { + useBrowserAutomationStore.setState({ + controllers: new Map([[KEY, { tabId: TAB, controller, controlEpoch: 1 }]]), + actingTargets: new Set([KEY]), + pointers: new Map(point ? [[KEY, point]] : []), + }); +} + +/** Renders the overlay beside a page placed at (12, 8) and drawn at half size, as a responsive viewport does. */ +function renderOverPlacedPage() { + const view = render( +
+
+ +
, + ); + const page = view.getByTestId("preview-webview"); + Object.defineProperty(page, "offsetWidth", { configurable: true, value: 800 }); + page.getBoundingClientRect = () => new DOMRect(12, 8, 400, 300); + return view; +} + +describe("mapAgentCursorPoint", () => { + it("scales a page CSS point by page zoom and the drawn page size, then offsets it by the page position", () => { + expect(mapAgentCursorPoint({ x: 100, y: 40, zoomFactor: 1.5 }, { left: 12, top: 8, scale: 0.5 })).toEqual({ x: 87, y: 38 }); + }); +}); + +describe("BrowserActingOverlay agent cursor", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + useBrowserAutomationStore.setState({ controllers: new Map(), actingTargets: new Set(), pointers: new Map() }); + }); + + it("lands on the agent's point through page zoom and the responsive page scale", () => { + setAgentState("agent", null); + renderOverPlacedPage(); + expect(screen.queryByTestId("browser-automation-pointer")).not.toBeInTheDocument(); + + act(() => setAgentState("agent", pointer("move", 1, 1.5))); + + expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ transform: "translate3d(87px, 38px, 0)" }); + }); + + it("pings on a click but not on a move", () => { + setAgentState("agent", pointer("move", 1)); + renderOverPlacedPage(); + expect(screen.queryByTestId("browser-automation-pointer-ping")).not.toBeInTheDocument(); + + act(() => setAgentState("agent", pointer("click", 2))); + + expect(screen.getByTestId("browser-automation-pointer-ping")).toBeInTheDocument(); + }); + + it("dims once the agent stops moving", () => { + setAgentState("agent", pointer("move", 1)); + renderOverPlacedPage(); + expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ opacity: "1" }); + + act(() => { + vi.advanceTimersByTime(AGENT_CURSOR_ACTIVE_MS); + }); + + expect(screen.getByTestId("browser-automation-pointer")).not.toHaveStyle({ opacity: "1" }); + }); + + it("hides the cursor while the user holds control", () => { + setAgentState("human", pointer("click", 1)); + renderOverPlacedPage(); + + expect(screen.queryByTestId("browser-automation-pointer")).not.toBeInTheDocument(); + }); +}); diff --git a/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx b/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx index 82c3d0b52..7da2ba0ee 100644 --- a/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx +++ b/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx @@ -356,13 +356,12 @@ describe("PreviewPanel: unavailable state", () => { const key = browserAutomationTargetKey(threadId, threadId, "web-preview"); useDiffStore.setState({ previewUrlByThread: { [threadId]: window.location.origin + "/fixture" } }); useBrowserAutomationStore.setState({ - controllers: new Map([[key, { tabId: "web-preview", controller: "agent", controlEpoch: 1, pointer: { x: 40, y: 50 } }]]), + controllers: new Map([[key, { tabId: "web-preview", controller: "agent", controlEpoch: 1 }]]), actingTargets: new Set([key]), }); try { render(); expect(screen.getByTestId("browser-acting-overlay")).toHaveStyle({ clipPath: "inset(0 112px 0 0)" }); - expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ left: "40px", top: "50px" }); } finally { useBrowserAutomationStore.setState({ controllers: new Map(), actingTargets: new Set() }); } @@ -836,7 +835,7 @@ describe("PreviewPanel: full panel state", () => { expect(overlay).toHaveStyle({ clipPath: "inset(0 112px 0 0)" }); }); - it("shows the agent frame and cursor while the active page is still opening", () => { + it("shows the agent frame while the active page is still opening", () => { useBrowserAutomationStore.setState({ pendingAgentOpens: new Map([ [ @@ -855,7 +854,6 @@ describe("PreviewPanel: full panel state", () => { render(); expect(screen.getByTestId("browser-acting-overlay")).toBeInTheDocument(); - expect(screen.getByTestId("browser-automation-pointer")).toBeInTheDocument(); }); it("paints no agent overlay while the agent holds the page but is idle", () => { diff --git a/apps/web/src/index.css b/apps/web/src/index.css index 0ee7c0c87..2e311b26b 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -686,6 +686,21 @@ diffs-container:has([data-jump-highlight="true"]) { } } +/* One ring per agent click; it ends transparent so nothing keeps animating while the agent is idle. */ +@keyframes agent-cursor-ping { + from { opacity: 0.9; transform: scale(0.4); } + to { opacity: 0; transform: scale(1.8); } +} +.animate-agent-cursor-ping { + animation: agent-cursor-ping 450ms ease-out forwards; +} +@media (prefers-reduced-motion: reduce) { + .animate-agent-cursor-ping { + animation: none; + opacity: 0; + } +} + /* Tailwind's looping utilities (loading dots, spinners) are not covered by the named-animation blocks above, so reduced-motion must settle them explicitly. Freezing leaves a static-but-visible loading affordance. */ diff --git a/apps/web/src/transport/desktop-bridge.d.ts b/apps/web/src/transport/desktop-bridge.d.ts index 2f62e1252..1b8943fa0 100644 --- a/apps/web/src/transport/desktop-bridge.d.ts +++ b/apps/web/src/transport/desktop-bridge.d.ts @@ -1,6 +1,7 @@ import type { AttachmentMeta, OpenInApp } from "./types"; import type { BrowserAutomationControllerState, + BrowserAutomationPointerEvent, BrowserHistory, BrowserCertificateState, BrowserAutomationHostDispatch, @@ -310,6 +311,10 @@ export interface PreviewAutomationBridge { } | { ok: false; error: string } >; + /** Draws the agent cursor inside one tab's page while the renderer records it; false when the tab is unavailable. */ + setRecordingCursor(target: { threadId: string; tabId: string; enabled: boolean }): Promise; + /** Subscribe to agent cursor points, reported before each agent pointer action. */ + onPointer(callback: (pointer: BrowserAutomationPointerEvent) => void): () => void; /** Subscribe to controller transitions for adopted tabs in this window. */ onControllerChanged(callback: (state: BrowserAutomationControllerState) => void): () => void; /** Subscribe to Browser host heartbeat pulses owned by Electron main. */ diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 1bfbbb5ac..7319abe0d 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -742,6 +742,7 @@ export { BrowserAutomationSnapshotSchema, BrowserAutomationPerformanceMetricsSchema, BrowserAutomationControllerStateSchema, + BrowserAutomationPointerEventSchema, BrowserAutomationHostCapabilitySchema, BrowserAutomationHostRegistrationSchema, BrowserAutomationHostDispatchTargetSchema, @@ -776,6 +777,7 @@ export type { BrowserAutomationTruncation, BrowserAutomationPerformanceMetrics, BrowserAutomationControllerState, + BrowserAutomationPointerEvent, BrowserAutomationHostRegistration, BrowserAutomationHostDispatchTarget, BrowserAutomationHostDispatch, diff --git a/packages/contracts/src/models/browser-automation.ts b/packages/contracts/src/models/browser-automation.ts index e590f566e..c3ad4072d 100644 --- a/packages/contracts/src/models/browser-automation.ts +++ b/packages/contracts/src/models/browser-automation.ts @@ -622,10 +622,6 @@ export const BrowserAutomationControllerStateSchema = lazySchema(() => controlEpoch: z.number().int().nonnegative(), providerSessionId: idSchema.optional(), operation: browserRequestOperationSchema.optional(), - pointer: z.object({ - x: z.number().finite().min(0).max(100_000), - y: z.number().finite().min(0).max(100_000), - }).strict().optional(), }) .strict(), ); @@ -635,6 +631,31 @@ export type BrowserAutomationControllerState = z.infer< ReturnType >; +/** + * Point the agent is about to move to or click, reported before the input is dispatched. + * `x` and `y` are CSS pixels relative to the page viewport, so they already account for page + * scroll; `zoomFactor` is the page zoom at that moment, which a host overlay needs to place the + * point in its own pixels. `sequence` increases with every event so a repeated click on the same + * point still reads as new. + */ +export const BrowserAutomationPointerEventSchema = lazySchema(() => + z + .object({ + tabId: idSchema, + phase: z.enum(["move", "click"]), + x: z.number().finite().min(0).max(100_000), + y: z.number().finite().min(0).max(100_000), + zoomFactor: z.number().finite().positive().max(10), + sequence: z.number().int().nonnegative(), + }) + .strict(), +); + +/** One agent cursor point for a Browser tab. */ +export type BrowserAutomationPointerEvent = z.infer< + ReturnType +>; + /** Runtime that owns one browser automation host connection. */ export const BrowserAutomationHostRuntimeSchema = z.enum(["electron", "web"]); /** Browser automation host runtime discriminator. */ From 3247a82360807e1d871ecf4fcabd71fc023b5dd9 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:59:12 +0100 Subject: [PATCH 2/2] fix(browser): validate agent points and record the first one Parse pointer events from the desktop bridge with the contract schema before they reach the overlay store. Report a recording as started only once the page cursor is on, so the agent's first point lands in the video. Re-admit control before a targeted scroll shows the mark, and delay the click ping until the mark finishes its travel. --- .../src/features/preview/automation/kernel.ts | 6 ++++- .../features/preview/preload/agent-cursor.ts | 10 ++++---- .../automation/BrowserAutomationHost.tsx | 6 ++++- .../__tests__/BrowserAutomationHost.test.tsx | 24 ++++++++++++++++++- .../browserAutomationRecorder.test.ts | 22 +++++++++++++++-- .../automation/browserAutomationRecorder.ts | 19 +++++++++++++-- apps/web/src/index.css | 6 +++-- apps/web/src/transport/desktop-bridge.d.ts | 5 ++-- 8 files changed, 82 insertions(+), 16 deletions(-) diff --git a/apps/desktop/src/features/preview/automation/kernel.ts b/apps/desktop/src/features/preview/automation/kernel.ts index 1dc4b0f73..c3e0c1dc9 100644 --- a/apps/desktop/src/features/preview/automation/kernel.ts +++ b/apps/desktop/src/features/preview/automation/kernel.ts @@ -1789,7 +1789,11 @@ export class BrowserAutomationKernel { const { state, webContents } = resolved; const point = request.args.target ? await this.resolvePoint(resolved, request.args.target) : { x: 1, y: 1 }; // An untargeted scroll has no point the user would recognise, so the cursor stays put. - if (request.args.target) this.emitPointer(state, "move", point); + if (request.args.target) { + // Resolving the point awaits the page, so the user may have taken control meanwhile. + this.admitAgentEffect(state, request); + this.emitPointer(state, "move", point); + } await this.withSyntheticInput(state, async () => { await this.ensureDebugger(state); await this.dispatchGuestInput( diff --git a/apps/desktop/src/features/preview/preload/agent-cursor.ts b/apps/desktop/src/features/preview/preload/agent-cursor.ts index 0cc58b0fd..8c3ef32d5 100644 --- a/apps/desktop/src/features/preview/preload/agent-cursor.ts +++ b/apps/desktop/src/features/preview/preload/agent-cursor.ts @@ -3,7 +3,7 @@ import type { PreviewGuestAgentPointerMessage } from "../contracts/guest-input.j /** Time after the last agent point before the in-page mark fades out and is removed. */ export const GUEST_AGENT_CURSOR_IDLE_MS = 700; -const FADE_MS = 150; +const MOTION_MS = 150; const MAX_VIEWPORT_COORDINATE = 100_000; // Matches --amber-500, the dark-theme --primary, so recordings show the same mark as the host. const MARK_COLOR = "oklch(72% 0.170 75)"; @@ -32,7 +32,9 @@ const SHADOW_STYLE = ` border-radius: 50%; border: 2px solid ${MARK_COLOR}; box-sizing: border-box; - animation: ping 450ms ease-out forwards; + opacity: 0; + /* Waits for the mark's travel so the ring appears where the click lands. */ + animation: ping 450ms ease-out ${MOTION_MS}ms forwards; } @keyframes ping { from { opacity: 0.9; transform: scale(0.4); } @@ -81,7 +83,7 @@ export function createGuestAgentCursor(document: Document): GuestAgentCursor { idleTimer = setTimeout(() => { idleTimer = null; mounted?.host.style.setProperty("opacity", "0", "important"); - removeTimer = setTimeout(unmount, FADE_MS); + removeTimer = setTimeout(unmount, MOTION_MS); }, GUEST_AGENT_CURSOR_IDLE_MS); }; @@ -116,7 +118,7 @@ function mountMark(document: Document): MountedMark { ["height", "0"], ["pointer-events", "none"], ["z-index", "2147483647"], - ["transition", `transform ${FADE_MS}ms ease-out, opacity ${FADE_MS}ms ease-out`], + ["transition", `transform ${MOTION_MS}ms ease-out, opacity ${MOTION_MS}ms ease-out`], ] as const) { host.style.setProperty(property, value, "important"); } diff --git a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx index 2f7c1512e..478113fc9 100644 --- a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx +++ b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx @@ -3,6 +3,7 @@ import { BROWSER_AUTOMATION_CONTRACT_VERSION, BROWSER_AUTOMATION_MAX_PENDING_REQUESTS, BrowserAutomationHostDispatchSchema, + BrowserAutomationPointerEventSchema, type BrowserAutomationHostDispatch, type BrowserAutomationHostDispatchTarget, type BrowserAutomationErrorCode, @@ -1514,7 +1515,10 @@ export function BrowserAutomationHost() { useEffect(() => { const bridge = window.desktopBridge?.preview?.automation; if (!bridge) return; - return bridge.onPointer((pointer) => useBrowserAutomationStore.getState().setPointer(pointer)); + return bridge.onPointer((input) => { + const pointer = BrowserAutomationPointerEventSchema().safeParse(input); + if (pointer.success) useBrowserAutomationStore.getState().setPointer(pointer.data); + }); }, []); useEffect(() => { diff --git a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx index fd224dfbf..9c18547df 100644 --- a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx +++ b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx @@ -219,6 +219,7 @@ describe("BrowserAutomationHost", () => { const listTabs = vi.fn(); let controllerChanged: ((state: BrowserAutomationControllerState) => void) | null; let hostHeartbeat: (() => void) | null; + let pointerReported: ((pointer: unknown) => void) | null; let nextGeneration: number; beforeEach(() => { @@ -227,6 +228,7 @@ describe("BrowserAutomationHost", () => { sessionStorage.clear(); controllerChanged = null; hostHeartbeat = null; + pointerReported = null; nextGeneration = 1; harness.transport.registerBrowserAutomationHost.mockImplementation(async () => { const generation = nextGeneration++; @@ -273,7 +275,12 @@ describe("BrowserAutomationHost", () => { if (controllerChanged === callback) controllerChanged = null; }; }, - onPointer: () => () => {}, + onPointer(callback: (pointer: unknown) => void) { + pointerReported = callback; + return () => { + if (pointerReported === callback) pointerReported = null; + }; + }, onHostHeartbeat(callback: () => void) { hostHeartbeat = callback; return () => { @@ -297,6 +304,7 @@ describe("BrowserAutomationHost", () => { useThreadStore.setState({ runningThreadIds: new Set() }); useBrowserAutomationStore.setState({ liveTargets: new Map(), + pointers: new Map(), controllers: new Map(), activeRequests: new Map(), pendingAgentOpens: new Map(), @@ -491,6 +499,20 @@ describe("BrowserAutomationHost", () => { view.unmount(); }); + it("shows only well-formed agent points from the desktop bridge", async () => { + const view = render(); + await waitFor(() => expect(pointerReported).not.toBeNull()); + const pointer = () => useBrowserAutomationStore.getState().pointers.get(browserAutomationTargetKey("workspace-1", "thread-1", "tab-1")); + + act(() => pointerReported?.({ tabId: "tab-1", phase: "click", x: -5, y: 10, zoomFactor: 1, sequence: 1 })); + act(() => pointerReported?.({ tabId: "tab-1", phase: "drag", x: 5, y: 10, zoomFactor: 1, sequence: 2 })); + expect(pointer()).toBeUndefined(); + + act(() => pointerReported?.({ tabId: "tab-1", phase: "click", x: 5, y: 10, zoomFactor: 1, sequence: 3 })); + expect(pointer()).toEqual({ tabId: "tab-1", phase: "click", x: 5, y: 10, zoomFactor: 1, sequence: 3 }); + view.unmount(); + }); + it("settles driver-owned tabs when the broker releases a provider session", async () => { const release = vi.spyOn(BrowserSessionDriver.prototype, "releaseProviderSession").mockResolvedValue(); const view = render(); diff --git a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts index 05c66a416..ef9b2b42d 100644 --- a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts +++ b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts @@ -69,9 +69,9 @@ describe("BrowserAutomationRecorder", () => { const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream; const getUserMedia = vi.fn(); const getMediaSourceId = vi.fn(); - const setRecordingCursor = vi.fn(); + const setRecordingCursor = vi.fn(async (_target: { threadId: string; tabId: string; enabled: boolean }) => true); const bridge = { getMediaSourceId, setRecordingCursor } as unknown as PreviewAutomationBridge; - const cursorStates = () => setRecordingCursor.mock.calls.map(([target]) => (target as { enabled: boolean }).enabled); + const cursorStates = () => setRecordingCursor.mock.calls.map(([target]) => target.enabled); beforeEach(() => { vi.clearAllMocks(); @@ -227,6 +227,24 @@ describe("BrowserAutomationRecorder", () => { expect(cursorStates()).toEqual([true, false, true, false]); }); + it("reports the recording started only once the page cursor is on, so the agent's first point is recorded", async () => { + let cursorOn!: (enabled: boolean) => void; + setRecordingCursor.mockReturnValueOnce(new Promise((resolve) => { cursorOn = resolve; })); + const recorder = new BrowserAutomationRecorder(); + let started = false; + const starting = recorder.start(dispatch("recordingStart", 1), bridge).then((result) => { + started = true; + return result; + }); + await vi.waitFor(() => expect(setRecordingCursor).toHaveBeenCalledOnce()); + expect(started).toBe(false); + + cursorOn(true); + + await expect(starting).resolves.toMatchObject({ ok: true, result: { operation: "recordingStart" } }); + recorder.dispose(); + }); + it("fails a never-resolving stop within the cleanup timeout and stops tracks", async () => { vi.useFakeTimers(); FakeMediaRecorder.stopEmits = false; diff --git a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts index 6f90bc179..775e385b0 100644 --- a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts +++ b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts @@ -227,7 +227,7 @@ export class BrowserAutomationRecorder { releaseCursor: () => { if (cursorReleased) return; cursorReleased = true; - void bridge.setRecordingCursor({ threadId, tabId, enabled: false }); + void bridge.setRecordingCursor({ threadId, tabId, enabled: false }).catch(warnRecordingCursorFailure); }, resolveStopped, }; @@ -266,6 +266,15 @@ export class BrowserAutomationRecorder { return session; } + private async enableRecordingCursor(dispatch: BrowserAutomationHostDispatch, bridge: PreviewAutomationBridge): Promise { + try { + const enabled = await bridge.setRecordingCursor({ threadId: dispatch.target.threadId, tabId: dispatch.target.tabId, enabled: true }); + if (!enabled) warnRecordingCursorFailure(new Error("The recorded tab is no longer the current target")); + } catch (error) { + warnRecordingCursorFailure(error); + } + } + private startRecorder( dispatch: BrowserAutomationHostDispatch, session: RecordingSession, @@ -331,7 +340,6 @@ export class BrowserAutomationRecorder { const session = this.configureRecorder(this.createSession(dispatch, bridge, key, recorder.value, stream.value), key, acquisition); const startFailure = this.startRecorder(dispatch, session, key, acquisition); if (startFailure) return startFailure; - void bridge.setRecordingCursor({ threadId: dispatch.target.threadId, tabId: dispatch.target.tabId, enabled: true }); const requestedDuration = dispatch.request.operation === "recordingStart" ? dispatch.request.args.maxDurationMs : MAX_RECORDING_DURATION_MS; @@ -340,6 +348,8 @@ export class BrowserAutomationRecorder { }, Math.min(requestedDuration, MAX_RECORDING_DURATION_MS)); this.sessions.set(key, session); this.clearPendingAcquisition(key, acquisition); + // Awaited so the agent's first point after recordingStart is already in the video. + await this.enableRecordingCursor(dispatch, bridge); return this.recordingStartedResponse(dispatch, session); } @@ -427,3 +437,8 @@ export class BrowserAutomationRecorder { for (const acquisition of this.pendingAcquisitions.values()) acquisition.cancelled = true; } } + +// The mark is presentation only, so a recording without it is still a valid recording. +function warnRecordingCursorFailure(error: unknown): void { + console.warn("Browser recording cursor could not be updated", error); +} diff --git a/apps/web/src/index.css b/apps/web/src/index.css index 2e311b26b..2eb77e21a 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -686,13 +686,15 @@ diffs-container:has([data-jump-highlight="true"]) { } } -/* One ring per agent click; it ends transparent so nothing keeps animating while the agent is idle. */ +/* One ring per agent click; it ends transparent so nothing keeps animating while the agent is idle. + The delay matches the mark's 150ms travel, so the ring appears where the click lands. */ @keyframes agent-cursor-ping { from { opacity: 0.9; transform: scale(0.4); } to { opacity: 0; transform: scale(1.8); } } .animate-agent-cursor-ping { - animation: agent-cursor-ping 450ms ease-out forwards; + opacity: 0; + animation: agent-cursor-ping 450ms ease-out 150ms forwards; } @media (prefers-reduced-motion: reduce) { .animate-agent-cursor-ping { diff --git a/apps/web/src/transport/desktop-bridge.d.ts b/apps/web/src/transport/desktop-bridge.d.ts index 1b8943fa0..8f43e1e8c 100644 --- a/apps/web/src/transport/desktop-bridge.d.ts +++ b/apps/web/src/transport/desktop-bridge.d.ts @@ -1,7 +1,6 @@ import type { AttachmentMeta, OpenInApp } from "./types"; import type { BrowserAutomationControllerState, - BrowserAutomationPointerEvent, BrowserHistory, BrowserCertificateState, BrowserAutomationHostDispatch, @@ -313,8 +312,8 @@ export interface PreviewAutomationBridge { >; /** Draws the agent cursor inside one tab's page while the renderer records it; false when the tab is unavailable. */ setRecordingCursor(target: { threadId: string; tabId: string; enabled: boolean }): Promise; - /** Subscribe to agent cursor points, reported before each agent pointer action. */ - onPointer(callback: (pointer: BrowserAutomationPointerEvent) => void): () => void; + /** Subscribe to agent cursor points, reported before each agent pointer action; validate with BrowserAutomationPointerEventSchema. */ + onPointer(callback: (pointer: unknown) => void): () => void; /** Subscribe to controller transitions for adopted tabs in this window. */ onControllerChanged(callback: (state: BrowserAutomationControllerState) => void): () => void; /** Subscribe to Browser host heartbeat pulses owned by Electron main. */