diff --git a/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts b/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts index 712e769fb..a314dd11e 100644 --- a/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts +++ b/apps/desktop/src/features/preview/__tests__/preview-feature.test.ts @@ -125,6 +125,7 @@ describe("Preview feature public interface", () => { "preview:automation.release-agent-control", "preview:automation.describe-target", "preview:automation.media-source", + "preview:automation.recording-cursor", "preview:open-guest-devtools", "preview:get-perf-counters", ]; diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts index b98a87d6b..a90d126f6 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel-races.test.ts @@ -160,6 +160,10 @@ class FakeWebContents extends NodeEvents.EventEmitter { return this.destroyed; } + getZoomFactor(): number { + return 1; + } + getURL(): string { return this.url; } diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts index 4091051de..4c2b2e3af 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-automation-kernel.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { BROWSER_AUTOMATION_CONTRACT_VERSION, type BrowserAutomationRequest } from "@mcode/contracts"; import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, PreviewGuestInputSuppressor, } from "../../contracts/guest-input.js"; @@ -159,6 +160,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { url = "https://example.test/"; title = "Example"; focused = true; + zoomFactor = 1; focus = vi.fn(() => { this.focused = true; }); @@ -166,6 +168,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { isDestroyed() { return this.destroyed; } getURL() { return this.url; } getTitle() { return this.title; } + getZoomFactor() { return this.zoomFactor; } isLoading() { return false; } isFocused() { return this.focused; } async executeJavaScript(source: string) { @@ -650,9 +653,12 @@ describe("BrowserAutomationKernel", () => { await expect(click("opaque-replacement", "semantic-replacement")).resolves.toMatchObject({ ok: true }); }); - it("publishes a resolved semantic target pointer before agent input", async () => { - const states: unknown[] = []; - const unsubscribe = kernel.subscribe(7, (state) => states.push(state)); + it("reports a move then a click at the target's centre before pressing the mouse", async () => { + const pointerEvents: Array<{ event: unknown; commandsBefore: number }> = []; + const unsubscribe = kernel.subscribePointer(7, (pointerEvent) => { + pointerEvents.push({ event: pointerEvent, commandsBefore: currentWebContents!.debugger.commands.length }); + }); + currentWebContents!.zoomFactor = 1.5; currentWebContents!.semanticElements.set("semantic-button", { attached: true, visible: true, x: 25, y: 40 }); await kernel.execute(event(), payload(request("click", { target: { semanticId: "semantic-button" }, @@ -660,15 +666,41 @@ describe("BrowserAutomationKernel", () => { clickCount: 1, timeoutMs: 1_000, }, { requestId: "semantic-pointer" }))); - expect(states).toContainEqual(expect.objectContaining({ - controller: "agent", - pointer: { x: 25, y: 40 }, - })); - expect(currentWebContents!.send).toHaveBeenCalledWith( - "mcode:browser-agent-input", - expect.objectContaining({ kind: "pointer", count: 1 }), - ); unsubscribe(); + + expect(pointerEvents.map(({ event }) => event)).toEqual([ + { tabId: "tab", phase: "move", x: 25, y: 40, zoomFactor: 1.5, sequence: 1 }, + { tabId: "tab", phase: "click", x: 25, y: 40, zoomFactor: 1.5, sequence: 2 }, + ]); + const firstPress = currentWebContents!.debugger.commands.findIndex(({ method, params }) => + method === "Input.dispatchMouseEvent" && (params as { type?: string }).type === "mousePressed"); + expect(firstPress).toBeGreaterThanOrEqual(0); + expect(pointerEvents.every(({ commandsBefore }) => commandsBefore <= firstPress)).toBe(true); + }); + + it("forwards agent pointer events to the page only while its recording cursor is on", async () => { + currentWebContents!.semanticElements.set("recorded-button", { attached: true, visible: true, x: 30, y: 60 }); + const click = (requestId: string) => kernel.execute(event(), payload(request("click", { + target: { semanticId: "recorded-button" }, + button: "left", + clickCount: 1, + timeoutMs: 1_000, + }, { requestId }))); + const pointerSends = () => currentWebContents!.send.mock.calls.filter(([channel]) => channel === PREVIEW_GUEST_AGENT_POINTER_CHANNEL); + + await click("before-recording"); + expect(pointerSends()).toEqual([]); + + expect(kernel.setRecordingCursor(event(), { threadId: "thread", tabId: "tab", enabled: true })).toBe(true); + await click("while-recording"); + expect(pointerSends().map(([, pointer]) => pointer)).toEqual([ + { phase: "move", x: 30, y: 60 }, + { phase: "click", x: 30, y: 60 }, + ]); + + expect(kernel.setRecordingCursor(event(), { threadId: "thread", tabId: "tab", enabled: false })).toBe(true); + await click("after-recording"); + expect(pointerSends()).toHaveLength(2); }); it("scrolls an off-screen semantic target into view before clicking", async () => { diff --git a/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts b/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts index 521e80e01..b8d39d77d 100644 --- a/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts +++ b/apps/desktop/src/features/preview/automation/__tests__/browser-executor-parity.test.ts @@ -103,6 +103,7 @@ class FakeWebContents extends NodeEvents.EventEmitter { stop = vi.fn(); send = vi.fn(); isDestroyed(): boolean { return this.destroyed; } + getZoomFactor(): number { return 1; } getURL(): string { return this.url; } getTitle(): string { return "Executor fixture"; } isLoading(): boolean { return false; } diff --git a/apps/desktop/src/features/preview/automation/index.ts b/apps/desktop/src/features/preview/automation/index.ts index 2f5b4b73e..386779132 100644 --- a/apps/desktop/src/features/preview/automation/index.ts +++ b/apps/desktop/src/features/preview/automation/index.ts @@ -22,16 +22,23 @@ export function registerBrowserAutomationHandlers(platform: NodeJS.Platform): vo ipcMain.handle("preview:automation.release-agent-control", (event, target: unknown) => automationKernel.releaseAgentControl(event, target)); ipcMain.handle("preview:automation.describe-target", (event, target: unknown) => automationKernel.describeTarget(event, target)); ipcMain.handle("preview:automation.media-source", (event, target: unknown) => automationKernel.getMediaSourceId(event, target)); + ipcMain.handle("preview:automation.recording-cursor", (event, target: unknown) => automationKernel.setRecordingCursor(event, target)); ipcMain.handle("preview:open-guest-devtools", (event, target: unknown) => automationKernel.openDevTools(event, target)); ipcMain.on("preview:automation.subscribe", (event: IpcMainEvent) => { const win = BrowserWindow.fromWebContents(event.sender); if (!win || win.isDestroyed()) return; if (subscribedRenderers.has(event.sender)) return; subscribedRenderers.add(event.sender); - const unsubscribe = automationKernel.subscribe(win.id, (state) => { + const unsubscribeController = automationKernel.subscribe(win.id, (state) => { if (!event.sender.isDestroyed()) event.sender.send("preview:automation.controller", state); }); - event.sender.once("destroyed", unsubscribe); + const unsubscribePointer = automationKernel.subscribePointer(win.id, (pointer) => { + if (!event.sender.isDestroyed()) event.sender.send("preview:automation.pointer", pointer); + }); + event.sender.once("destroyed", () => { + unsubscribeController(); + unsubscribePointer(); + }); }); ipcMain.on(BROWSER_AUTOMATION_HEARTBEAT_SUBSCRIBE_CHANNEL, (event: IpcMainEvent) => { heartbeatPulse.subscribe(event.sender); diff --git a/apps/desktop/src/features/preview/automation/kernel.ts b/apps/desktop/src/features/preview/automation/kernel.ts index a49716357..c3e0c1dc9 100644 --- a/apps/desktop/src/features/preview/automation/kernel.ts +++ b/apps/desktop/src/features/preview/automation/kernel.ts @@ -24,11 +24,16 @@ import { type BrowserAutomationHostDispatchTarget, type BrowserAutomationHostDispatch, type BrowserAutomationControllerState, + type BrowserAutomationPointerEvent, } from "@mcode/contracts"; import { findAdoptedWebContentsForWindow } from "../surfaces/registry.js"; import { resolveActivePreviewWebContentsForWindow } from "../surfaces/active-web-contents.js"; import { loadPreviewGuestUrl } from "../navigation/guest-navigation.js"; -import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL } from "../contracts/guest-input.js"; +import { + PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, + type PreviewGuestAgentPointerMessage, +} from "../contracts/guest-input.js"; import { getSession, getThreadTabSet } from "../state/window-session.js"; import { BrowserAutomationCancelledError, @@ -616,6 +621,10 @@ export class BrowserAutomationKernel { private readonly targetGenerations = new Map(); private readonly cancellations = new Map void }>(); private readonly subscribers = new Map void>>(); + private readonly pointerSubscribers = new Map void>>(); + /** Target keys whose page is being recorded, so the guest must draw the agent cursor itself. */ + private readonly recordingCursorTargets = new Set(); + private pointerSequence = 0; private readonly rendererOperationLeases = new Map(); /** Validates and executes one request from a trusted renderer. */ @@ -974,6 +983,46 @@ export class BrowserAutomationKernel { }; } + /** Subscribes a renderer window to agent cursor points, reported before each pointer action. */ + subscribePointer(windowId: number, callback: (event: BrowserAutomationPointerEvent) => void): () => void { + const listeners = this.pointerSubscribers.get(windowId) ?? new Set(); + listeners.add(callback); + this.pointerSubscribers.set(windowId, listeners); + return () => { + listeners.delete(callback); + if (listeners.size === 0) this.pointerSubscribers.delete(windowId); + }; + } + + /** + * Turns the in-page agent cursor on or off for one tab while the renderer records it. + * Turning it off never requires a live tab, so a recording that outlives its tab still cleans up. + */ + setRecordingCursor(event: IpcMainInvokeEvent, input: unknown): boolean { + const value = asRecord(input); + let threadId: string; + let tabId: string; + try { + threadId = assertShortId(value.threadId, "threadId"); + tabId = assertShortId(value.tabId, "tabId"); + } catch { + return false; + } + if (typeof value.enabled !== "boolean") return false; + if (!value.enabled) { + const win = BrowserWindow.fromWebContents(event.sender); + if (!win) return false; + this.recordingCursorTargets.delete(targetKey(win.id, threadId, tabId)); + return true; + } + try { + this.recordingCursorTargets.add(this.resolveCurrentTarget(event, threadId, tabId).state.key); + return true; + } catch { + return false; + } + } + /** Releases target listeners, debugger sessions, queues, and buffers for one window. */ disposeWindow(windowId: number): void { for (const [key, state] of this.targets) { @@ -989,6 +1038,10 @@ export class BrowserAutomationKernel { if (key.startsWith(`[${windowId},`)) this.threadControls.delete(key); } this.subscribers.delete(windowId); + this.pointerSubscribers.delete(windowId); + for (const key of this.recordingCursorTargets) { + if (key.startsWith(`[${windowId},`)) this.recordingCursorTargets.delete(key); + } } /** Returns internal bounded counters for tests and performance diagnostics. */ @@ -1187,7 +1240,6 @@ export class BrowserAutomationKernel { state: TargetState, event: BrowserControlEvent, request?: BrowserAutomationRequest, - pointer?: { x: number; y: number }, ): boolean { const key = JSON.stringify([state.windowId, state.threadId]); const previous = this.threadControls.get(key) ?? createBrowserThreadControl(); @@ -1196,7 +1248,7 @@ export class BrowserAutomationKernel { this.threadControls.set(key, next); if (event.type === "agent-effect") { state.controlEpoch = next.epochByTab.get(state.tabId) ?? 0; - this.emitController(state, "agent", request, pointer); + this.emitController(state, "agent", request); } else { this.publishThreadControl(state, next); } @@ -1236,19 +1288,40 @@ export class BrowserAutomationKernel { } } - private admitAgentEffect(state: TargetState, request: BrowserAutomationRequest, pointer?: { x: number; y: number }): void { + private admitAgentEffect(state: TargetState, request: BrowserAutomationRequest): void { this.applyControlEvent(state, { type: "agent-effect", tabId: state.tabId, expectedEpoch: request.expectedControlEpoch, - }, request, pointer); + }, request); + } + + /** Reports where the agent is about to act; runs after admission so a held tab never shows a cursor. */ + private emitPointer(state: TargetState, phase: BrowserAutomationPointerEvent["phase"], point: { x: number; y: number }): void { + this.pointerSequence += 1; + const event: BrowserAutomationPointerEvent = { + tabId: state.tabId, + phase, + x: point.x, + y: point.y, + zoomFactor: state.webContents.getZoomFactor(), + sequence: this.pointerSequence, + }; + for (const listener of this.pointerSubscribers.get(state.windowId) ?? []) listener(event); + if (!this.recordingCursorTargets.has(state.key) || state.webContents.isDestroyed()) return; + const message: PreviewGuestAgentPointerMessage = { phase, x: point.x, y: point.y }; + state.webContents.send(PREVIEW_GUEST_AGENT_POINTER_CHANNEL, message); + } + + private emitPointerMoveThenClick(state: TargetState, point: { x: number; y: number }): void { + this.emitPointer(state, "move", point); + this.emitPointer(state, "click", point); } private emitController( state: TargetState, controller: "none" | "human" | "agent", request?: BrowserAutomationRequest, - pointer?: { x: number; y: number }, ): void { const payload: BrowserAutomationControllerState = { tabId: state.tabId, @@ -1262,7 +1335,6 @@ export class BrowserAutomationKernel { : {}), } : {}), - ...(pointer ? { pointer } : {}), }; state.controller = payload; for (const listener of this.subscribers.get(state.windowId) ?? []) listener(payload); @@ -1601,7 +1673,8 @@ export class BrowserAutomationKernel { ): Promise { const { state, webContents } = resolved; const point = await this.resolvePoint(resolved, request.args.target); - this.admitAgentEffect(state, request, point); + this.admitAgentEffect(state, request); + this.emitPointerMoveThenClick(state, point); await this.withSyntheticInput(state, async () => { await this.dispatchClick(state, webContents, point, request, signal, markEffect); this.throwIfAborted(signal); @@ -1667,7 +1740,8 @@ export class BrowserAutomationKernel { ): Promise { if (!request.args.target) return; const point = await this.resolvePoint(resolved, request.args.target); - this.admitAgentEffect(resolved.state, request, point); + this.admitAgentEffect(resolved.state, request); + this.emitPointerMoveThenClick(resolved.state, point); await this.withSyntheticInput(resolved.state, () => this.clickPoint(resolved.state, point, markEffect)); this.throwIfAborted(signal); } @@ -1714,6 +1788,12 @@ export class BrowserAutomationKernel { ): Promise { const { state, webContents } = resolved; const point = request.args.target ? await this.resolvePoint(resolved, request.args.target) : { x: 1, y: 1 }; + // An untargeted scroll has no point the user would recognise, so the cursor stays put. + if (request.args.target) { + // Resolving the point awaits the page, so the user may have taken control meanwhile. + this.admitAgentEffect(state, request); + this.emitPointer(state, "move", point); + } await this.withSyntheticInput(state, async () => { await this.ensureDebugger(state); await this.dispatchGuestInput( diff --git a/apps/desktop/src/features/preview/contracts/guest-input.ts b/apps/desktop/src/features/preview/contracts/guest-input.ts index c28a73164..8ff58f64c 100644 --- a/apps/desktop/src/features/preview/contracts/guest-input.ts +++ b/apps/desktop/src/features/preview/contracts/guest-input.ts @@ -16,6 +16,19 @@ export const PREVIEW_GUEST_CLIPBOARD_TRUST_CHANNEL = "mcode:browser-clipboard-tr /** Guest preload channel that reserves trusted events generated by agent CDP input. */ export const PREVIEW_GUEST_AGENT_INPUT_CHANNEL = "mcode:browser-agent-input"; +/** + * Guest preload channel that draws the agent cursor inside the page. The host only sends it while + * the tab is being recorded, because a tab capture sees the page and not the overlay above it. + */ +export const PREVIEW_GUEST_AGENT_POINTER_CHANNEL = "mcode:browser-agent-pointer"; + +/** Agent cursor point in viewport CSS pixels, sent to the guest preload before the agent acts. */ +export interface PreviewGuestAgentPointerMessage { + readonly phase: "move" | "click"; + readonly x: number; + readonly y: number; +} + /** Trusted guest interaction categories that transfer control from an agent to the human. */ export const PREVIEW_GUEST_HUMAN_INPUT_KINDS = [ "keyboard", diff --git a/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts b/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts new file mode 100644 index 000000000..df294c8dc --- /dev/null +++ b/apps/desktop/src/features/preview/preload/__tests__/agent-cursor.test.ts @@ -0,0 +1,62 @@ +// @vitest-environment jsdom +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createGuestAgentCursor, GUEST_AGENT_CURSOR_IDLE_MS } from "../agent-cursor.js"; + +const mark = (): HTMLElement | null => document.querySelector("mcode-agent-cursor"); + +describe("guest agent cursor", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + document.documentElement.innerHTML = ""; + }); + + it("draws an inert mark at the agent's viewport point", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "move", x: 120, y: 48 }); + + const element = mark(); + expect(element?.getAttribute("aria-hidden")).toBe("true"); + expect(element?.style.pointerEvents).toBe("none"); + expect(element?.style.transform).toBe("translate(120px, 48px)"); + }); + + it("pings once per click, not per move", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "move", x: 10, y: 10 }); + expect(cursor.pingCount()).toBe(0); + cursor.show({ phase: "click", x: 10, y: 10 }); + cursor.show({ phase: "move", x: 30, y: 30 }); + cursor.show({ phase: "click", x: 30, y: 30 }); + + expect(cursor.pingCount()).toBe(2); + }); + + it("removes the mark once the agent goes idle so nothing keeps animating", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "click", x: 10, y: 10 }); + vi.advanceTimersByTime(GUEST_AGENT_CURSOR_IDLE_MS - 1); + cursor.show({ phase: "move", x: 20, y: 20 }); + vi.advanceTimersByTime(GUEST_AGENT_CURSOR_IDLE_MS - 1); + expect(mark()).not.toBeNull(); + + vi.advanceTimersByTime(1_000); + expect(mark()).toBeNull(); + }); + + it("ignores malformed host messages", () => { + const cursor = createGuestAgentCursor(document); + + cursor.show({ phase: "drag", x: 1, y: 1 }); + cursor.show({ phase: "move", x: Number.NaN, y: 1 }); + cursor.show(null); + + expect(mark()).toBeNull(); + }); +}); diff --git a/apps/desktop/src/features/preview/preload/agent-cursor.ts b/apps/desktop/src/features/preview/preload/agent-cursor.ts new file mode 100644 index 000000000..8c3ef32d5 --- /dev/null +++ b/apps/desktop/src/features/preview/preload/agent-cursor.ts @@ -0,0 +1,155 @@ +import type { PreviewGuestAgentPointerMessage } from "../contracts/guest-input.js"; + +/** Time after the last agent point before the in-page mark fades out and is removed. */ +export const GUEST_AGENT_CURSOR_IDLE_MS = 700; + +const MOTION_MS = 150; +const MAX_VIEWPORT_COORDINATE = 100_000; +// Matches --amber-500, the dark-theme --primary, so recordings show the same mark as the host. +const MARK_COLOR = "oklch(72% 0.170 75)"; +// The pointer tip sits at (4, 4) in the 24-unit icon box; shifting by it puts the tip on the point. +const TIP_OFFSET_PX = 3.3; + +const SHADOW_STYLE = ` + svg { + position: absolute; + left: ${-TIP_OFFSET_PX}px; + top: ${-TIP_OFFSET_PX}px; + width: 20px; + height: 20px; + fill: ${MARK_COLOR}; + stroke: ${MARK_COLOR}; + stroke-width: 2; + stroke-linejoin: round; + filter: drop-shadow(0 0 4px rgb(0 0 0 / 0.45)); + } + .ping { + position: absolute; + left: -10px; + top: -10px; + width: 20px; + height: 20px; + border-radius: 50%; + border: 2px solid ${MARK_COLOR}; + box-sizing: border-box; + opacity: 0; + /* Waits for the mark's travel so the ring appears where the click lands. */ + animation: ping 450ms ease-out ${MOTION_MS}ms forwards; + } + @keyframes ping { + from { opacity: 0.9; transform: scale(0.4); } + to { opacity: 0; transform: scale(1.8); } + } + @media (prefers-reduced-motion: reduce) { + .ping { animation: none; opacity: 0; } + } +`; + +const POINTER_PATH = + "M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"; + +/** In-page agent cursor drawn by the guest preload so tab recordings include it. */ +export interface GuestAgentCursor { + /** Moves the mark to one host-reported point; malformed messages are ignored. */ + show(message: unknown): void; + /** Number of click pings drawn since creation. */ + pingCount(): number; +} + +interface MountedMark { + readonly host: HTMLElement; + readonly root: ShadowRoot; +} + +/** + * Creates the in-page agent cursor. The mark is inert: it never takes input, is hidden from + * assistive technology, and is removed once the agent goes idle so no animation keeps running. + */ +export function createGuestAgentCursor(document: Document): GuestAgentCursor { + let mounted: MountedMark | null = null; + let idleTimer: ReturnType | null = null; + let removeTimer: ReturnType | null = null; + let pings = 0; + + const unmount = () => { + mounted?.host.remove(); + mounted = null; + }; + + const scheduleIdle = () => { + if (idleTimer) clearTimeout(idleTimer); + if (removeTimer) clearTimeout(removeTimer); + removeTimer = null; + idleTimer = setTimeout(() => { + idleTimer = null; + mounted?.host.style.setProperty("opacity", "0", "important"); + removeTimer = setTimeout(unmount, MOTION_MS); + }, GUEST_AGENT_CURSOR_IDLE_MS); + }; + + return { + show(message) { + const pointer = parsePointerMessage(message); + if (!pointer) return; + const mark = mounted?.host.isConnected ? mounted : mountMark(document); + mounted = mark; + mark.host.style.setProperty("opacity", "1", "important"); + mark.host.style.setProperty("transform", `translate(${pointer.x}px, ${pointer.y}px)`, "important"); + if (pointer.phase === "click") { + pings += 1; + drawPing(mark.root); + } + scheduleIdle(); + }, + pingCount: () => pings, + }; +} + +function mountMark(document: Document): MountedMark { + const host = document.createElement("mcode-agent-cursor"); + host.setAttribute("aria-hidden", "true"); + // Inline !important keeps page stylesheets from moving, hiding, or making the mark clickable. + for (const [property, value] of [ + ["all", "initial"], + ["position", "fixed"], + ["left", "0"], + ["top", "0"], + ["width", "0"], + ["height", "0"], + ["pointer-events", "none"], + ["z-index", "2147483647"], + ["transition", `transform ${MOTION_MS}ms ease-out, opacity ${MOTION_MS}ms ease-out`], + ] as const) { + host.style.setProperty(property, value, "important"); + } + const root = host.attachShadow({ mode: "closed" }); + const style = document.createElement("style"); + style.textContent = SHADOW_STYLE; + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("viewBox", "0 0 24 24"); + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", POINTER_PATH); + svg.append(path); + root.append(style, svg); + document.documentElement.append(host); + return { host, root }; +} + +function drawPing(root: ShadowRoot): void { + root.querySelector(".ping")?.remove(); + const ping = root.ownerDocument.createElement("span"); + ping.className = "ping"; + root.append(ping); +} + +function parsePointerMessage(message: unknown): PreviewGuestAgentPointerMessage | null { + if (!message || typeof message !== "object") return null; + const { phase, x, y } = message as Partial>; + if (phase !== "move" && phase !== "click") return null; + if (!isViewportCoordinate(x) || !isViewportCoordinate(y)) return null; + return { phase, x, y }; +} + +function isViewportCoordinate(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= MAX_VIEWPORT_COORDINATE; +} diff --git a/apps/desktop/src/features/preview/preload/guest-input.ts b/apps/desktop/src/features/preview/preload/guest-input.ts index d688fcfc8..15b9a503a 100644 --- a/apps/desktop/src/features/preview/preload/guest-input.ts +++ b/apps/desktop/src/features/preview/preload/guest-input.ts @@ -1,12 +1,14 @@ import { ipcRenderer } from "electron"; import { PREVIEW_GUEST_AGENT_INPUT_CHANNEL, + PREVIEW_GUEST_AGENT_POINTER_CHANNEL, PREVIEW_GUEST_CLIPBOARD_TRUST_CHANNEL, PREVIEW_GUEST_HUMAN_INPUT_CHANNEL, PREVIEW_GUEST_SCROLL_CHANNEL, PreviewGuestInputSuppressor, toPreviewGuestHumanInputMessage, } from "../contracts/guest-input.js"; +import { createGuestAgentCursor, type GuestAgentCursor } from "./agent-cursor.js"; const suppressor = new PreviewGuestInputSuppressor(); @@ -22,6 +24,14 @@ ipcRenderer.on(PREVIEW_GUEST_AGENT_INPUT_CHANNEL, (_event, input: unknown) => { } }); +// Created on the first point: the host only sends points while recording, so most pages never pay for it. +let agentCursor: GuestAgentCursor | null = null; + +ipcRenderer.on(PREVIEW_GUEST_AGENT_POINTER_CHANNEL, (_event, pointer: unknown) => { + agentCursor ??= createGuestAgentCursor(document); + agentCursor.show(pointer); +}); + for (const eventType of ["keydown", "pointerdown", "touchstart", "wheel"] as const) { window.addEventListener( eventType, diff --git a/apps/desktop/src/main/preload.ts b/apps/desktop/src/main/preload.ts index 95facb9df..d6d9623ca 100644 --- a/apps/desktop/src/main/preload.ts +++ b/apps/desktop/src/main/preload.ts @@ -517,12 +517,21 @@ contextBridge.exposeInMainWorld("desktopBridge", { }): Promise { return ipcRenderer.invoke("preview:automation.media-source", target); }, + setRecordingCursor(target: { threadId: string; tabId: string; enabled: boolean }): Promise { + return ipcRenderer.invoke("preview:automation.recording-cursor", target); + }, onControllerChanged(callback: (state: unknown) => void): () => void { const listener = (_event: unknown, state: unknown) => callback(state); ipcRenderer.send("preview:automation.subscribe"); ipcRenderer.on("preview:automation.controller", listener); return () => ipcRenderer.removeListener("preview:automation.controller", listener); }, + onPointer(callback: (pointer: unknown) => void): () => void { + const listener = (_event: unknown, pointer: unknown) => callback(pointer); + ipcRenderer.send("preview:automation.subscribe"); + ipcRenderer.on("preview:automation.pointer", listener); + return () => ipcRenderer.removeListener("preview:automation.pointer", listener); + }, onHostHeartbeat(callback: () => void): () => void { const listener = () => callback(); ipcRenderer.on("preview:automation.heartbeat", listener); diff --git a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx index f194fbf6e..478113fc9 100644 --- a/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx +++ b/apps/web/src/features/preview/automation/BrowserAutomationHost.tsx @@ -3,6 +3,7 @@ import { BROWSER_AUTOMATION_CONTRACT_VERSION, BROWSER_AUTOMATION_MAX_PENDING_REQUESTS, BrowserAutomationHostDispatchSchema, + BrowserAutomationPointerEventSchema, type BrowserAutomationHostDispatch, type BrowserAutomationHostDispatchTarget, type BrowserAutomationErrorCode, @@ -1511,6 +1512,15 @@ export function BrowserAutomationHost() { sessionDriverRef.current?.invalidateTargetObservations(workspaceId, threadId, tabId); }), []); + useEffect(() => { + const bridge = window.desktopBridge?.preview?.automation; + if (!bridge) return; + return bridge.onPointer((input) => { + const pointer = BrowserAutomationPointerEventSchema().safeParse(input); + if (pointer.success) useBrowserAutomationStore.getState().setPointer(pointer.data); + }); + }, []); + useEffect(() => { const bridge = window.desktopBridge?.preview?.automation; if (!bridge) return; diff --git a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx index 9b53fc101..9c18547df 100644 --- a/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx +++ b/apps/web/src/features/preview/automation/__tests__/BrowserAutomationHost.test.tsx @@ -219,6 +219,7 @@ describe("BrowserAutomationHost", () => { const listTabs = vi.fn(); let controllerChanged: ((state: BrowserAutomationControllerState) => void) | null; let hostHeartbeat: (() => void) | null; + let pointerReported: ((pointer: unknown) => void) | null; let nextGeneration: number; beforeEach(() => { @@ -227,6 +228,7 @@ describe("BrowserAutomationHost", () => { sessionStorage.clear(); controllerChanged = null; hostHeartbeat = null; + pointerReported = null; nextGeneration = 1; harness.transport.registerBrowserAutomationHost.mockImplementation(async () => { const generation = nextGeneration++; @@ -273,6 +275,12 @@ describe("BrowserAutomationHost", () => { if (controllerChanged === callback) controllerChanged = null; }; }, + onPointer(callback: (pointer: unknown) => void) { + pointerReported = callback; + return () => { + if (pointerReported === callback) pointerReported = null; + }; + }, onHostHeartbeat(callback: () => void) { hostHeartbeat = callback; return () => { @@ -296,6 +304,7 @@ describe("BrowserAutomationHost", () => { useThreadStore.setState({ runningThreadIds: new Set() }); useBrowserAutomationStore.setState({ liveTargets: new Map(), + pointers: new Map(), controllers: new Map(), activeRequests: new Map(), pendingAgentOpens: new Map(), @@ -490,6 +499,20 @@ describe("BrowserAutomationHost", () => { view.unmount(); }); + it("shows only well-formed agent points from the desktop bridge", async () => { + const view = render(); + await waitFor(() => expect(pointerReported).not.toBeNull()); + const pointer = () => useBrowserAutomationStore.getState().pointers.get(browserAutomationTargetKey("workspace-1", "thread-1", "tab-1")); + + act(() => pointerReported?.({ tabId: "tab-1", phase: "click", x: -5, y: 10, zoomFactor: 1, sequence: 1 })); + act(() => pointerReported?.({ tabId: "tab-1", phase: "drag", x: 5, y: 10, zoomFactor: 1, sequence: 2 })); + expect(pointer()).toBeUndefined(); + + act(() => pointerReported?.({ tabId: "tab-1", phase: "click", x: 5, y: 10, zoomFactor: 1, sequence: 3 })); + expect(pointer()).toEqual({ tabId: "tab-1", phase: "click", x: 5, y: 10, zoomFactor: 1, sequence: 3 }); + view.unmount(); + }); + it("settles driver-owned tabs when the broker releases a provider session", async () => { const release = vi.spyOn(BrowserSessionDriver.prototype, "releaseProviderSession").mockResolvedValue(); const view = render(); diff --git a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts index e85669abe..ef9b2b42d 100644 --- a/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts +++ b/apps/web/src/features/preview/automation/__tests__/browserAutomationRecorder.test.ts @@ -69,7 +69,9 @@ describe("BrowserAutomationRecorder", () => { const stream = { getTracks: () => [{ stop: stopTrack }] } as unknown as MediaStream; const getUserMedia = vi.fn(); const getMediaSourceId = vi.fn(); - const bridge = { getMediaSourceId } as unknown as PreviewAutomationBridge; + const setRecordingCursor = vi.fn(async (_target: { threadId: string; tabId: string; enabled: boolean }) => true); + const bridge = { getMediaSourceId, setRecordingCursor } as unknown as PreviewAutomationBridge; + const cursorStates = () => setRecordingCursor.mock.calls.map(([target]) => target.enabled); beforeEach(() => { vi.clearAllMocks(); @@ -206,6 +208,43 @@ describe("BrowserAutomationRecorder", () => { } }); + it("draws the page cursor only while a recording runs, however the recording ends", async () => { + const recorder = new BrowserAutomationRecorder(); + await recorder.start(dispatch("recordingStart", 1), bridge); + expect(setRecordingCursor).toHaveBeenLastCalledWith({ threadId: "thread", tabId: "tab", enabled: true }); + FakeMediaRecorder.instances[0]!.emit(400 * 1_024); + await Promise.resolve(); + expect(cursorStates()).toEqual([true, false]); + await recorder.stop(dispatch("recordingStop", 2)); + expect(cursorStates()).toEqual([true, false]); + + vi.useFakeTimers(); + FakeMediaRecorder.stopEmits = false; + await recorder.start(dispatch("recordingStart", 3), bridge); + const stopping = recorder.stop(dispatch("recordingStop", 4)); + await vi.advanceTimersByTimeAsync(1_000); + await stopping; + expect(cursorStates()).toEqual([true, false, true, false]); + }); + + it("reports the recording started only once the page cursor is on, so the agent's first point is recorded", async () => { + let cursorOn!: (enabled: boolean) => void; + setRecordingCursor.mockReturnValueOnce(new Promise((resolve) => { cursorOn = resolve; })); + const recorder = new BrowserAutomationRecorder(); + let started = false; + const starting = recorder.start(dispatch("recordingStart", 1), bridge).then((result) => { + started = true; + return result; + }); + await vi.waitFor(() => expect(setRecordingCursor).toHaveBeenCalledOnce()); + expect(started).toBe(false); + + cursorOn(true); + + await expect(starting).resolves.toMatchObject({ ok: true, result: { operation: "recordingStart" } }); + recorder.dispose(); + }); + it("fails a never-resolving stop within the cleanup timeout and stops tracks", async () => { vi.useFakeTimers(); FakeMediaRecorder.stopEmits = false; diff --git a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts index 5380dab53..775e385b0 100644 --- a/apps/web/src/features/preview/automation/browserAutomationRecorder.ts +++ b/apps/web/src/features/preview/automation/browserAutomationRecorder.ts @@ -42,6 +42,8 @@ interface RecordingSession { stopped: Promise; stopTimer: number; error: string | null; + /** Turns the in-page agent cursor off; tab capture cannot see the host overlay, so the page draws it. */ + readonly releaseCursor: () => void; } interface RecordingSessionWithResolver extends RecordingSession { @@ -197,10 +199,13 @@ export class BrowserAutomationRecorder { private createSession( dispatch: BrowserAutomationHostDispatch, + bridge: PreviewAutomationBridge, key: string, recorder: MediaRecorder, stream: MediaStream, ): RecordingSessionWithResolver { + const { threadId, tabId } = dispatch.target; + let cursorReleased = false; let resolveStopped!: () => void; const stopped = new Promise((resolve) => { resolveStopped = resolve; @@ -219,6 +224,11 @@ export class BrowserAutomationRecorder { stopped, stopTimer: 0, error: null, + releaseCursor: () => { + if (cursorReleased) return; + cursorReleased = true; + void bridge.setRecordingCursor({ threadId, tabId, enabled: false }).catch(warnRecordingCursorFailure); + }, resolveStopped, }; } @@ -239,6 +249,7 @@ export class BrowserAutomationRecorder { session.retainedBytes += event.data.size; }; recorder.onstop = () => { + session.releaseCursor(); window.clearTimeout(session.stopTimer); this.stopStream(stream); this.clearPendingAcquisition(key, acquisition); @@ -246,6 +257,7 @@ export class BrowserAutomationRecorder { }; recorder.onerror = () => { session.error = "The browser recording encoder failed"; + session.releaseCursor(); window.clearTimeout(session.stopTimer); if (recorder.state !== "inactive") recorder.stop(); this.stopStream(stream); @@ -254,6 +266,15 @@ export class BrowserAutomationRecorder { return session; } + private async enableRecordingCursor(dispatch: BrowserAutomationHostDispatch, bridge: PreviewAutomationBridge): Promise { + try { + const enabled = await bridge.setRecordingCursor({ threadId: dispatch.target.threadId, tabId: dispatch.target.tabId, enabled: true }); + if (!enabled) warnRecordingCursorFailure(new Error("The recorded tab is no longer the current target")); + } catch (error) { + warnRecordingCursorFailure(error); + } + } + private startRecorder( dispatch: BrowserAutomationHostDispatch, session: RecordingSession, @@ -316,7 +337,7 @@ export class BrowserAutomationRecorder { } const recorder = this.createRecorder(dispatch, stream.value, key, acquisition); if (!recorder.ok) return recorder.response; - const session = this.configureRecorder(this.createSession(dispatch, key, recorder.value, stream.value), key, acquisition); + const session = this.configureRecorder(this.createSession(dispatch, bridge, key, recorder.value, stream.value), key, acquisition); const startFailure = this.startRecorder(dispatch, session, key, acquisition); if (startFailure) return startFailure; const requestedDuration = dispatch.request.operation === "recordingStart" @@ -327,6 +348,8 @@ export class BrowserAutomationRecorder { }, Math.min(requestedDuration, MAX_RECORDING_DURATION_MS)); this.sessions.set(key, session); this.clearPendingAcquisition(key, acquisition); + // Awaited so the agent's first point after recordingStart is already in the video. + await this.enableRecordingCursor(dispatch, bridge); return this.recordingStartedResponse(dispatch, session); } @@ -356,6 +379,7 @@ export class BrowserAutomationRecorder { ]); window.clearTimeout(stopTimeout); this.sessions.delete(key); + session.releaseCursor(); if (stopTimedOut) return failure(dispatch, "The browser recording encoder did not stop"); if (session.error) return failure(dispatch, session.error); const retained = new Uint8Array(await new Blob(session.chunks, { type: "video/webm" }).arrayBuffer()); @@ -398,6 +422,7 @@ export class BrowserAutomationRecorder { const session = this.sessions.get(key); if (!session) return; this.sessions.delete(session.targetKey); + session.releaseCursor(); window.clearTimeout(session.stopTimer); if (session.recorder.state !== "inactive") session.recorder.stop(); for (const track of session.stream.getTracks()) track.stop(); @@ -412,3 +437,8 @@ export class BrowserAutomationRecorder { for (const acquisition of this.pendingAcquisitions.values()) acquisition.cancelled = true; } } + +// The mark is presentation only, so a recording without it is still a valid recording. +function warnRecordingCursorFailure(error: unknown): void { + console.warn("Browser recording cursor could not be updated", error); +} diff --git a/apps/web/src/features/preview/automation/browserAutomationStore.ts b/apps/web/src/features/preview/automation/browserAutomationStore.ts index f8fcd67d3..e9249cb3a 100644 --- a/apps/web/src/features/preview/automation/browserAutomationStore.ts +++ b/apps/web/src/features/preview/automation/browserAutomationStore.ts @@ -1,6 +1,7 @@ import type { BrowserAutomationControllerState, BrowserAutomationHostDispatch, + BrowserAutomationPointerEvent, } from "@mcode/contracts"; import { BROWSER_AUTOMATION_MAX_INSPECT_TABS, BROWSER_AUTOMATION_MAX_PENDING_REQUESTS } from "@mcode/contracts"; import { create } from "zustand"; @@ -48,6 +49,8 @@ interface BrowserAutomationState { readonly liveTargets: ReadonlyMap; readonly lifecycleTabs: ReadonlyMap; readonly controllers: ReadonlyMap; + /** Latest agent cursor point per target, in the page's viewport CSS pixels. */ + readonly pointers: ReadonlyMap; readonly activeRequests: ReadonlyMap; readonly pendingAgentOpens: ReadonlyMap; /** Targets whose agent has a Browser effect running or finished within {@link BROWSER_ACTING_LINGER_MS}. */ @@ -66,6 +69,7 @@ interface BrowserAutomationState { releaseThreadTargets: (workspaceId: string, threadId: string) => void; releaseWorkspaceTargets: (workspaceId: string) => void; setController: (state: BrowserAutomationControllerState) => void; + setPointer: (pointer: BrowserAutomationPointerEvent) => void; setControllerForTarget: ( workspaceId: string, threadId: string, @@ -202,10 +206,10 @@ export function isBrowserAutomationAgentControlled( ) !== null; } -/** Resolve a controller event only when its tab id maps to one exact live target. */ -export function resolveBrowserAutomationControllerTarget( +/** Resolve a controller or pointer event only when its tab id maps to one exact live target. */ +export function resolveBrowserAutomationControllerTarget( targets: Iterable, - controller: BrowserAutomationControllerState, + controller: TEvent, ): BrowserAutomationLiveTarget | null { const candidates = [...targets].filter((candidate) => candidate.tabId === controller.tabId); return candidates.length === 1 ? candidates[0]! : null; @@ -255,6 +259,7 @@ export const useBrowserAutomationStore = create((set, ge liveTargets: new Map(), lifecycleTabs: new Map(), controllers: new Map(), + pointers: new Map(), activeRequests: new Map(), pendingAgentOpens: new Map(), actingTargets: new Set(), @@ -317,6 +322,8 @@ export const useBrowserAutomationStore = create((set, ge liveTargets.delete(key); const controllers = new Map(state.controllers); controllers.delete(key); + const pointers = new Map(state.pointers); + pointers.delete(key); const viewportCoordinators = new Map(state.viewportCoordinators); viewportCoordinators.delete(key); const viewportState = coordinator?.snapshot(); @@ -328,7 +335,7 @@ export const useBrowserAutomationStore = create((set, ge viewportByTarget.set(key, viewportState.confirmed); } } - return { liveTargets, controllers, viewportByTarget, viewportStateByTarget, viewportCoordinators }; + return { liveTargets, controllers, pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators }; }); }, unregisterTarget: (workspaceId, threadId, tabId) => { @@ -341,6 +348,8 @@ export const useBrowserAutomationStore = create((set, ge liveTargets.delete(key); const controllers = new Map(state.controllers); controllers.delete(key); + const pointers = new Map(state.pointers); + pointers.delete(key); const viewportByTarget = new Map(state.viewportByTarget); viewportByTarget.delete(key); const viewportStateByTarget = new Map(state.viewportStateByTarget); @@ -358,6 +367,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -390,6 +400,7 @@ export const useBrowserAutomationStore = create((set, ge set((state) => { const liveTargets = new Map(state.liveTargets); const controllers = new Map(state.controllers); + const pointers = new Map(state.pointers); const viewportByTarget = new Map(state.viewportByTarget); const viewportStateByTarget = new Map(state.viewportStateByTarget); const viewportCoordinators = new Map(state.viewportCoordinators); @@ -399,6 +410,7 @@ export const useBrowserAutomationStore = create((set, ge const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); liveTargets.delete(key); controllers.delete(key); + pointers.delete(key); viewportByTarget.delete(key); viewportStateByTarget.delete(key); viewportCoordinators.delete(key); @@ -409,6 +421,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -429,6 +442,7 @@ export const useBrowserAutomationStore = create((set, ge set((state) => { const liveTargets = new Map(state.liveTargets); const controllers = new Map(state.controllers); + const pointers = new Map(state.pointers); const viewportByTarget = new Map(state.viewportByTarget); const viewportStateByTarget = new Map(state.viewportStateByTarget); const viewportCoordinators = new Map(state.viewportCoordinators); @@ -438,6 +452,7 @@ export const useBrowserAutomationStore = create((set, ge const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); liveTargets.delete(key); controllers.delete(key); + pointers.delete(key); viewportByTarget.delete(key); viewportStateByTarget.delete(key); viewportCoordinators.delete(key); @@ -448,6 +463,7 @@ export const useBrowserAutomationStore = create((set, ge return { liveTargets, controllers, + pointers, viewportByTarget, viewportStateByTarget, viewportCoordinators, @@ -460,6 +476,17 @@ export const useBrowserAutomationStore = create((set, ge if (!target) return; get().setControllerForTarget(target.workspaceId, target.threadId, target.tabId, controller); }, + setPointer: (pointer) => { + const target = resolveBrowserAutomationControllerTarget(get().liveTargets.values(), pointer); + if (!target) return; + set((state) => { + const key = browserAutomationTargetKey(target.workspaceId, target.threadId, target.tabId); + const current = state.pointers.get(key); + // IPC delivers in order today; the guard keeps a late duplicate from rewinding the mark. + if (current && current.sequence >= pointer.sequence) return state; + return { pointers: new Map(state.pointers).set(key, pointer) }; + }); + }, setControllerForTarget: (workspaceId, threadId, tabId, controller) => set((state) => { const key = browserAutomationTargetKey(workspaceId, threadId, tabId); diff --git a/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx b/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx index b6f9a13af..d70792ece 100644 --- a/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx +++ b/apps/web/src/features/preview/surfaces/BrowserActingOverlay.tsx @@ -1,4 +1,6 @@ +import type { BrowserAutomationPointerEvent } from "@mcode/contracts"; import { MousePointer2 } from "lucide-react"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; import { browserAutomationTargetKey, useBrowserAutomationStore } from "../automation/browserAutomationStore"; import { useBrowserActing } from "../state/useBrowserActing"; @@ -7,6 +9,114 @@ const EDGE_BOX_SHADOW = [ "inset 0 0 28px color-mix(in oklab, var(--primary) 35%, transparent)", ].join(", "); +/** How long the agent cursor stays fully opaque after the agent's last point before it dims. */ +export const AGENT_CURSOR_ACTIVE_MS = 700; + +const AGENT_CURSOR_IDLE_OPACITY = 0.35; +// The pointer tip sits at (4, 4) in the 24-unit icon box; at 20px that is 3.3px in from the corner. +const AGENT_CURSOR_TIP_OFFSET_PX = 3.3; + +/** Where the hosted page is drawn inside the overlay: its top-left corner and its drawn-to-layout size ratio. */ +export interface AgentCursorPagePlacement { + readonly left: number; + readonly top: number; + readonly scale: number; +} + +/** + * Maps an agent point from page CSS pixels to overlay pixels. Page zoom enlarges CSS pixels + * inside the page, and a responsive viewport draws the whole page smaller or larger. + */ +export function mapAgentCursorPoint( + point: Pick, + page: AgentCursorPagePlacement, +): { readonly x: number; readonly y: number } { + return { + x: page.left + point.x * point.zoomFactor * page.scale, + y: page.top + point.y * point.zoomFactor * page.scale, + }; +} + +/** Measures the page against `origin`, an element covering the overlay, whose surface sits beside the overlay. */ +function measurePagePlacement(origin: HTMLElement, threadId: string, tabId: string): AgentCursorPagePlacement { + const surface = origin.closest('[data-testid="browser-acting-overlay"]')?.parentElement; + const page = [...(surface?.querySelectorAll('[data-testid="preview-webview"]') ?? [])] + .find((candidate) => candidate.dataset.threadId === threadId && candidate.dataset.tabId === tabId); + if (!page) return { left: 0, top: 0, scale: 1 }; + const overlayRect = origin.getBoundingClientRect(); + const pageRect = page.getBoundingClientRect(); + // offsetWidth ignores CSS transforms, so the ratio is the responsive viewport's scale. + const scale = page.offsetWidth > 0 ? pageRect.width / page.offsetWidth : 1; + return { left: pageRect.left - overlayRect.left, top: pageRect.top - overlayRect.top, scale }; +} + +interface AgentCursorProps { + readonly threadId: string; + readonly tabId: string; + readonly pointer: BrowserAutomationPointerEvent; +} + +/** Presentation-only mark at the agent's latest point; it never takes input or moves the OS pointer. */ +function AgentCursor({ threadId, tabId, pointer }: AgentCursorProps) { + const originRef = useRef(null); + const [position, setPosition] = useState<{ readonly x: number; readonly y: number } | null>(null); + const [idleSequence, setIdleSequence] = useState(null); + + useLayoutEffect(() => { + const origin = originRef.current; + if (!origin) return; + setPosition(mapAgentCursorPoint(pointer, measurePagePlacement(origin, threadId, tabId))); + }, [pointer, threadId, tabId]); + + useEffect(() => { + const timer = window.setTimeout(() => setIdleSequence(pointer.sequence), AGENT_CURSOR_ACTIVE_MS); + return () => window.clearTimeout(timer); + }, [pointer.sequence]); + + return ( +
+ {position && } +
+ ); +} + +function AgentCursorMark({ + position, + active, + pointer, +}: { + readonly position: { readonly x: number; readonly y: number }; + readonly active: boolean; + readonly pointer: BrowserAutomationPointerEvent; +}) { + return ( +
+ {pointer.phase === "click" && ( + + )} + +
+ ); +} + interface BrowserActingOverlayProps { readonly workspaceId: string; readonly threadId: string; @@ -15,13 +125,13 @@ interface BrowserActingOverlayProps { } /** - * Click-through edge wash and pointer shown over the page while the agent acts on the active + * Click-through edge wash and agent cursor shown over the page while the agent acts on the active * tab. It renders above the hosted page, which paints from the detached surface host root. */ export function BrowserActingOverlay({ workspaceId, threadId, tabId, coveredRight }: BrowserActingOverlayProps) { const acting = useBrowserActing(workspaceId, threadId, tabId); const pointer = useBrowserAutomationStore( - (state) => state.controllers.get(browserAutomationTargetKey(workspaceId, threadId, tabId))?.pointer ?? null, + (state) => state.pointers.get(browserAutomationTargetKey(workspaceId, threadId, tabId)) ?? null, ); if (!acting) return null; return ( @@ -36,16 +146,7 @@ export function BrowserActingOverlay({ workspaceId, threadId, tabId, coveredRigh Agent controls Browser - + {pointer && } ); } diff --git a/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx b/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx new file mode 100644 index 000000000..178f14aaa --- /dev/null +++ b/apps/web/src/features/preview/surfaces/__tests__/BrowserActingOverlay.test.tsx @@ -0,0 +1,92 @@ +import { act, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { BrowserAutomationControllerState, BrowserAutomationPointerEvent } from "@mcode/contracts"; +import { browserAutomationTargetKey, useBrowserAutomationStore } from "../../automation/browserAutomationStore"; +import { AGENT_CURSOR_ACTIVE_MS, BrowserActingOverlay, mapAgentCursorPoint } from "../BrowserActingOverlay"; + +const WORKSPACE = "workspace"; +const THREAD = "thread"; +const TAB = "tab"; +const KEY = browserAutomationTargetKey(WORKSPACE, THREAD, TAB); + +function pointer(phase: BrowserAutomationPointerEvent["phase"], sequence: number, zoomFactor = 1): BrowserAutomationPointerEvent { + return { tabId: TAB, phase, x: 100, y: 40, zoomFactor, sequence }; +} + +function setAgentState(controller: BrowserAutomationControllerState["controller"], point: BrowserAutomationPointerEvent | null) { + useBrowserAutomationStore.setState({ + controllers: new Map([[KEY, { tabId: TAB, controller, controlEpoch: 1 }]]), + actingTargets: new Set([KEY]), + pointers: new Map(point ? [[KEY, point]] : []), + }); +} + +/** Renders the overlay beside a page placed at (12, 8) and drawn at half size, as a responsive viewport does. */ +function renderOverPlacedPage() { + const view = render( +
+
+ +
, + ); + const page = view.getByTestId("preview-webview"); + Object.defineProperty(page, "offsetWidth", { configurable: true, value: 800 }); + page.getBoundingClientRect = () => new DOMRect(12, 8, 400, 300); + return view; +} + +describe("mapAgentCursorPoint", () => { + it("scales a page CSS point by page zoom and the drawn page size, then offsets it by the page position", () => { + expect(mapAgentCursorPoint({ x: 100, y: 40, zoomFactor: 1.5 }, { left: 12, top: 8, scale: 0.5 })).toEqual({ x: 87, y: 38 }); + }); +}); + +describe("BrowserActingOverlay agent cursor", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + useBrowserAutomationStore.setState({ controllers: new Map(), actingTargets: new Set(), pointers: new Map() }); + }); + + it("lands on the agent's point through page zoom and the responsive page scale", () => { + setAgentState("agent", null); + renderOverPlacedPage(); + expect(screen.queryByTestId("browser-automation-pointer")).not.toBeInTheDocument(); + + act(() => setAgentState("agent", pointer("move", 1, 1.5))); + + expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ transform: "translate3d(87px, 38px, 0)" }); + }); + + it("pings on a click but not on a move", () => { + setAgentState("agent", pointer("move", 1)); + renderOverPlacedPage(); + expect(screen.queryByTestId("browser-automation-pointer-ping")).not.toBeInTheDocument(); + + act(() => setAgentState("agent", pointer("click", 2))); + + expect(screen.getByTestId("browser-automation-pointer-ping")).toBeInTheDocument(); + }); + + it("dims once the agent stops moving", () => { + setAgentState("agent", pointer("move", 1)); + renderOverPlacedPage(); + expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ opacity: "1" }); + + act(() => { + vi.advanceTimersByTime(AGENT_CURSOR_ACTIVE_MS); + }); + + expect(screen.getByTestId("browser-automation-pointer")).not.toHaveStyle({ opacity: "1" }); + }); + + it("hides the cursor while the user holds control", () => { + setAgentState("human", pointer("click", 1)); + renderOverPlacedPage(); + + expect(screen.queryByTestId("browser-automation-pointer")).not.toBeInTheDocument(); + }); +}); diff --git a/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx b/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx index 82c3d0b52..7da2ba0ee 100644 --- a/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx +++ b/apps/web/src/features/preview/surfaces/__tests__/PreviewPanel.test.tsx @@ -356,13 +356,12 @@ describe("PreviewPanel: unavailable state", () => { const key = browserAutomationTargetKey(threadId, threadId, "web-preview"); useDiffStore.setState({ previewUrlByThread: { [threadId]: window.location.origin + "/fixture" } }); useBrowserAutomationStore.setState({ - controllers: new Map([[key, { tabId: "web-preview", controller: "agent", controlEpoch: 1, pointer: { x: 40, y: 50 } }]]), + controllers: new Map([[key, { tabId: "web-preview", controller: "agent", controlEpoch: 1 }]]), actingTargets: new Set([key]), }); try { render(); expect(screen.getByTestId("browser-acting-overlay")).toHaveStyle({ clipPath: "inset(0 112px 0 0)" }); - expect(screen.getByTestId("browser-automation-pointer")).toHaveStyle({ left: "40px", top: "50px" }); } finally { useBrowserAutomationStore.setState({ controllers: new Map(), actingTargets: new Set() }); } @@ -836,7 +835,7 @@ describe("PreviewPanel: full panel state", () => { expect(overlay).toHaveStyle({ clipPath: "inset(0 112px 0 0)" }); }); - it("shows the agent frame and cursor while the active page is still opening", () => { + it("shows the agent frame while the active page is still opening", () => { useBrowserAutomationStore.setState({ pendingAgentOpens: new Map([ [ @@ -855,7 +854,6 @@ describe("PreviewPanel: full panel state", () => { render(); expect(screen.getByTestId("browser-acting-overlay")).toBeInTheDocument(); - expect(screen.getByTestId("browser-automation-pointer")).toBeInTheDocument(); }); it("paints no agent overlay while the agent holds the page but is idle", () => { diff --git a/apps/web/src/index.css b/apps/web/src/index.css index 0ee7c0c87..2eb77e21a 100644 --- a/apps/web/src/index.css +++ b/apps/web/src/index.css @@ -686,6 +686,23 @@ diffs-container:has([data-jump-highlight="true"]) { } } +/* One ring per agent click; it ends transparent so nothing keeps animating while the agent is idle. + The delay matches the mark's 150ms travel, so the ring appears where the click lands. */ +@keyframes agent-cursor-ping { + from { opacity: 0.9; transform: scale(0.4); } + to { opacity: 0; transform: scale(1.8); } +} +.animate-agent-cursor-ping { + opacity: 0; + animation: agent-cursor-ping 450ms ease-out 150ms forwards; +} +@media (prefers-reduced-motion: reduce) { + .animate-agent-cursor-ping { + animation: none; + opacity: 0; + } +} + /* Tailwind's looping utilities (loading dots, spinners) are not covered by the named-animation blocks above, so reduced-motion must settle them explicitly. Freezing leaves a static-but-visible loading affordance. */ diff --git a/apps/web/src/transport/desktop-bridge.d.ts b/apps/web/src/transport/desktop-bridge.d.ts index 2f62e1252..8f43e1e8c 100644 --- a/apps/web/src/transport/desktop-bridge.d.ts +++ b/apps/web/src/transport/desktop-bridge.d.ts @@ -310,6 +310,10 @@ export interface PreviewAutomationBridge { } | { ok: false; error: string } >; + /** Draws the agent cursor inside one tab's page while the renderer records it; false when the tab is unavailable. */ + setRecordingCursor(target: { threadId: string; tabId: string; enabled: boolean }): Promise; + /** Subscribe to agent cursor points, reported before each agent pointer action; validate with BrowserAutomationPointerEventSchema. */ + onPointer(callback: (pointer: unknown) => void): () => void; /** Subscribe to controller transitions for adopted tabs in this window. */ onControllerChanged(callback: (state: BrowserAutomationControllerState) => void): () => void; /** Subscribe to Browser host heartbeat pulses owned by Electron main. */ diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 1bfbbb5ac..7319abe0d 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -742,6 +742,7 @@ export { BrowserAutomationSnapshotSchema, BrowserAutomationPerformanceMetricsSchema, BrowserAutomationControllerStateSchema, + BrowserAutomationPointerEventSchema, BrowserAutomationHostCapabilitySchema, BrowserAutomationHostRegistrationSchema, BrowserAutomationHostDispatchTargetSchema, @@ -776,6 +777,7 @@ export type { BrowserAutomationTruncation, BrowserAutomationPerformanceMetrics, BrowserAutomationControllerState, + BrowserAutomationPointerEvent, BrowserAutomationHostRegistration, BrowserAutomationHostDispatchTarget, BrowserAutomationHostDispatch, diff --git a/packages/contracts/src/models/browser-automation.ts b/packages/contracts/src/models/browser-automation.ts index e590f566e..c3ad4072d 100644 --- a/packages/contracts/src/models/browser-automation.ts +++ b/packages/contracts/src/models/browser-automation.ts @@ -622,10 +622,6 @@ export const BrowserAutomationControllerStateSchema = lazySchema(() => controlEpoch: z.number().int().nonnegative(), providerSessionId: idSchema.optional(), operation: browserRequestOperationSchema.optional(), - pointer: z.object({ - x: z.number().finite().min(0).max(100_000), - y: z.number().finite().min(0).max(100_000), - }).strict().optional(), }) .strict(), ); @@ -635,6 +631,31 @@ export type BrowserAutomationControllerState = z.infer< ReturnType >; +/** + * Point the agent is about to move to or click, reported before the input is dispatched. + * `x` and `y` are CSS pixels relative to the page viewport, so they already account for page + * scroll; `zoomFactor` is the page zoom at that moment, which a host overlay needs to place the + * point in its own pixels. `sequence` increases with every event so a repeated click on the same + * point still reads as new. + */ +export const BrowserAutomationPointerEventSchema = lazySchema(() => + z + .object({ + tabId: idSchema, + phase: z.enum(["move", "click"]), + x: z.number().finite().min(0).max(100_000), + y: z.number().finite().min(0).max(100_000), + zoomFactor: z.number().finite().positive().max(10), + sequence: z.number().int().nonnegative(), + }) + .strict(), +); + +/** One agent cursor point for a Browser tab. */ +export type BrowserAutomationPointerEvent = z.infer< + ReturnType +>; + /** Runtime that owns one browser automation host connection. */ export const BrowserAutomationHostRuntimeSchema = z.enum(["electron", "web"]); /** Browser automation host runtime discriminator. */