diff --git a/apps/extension/src/content/__tests__/cooperation-capture.test.ts b/apps/extension/src/content/__tests__/cooperation-capture.test.ts index e67f0c18..8f0b3e21 100644 --- a/apps/extension/src/content/__tests__/cooperation-capture.test.ts +++ b/apps/extension/src/content/__tests__/cooperation-capture.test.ts @@ -20,9 +20,10 @@ afterEach(() => { vi.restoreAllMocks(); }); -function collector() { +function collector(shadowRoot?: (element: HTMLElement) => ShadowRoot | null) { let receive!: (message: unknown, sender: unknown, reply: (result: unknown) => void) => void; vi.stubGlobal("chrome", { + dom: shadowRoot ? { openOrClosedShadowRoot: shadowRoot } : undefined, runtime: { sendMessage: vi.fn(async () => {}), onMessage: { @@ -404,3 +405,109 @@ it("announces renderer readiness once after the collection listener exists", () expect(chrome.runtime.sendMessage).toHaveBeenCalledWith({ kind: COOPERATION_READY }); expect(read()).toHaveProperty("document"); }); + +it("classifies a non-submitted media timeline without masking business fields", () => { + document.body.innerHTML = + ''; + const read = collector(); + const before = read()!; + expect(before.fields[0]).toMatchObject({ value: "1", kind: "media_progress" }); + expect(before.fields.slice(1).every((field) => field.kind === undefined)).toBe(true); + const progress = document.querySelector("#timeline")!; + progress.value = "2"; + expect(read()!.fields[0]).toMatchObject({ value: "2", kind: "media_progress" }); + progress.name = "order_amount"; + expect(read()!.fields[0].kind).toBeUndefined(); +}); + +it.each([ + "form", + "external", + "ambiguous", + "missing", +])("does not classify a %s timeline as incidental", (kind) => { + document.body.innerHTML = + ''; + const timeline = document.querySelector("#timeline")!; + if (kind === "form") { + const form = document.createElement("form"); + document.body.append(form); + form.append(timeline); + } + if (kind === "external") { + const form = document.createElement("form"); + form.id = "order"; + document.body.append(form); + timeline.setAttribute("form", "order"); + } + if (kind === "ambiguous") document.body.append(document.createElement("video")); + if (kind === "missing") document.querySelector("video")!.remove(); + expect(collector()()!.fields[0].kind).toBeUndefined(); +}); + +it("reports programmatic focus changes even without a human event", () => { + document.body.innerHTML = ''; + const read = collector(); + const first = read()!.focus; + document.querySelector("#order")!.focus(); + const current = read()!; + expect(current.focus).not.toBe(first); + expect(current.human).toBe(0); +}); + +it("records media sources independently of playback time, including runtime-only source changes", () => { + document.body.innerHTML = ''; + const video = document.querySelector("video")!; + const source = video.querySelector("source")!; + const read = collector(); + const first = read()!; + video.currentTime = 1; + expect(read()!.media).toEqual(first.media); + source.src = "two.webm"; + const changed = read()!; + expect(changed.media).not.toEqual(first.media); + expect(changed.revision).toBeGreaterThan(first.revision); + Object.defineProperty(video, "currentSrc", { configurable: true, value: "selected.webm" }); + const selected = read()!; + expect(selected.media![0].currentSrc).toBe("selected.webm"); + expect(selected.revision).toBeGreaterThan(changed.revision); + Object.defineProperty(video, "srcObject", { configurable: true, value: {} }); + const stream = read()!; + expect(stream.media![0].srcObject).toEqual(expect.any(String)); + Object.defineProperty(video, "srcObject", { configurable: true, value: {} }); + expect(read()!.media![0].srcObject).not.toBe(stream.media![0].srcObject); +}); + +it("collects media and deep focus in open shadow roots", () => { + document.body.innerHTML = '
'; + const root = document.querySelector("#host")!.attachShadow({ mode: "open" }); + root.innerHTML = ''; + const read = collector(); + expect(read()!.focusIsDocument).toBe(true); + root.querySelector("button")!.focus(); + const focused = read()!; + expect(focused.focusIsDocument).toBe(false); + expect(focused.media).toHaveLength(1); + root.querySelector("source")!.src = "two.ogg"; + expect(read()!.media).not.toEqual(focused.media); +}); + +it("collects source and recipient changes inside closed shadow roots", () => { + document.body.innerHTML = '
'; + const host = document.querySelector("#host")!; + const root = host.attachShadow({ mode: "closed" }); + root.innerHTML = + ''; + const read = collector((element) => (element === host ? root : null)); + const buttons = root.querySelectorAll("button"); + buttons[0].focus(); + const first = read()!; + expect(host.shadowRoot).toBeNull(); + expect(first.media).toHaveLength(1); + buttons[1].focus(); + root.querySelector("source")!.src = "two.webm"; + const changed = read()!; + expect(changed.focus).not.toBe(first.focus); + expect(changed.media).not.toEqual(first.media); + expect(changed.revision).toBeGreaterThan(first.revision); +}); diff --git a/apps/extension/src/content/cooperation-capture.ts b/apps/extension/src/content/cooperation-capture.ts index 77cf2c99..98fa1a81 100644 --- a/apps/extension/src/content/cooperation-capture.ts +++ b/apps/extension/src/content/cooperation-capture.ts @@ -1,3 +1,4 @@ +import { playbackProgressMedia } from "@/cooperation/media-progress"; import { COOPERATION_ARM, COOPERATION_DISABLE, @@ -42,9 +43,9 @@ export function attachCooperationCapture() { const keys = new Set(); const fields = new Map(); const sampledValues = new WeakMap(); - const ids = new WeakMap(); + const ids = new WeakMap(); let nextId = 0; - const id = (element: Element) => { + const id = (element: object) => { let value = ids.get(element); if (!value) ids.set(element, (value = String(++nextId))); return value; @@ -101,6 +102,20 @@ export function attachCooperationCapture() { }; const observer = new MutationObserver(mutations); const roots = new Set(); + // Page JS cannot see closed roots, but the extension can. Observation also + // traverses these roots, so source and focus guards must cover the same DOM. + const shadowRoot = (element: Element): ShadowRoot | null => + element.shadowRoot ?? + (element instanceof HTMLElement ? (chrome.dom?.openOrClosedShadowRoot(element) ?? null) : null); + const activeElement = () => { + let active = document.activeElement; + while (active) { + const nested = shadowRoot(active)?.activeElement; + if (!nested) break; + active = nested; + } + return active; + }; const controls = function* (root: Document | ShadowRoot): Generator { if (!roots.has(root)) { observer.observe(root, { @@ -114,17 +129,24 @@ export function attachCooperationCapture() { for (const element of root.querySelectorAll("*")) { if (overlay(element)) continue; if (element.matches("input,textarea,select,[contenteditable=true]")) yield element; - if (element.shadowRoot) yield* controls(element.shadowRoot); + const nested = shadowRoot(element); + if (nested) yield* controls(nested); } }; const read = () => { mutations(observer.takeRecords()); + const active = activeElement(); + state.focus = active ? id(active) : undefined; + state.focusIsDocument = + (active === document.body || active === document.documentElement) && + !active?.matches('[tabindex],[role],[contenteditable]:not([contenteditable="false"])'); if ([...roots].some((root) => root instanceof ShadowRoot && !root.host.isConnected)) { observer.disconnect(); roots.clear(); } for (const element of fields.keys()) if (!element.isConnected) fields.delete(element); let fieldCount = 0; + const mediaByRoot = new Map(); state.fieldsComplete = true; for (const target of controls(document)) { if ( @@ -140,6 +162,34 @@ export function attachCooperationCapture() { : (target.textContent ?? ""); const value = fullValue.slice(0, 500); const name = label(target); + let mediaProgress = false; + if (target instanceof HTMLInputElement && target.type === "range") { + const root = target.getRootNode() as Document | ShadowRoot; + let media = mediaByRoot.get(root); + if (!media) { + media = [...root.querySelectorAll("video,audio")].map((element) => ({ id: element.id })); + mediaByRoot.set(root, media); + } + const labelledBy = (target.getAttribute("aria-labelledby") ?? "") + .split(/\s+/) + .filter(Boolean) + .map((id) => root.getElementById(id)?.textContent ?? "") + .join(" "); + const labels = [...new Set(target.labels ?? [])] + .map((label) => label.textContent) + .join(" "); + mediaProgress = !!playbackProgressMedia( + { + tag: "input", + name: target.getAttribute("aria-label") || labelledBy || labels, + attrs: Object.fromEntries( + [...target.attributes].map((attr) => [attr.name, attr.value]), + ), + inForm: target.form !== null, + }, + media, + ); + } fieldCount++; if (fieldCount > MAX_CHANGES || fullValue.length > 500) state.fieldsComplete = false; const sampled = sampledValues.get(target); @@ -147,9 +197,44 @@ export function attachCooperationCapture() { sampledValues.set(target, fullValue); const old = fields.get(target); if (fields.size < MAX_CHANGES || old) - fields.set(target, { id: id(target), label: name, value }); + fields.set(target, { + id: id(target), + label: name, + value, + ...(mediaProgress ? { kind: "media_progress" as const } : {}), + }); } for (const element of fields.keys()) if (!element.isConnected) fields.delete(element); + const media = [...roots].flatMap((root) => + [...root.querySelectorAll("video,audio")].map((element) => { + const stream = element.srcObject; + return { + id: id(element), + src: element.getAttribute("src"), + currentSrc: element.currentSrc, + srcObject: stream ? id(stream) : null, + tracks: + typeof MediaStream !== "undefined" && stream instanceof MediaStream + ? stream + .getTracks() + .map((track) => track.id) + .sort() + : [], + sources: [...element.children] + .filter((child): child is HTMLSourceElement => child instanceof HTMLSourceElement) + .map((source) => ({ + id: id(source), + src: source.src, + type: source.type, + media: source.media, + })), + }; + }), + ); + // currentSrc/srcObject may change without a DOM mutation. Keep the source + // evidence fresh even when the page's mutation counter did not advance. + if (state.media && JSON.stringify(state.media) !== JSON.stringify(media)) state.revision++; + state.media = media; state.busy = composing || pointer || keys.size > 0; state.fields = [...fields.values()]; return { ...state }; @@ -219,8 +304,7 @@ export function attachCooperationCapture() { clearTimeout(focusTimer); focusTimer = setTimeout(() => { if (!enabled) return; - let active = document.activeElement; - while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement; + const active = activeElement(); if (state.editing) state.revision++; state.editing = state.editing && editable(active); composing = false; diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 335d748a..a5b2090d 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -1,4 +1,4 @@ -import type { VomScene } from "@browser-skill/vom"; +import type { VomNode, VomScene } from "@browser-skill/vom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; @@ -39,6 +39,7 @@ async function fixture(options: { url?: string; sync?: boolean } = {}) { let now = 10000; let frame: FrameState = { document: "doc-a", + focusIsDocument: true, revision: 0, structure: 0, human: 0, @@ -860,6 +861,80 @@ describe("shared-tab dispatch fences", () => { expect(f.native).not.toHaveBeenCalled(); }); + it.each([ + ["e5"], + ["e4", "e5"], + ["e5", "e4"], + ])("keeps every untargeted key dependency exact after learning: %j", async (...dependsOn) => { + live((tick) => scene(`10:00:0${tick}`)); + refs(); + const f = await fixture(); + expect((await run(f, f.first.context_id, { ref: "e4" })).message).toContain( + "changed by itself", + ); + expect( + await run(f, (await f.command("sync")).context_id, { + key: "Space", + dependsOn, + }), + ).toMatchObject({ message: changed, data: { effect_state: "none" } }); + expect(f.native).not.toHaveBeenCalled(); + }); + + it.each([ + "focused", + "unobserved", + "ambiguous", + ])("protects %s keyboard recipients independently of declared dependencies", async (focus) => { + live( + page((tick) => [ + [1, null, "main"], + [2, 1, "form"], + [4, 2, "button", { name: "Pause" }], + [6, 1, "form"], + [7, 6, "span", { text: `$${100 + tick}` }], + [8, 6, "button", { name: "Buy", focused: focus !== "unobserved" }], + ...(focus === "ambiguous" + ? [[9, 1, "button", { name: "Other", focused: true }] as Spec] + : []), + ]), + ); + refs(); + const f = await fixture({ sync: false }); + f.change({ focus: "buy", focusIsDocument: false }, false); + const synced = await f.command("sync"); + expect(await run(f, synced.context_id, { key: "Enter", dependsOn: ["e4"] })).toMatchObject({ + message: changed, + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + }); + + it("keeps untargeted scripts strict even when they declare unrelated dependencies", async () => { + live((tick) => scene(`10:00:0${tick}`)); + refs(); + const f = await fixture(); + f.change({ structure: 100 }); + const result = await f.controller.run( + { + id: "script", + method: "tool.evaluate", + params: { + session_id: "shared", + tab_id: 7, + context_id: f.first.context_id, + expression: "buy()", + depends_on: ["e4"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Runtime.evaluate", { expression: "buy()" }), + ); + expect(result).toMatchObject({ message: changed, data: { effect_state: "none" } }); + expect(f.native).not.toHaveBeenCalled(); + }); + it("does not let a refused context retry with newly learned tolerance", async () => { live((tick) => scene(`10:00:0${tick}`)); const f = await fixture(); @@ -1006,6 +1081,13 @@ describe("shared-tab dispatch fences", () => { f.first.context_id, async (runner) => { await runner.send(7, "Page.navigate", { url: receipt.final_url }); + // The completed navigation may replace media before the old document + // disappears; that is its effect, not grounds to revoke its receipt. + f.change({ + media: [ + { id: "next", src: "next.webm", currentSrc: "next.webm", srcObject: null, sources: [] }, + ], + }); f.controller.invalidate(7); f.change({ document: "doc-b" }); return receipt; @@ -2161,3 +2243,416 @@ describe("default collaboration preserves the original unattended lifetime", () expect(f.sessionActivity).toHaveBeenLastCalledWith("shared", false); }); }); + +describe("looping preview action guards", () => { + type PreviewNode = Pick< + VomNode, + "id" | "parentId" | "tag" | "role" | "name" | "attrs" | "value" | "inForm" + >; + const previewScene = (seconds: number, range = false): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: ( + [ + { id: 1, parentId: null, tag: "main", role: "main", name: "" }, + { id: 2, parentId: 1, tag: "form", role: "form", name: "Player controls" }, + { id: 4, parentId: 2, tag: "button", role: "button", name: "Pause" }, + { + id: 6, + parentId: 1, + tag: "video", + role: "Video", + name: "Preview", + attrs: { id: "video", src: "clip.webm" }, + }, + range + ? { + id: 5, + parentId: 1, + tag: "input", + role: "slider", + name: "Playback progress", + inForm: false, + value: String(seconds), + attrs: { type: "range" }, + } + : { id: 5, parentId: 1, tag: "aside", role: "region", name: `Playback time: ${seconds}` }, + ] as PreviewNode[] + ).map((node) => ({ + ...node, + backendNodeId: node.id, + frameId: "frame", + rect: { x: 0, y: 0, w: 20, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + })), + }); + it.each([ + { + name: "untargeted Space is rejected by unrelated timer text", + ref: false, + deps: false, + range: false, + changes: true, + allowed: false, + }, + { + name: "targeted Space with declared dependencies permits outside timer text", + ref: true, + deps: true, + range: false, + changes: true, + allowed: true, + }, + { + name: "untargeted Space honors its declared dependencies", + ref: false, + deps: true, + range: false, + changes: true, + allowed: true, + }, + { + name: "outside playback range does not revoke a scoped action", + ref: true, + deps: true, + range: true, + changes: true, + allowed: true, + }, + { + name: "revision changes with unchanged semantics permit Space", + ref: false, + deps: false, + range: false, + changes: false, + allowed: true, + }, + ])("$name", async ({ ref, deps, range, changes, allowed }) => { + vi.mocked(handleSnapshot).mockImplementationOnce(async (_m, _p, options) => { + options!.onScene?.(previewScene(1, range)); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + vi.mocked(captureVomObservation).mockImplementation(async (_c, _t, _u, options) => { + options?.onScene?.(previewScene(changes ? 2 : 1, range)); + return {} as Awaited>; + }); + const f = await fixture(); + f.change({ structure: 1 }); + // A ready coordination state does not mean page semantics stopped changing. + expect(await f.command("wait")).toMatchObject({ ready: true }); + const result = await f.controller.run( + { + id: "preview-space", + method: "tool.press", + params: { + session_id: "shared", + tab_id: 7, + context_id: f.first.context_id, + key: "Space", + ...(ref ? { ref: "e4" } : {}), + ...(deps ? { depends_on: ["e4"] } : {}), + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }), + ); + if (allowed) { + expect(result).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); + } else { + expect(result).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + } + if (deps) expect(resolveActionTarget).toHaveBeenCalled(); + else if (!ref) expect(resolveActionTarget).not.toHaveBeenCalled(); + }); +}); + +it.each([ + undefined, + "media_progress" as const, +])("sync tolerates only classified media progress: %s", async (kind) => { + const f = await fixture({ sync: false }); + const field = { + id: "timeline", + label: "Playback progress", + value: "1", + ...(kind ? { kind } : {}), + }; + f.change({ fields: [field] }, false); + vi.mocked(handleSnapshot).mockImplementationOnce(async () => { + f.change({ fields: [{ ...field, value: "2" }] }, false); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + const result = await f.command("sync"); + if (kind) expect(result.context_id).toEqual(expect.any(String)); + else + expect(result).toMatchObject({ + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); +}); + +it("rejects untargeted key dependencies that cannot be resolved", async () => { + const f = await fixture(); + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + code: "not_found", + message: "Dependency disappeared", + }); + const result = await f.controller.run( + { + id: "key", + method: "tool.press", + params: { + session_id: "shared", + context_id: f.first.context_id, + key: "Space", + depends_on: ["#gone"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }), + ); + expect(result).toMatchObject({ code: "not_found" }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("does not retarget an untargeted key when page code changes focus", async () => { + const f = await fixture(); + const result = await f.act( + f.first.context_id, + async (runner) => { + f.change({ focus: "order-amount" }, false); + return runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }); + }, + "tool.press", + ); + expect(result).toMatchObject({ + message: "The keyboard focus changed; synchronize before pressing a key", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("allows focus changes caused by a key that has already started", async () => { + const f = await fixture(); + const result = await f.act( + f.first.context_id, + async (runner) => { + await runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }); + f.change({ focus: "next-field" }, false); + await runner.send(7, "Input.dispatchKeyEvent", { type: "char", key: " ", text: " " }); + await runner.send(7, "Input.dispatchKeyEvent", { type: "keyUp", key: " " }); + return { ok: true }; + }, + "tool.press", + ); + expect(result).toEqual({ ok: true }); + expect(f.native).toHaveBeenCalledTimes(3); +}); + +it("refreshes timeline evidence on a delta when that value is an explicit decision dependency", async () => { + const scene = (value: string): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: [ + { + id: 4, + parentId: null, + backendNodeId: 4, + frameId: "frame", + tag: "input", + role: "slider", + name: "Playback progress", + value, + attrs: { type: "range" }, + rect: { x: 0, y: 0, w: 100, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + const f = await fixture({ sync: false }); + const field = { + id: "timeline", + label: "Playback progress", + kind: "media_progress" as const, + value: "1", + }; + f.change({ fields: [field] }, false); + vi.mocked(handleSnapshot).mockImplementationOnce(async (_m, _p, deps) => { + deps!.onScene?.(scene("1")); + return { text: "Playback progress 1", ref_count: 1, tab_id: 7, truncated: false }; + }); + await f.command("sync"); + f.change({ fields: [{ ...field, value: "2" }] }, false); + vi.mocked(captureVomObservation).mockImplementation(async (_c, _t, _u, options) => { + options?.onScene?.(scene("2")); + return {} as Awaited>; + }); + const synced = await f.command("sync"); + expect(synced).toMatchObject({ + kind: "delta", + changes: [expect.objectContaining({ value: "2" })], + }); + expect(captureVomObservation).toHaveBeenCalledOnce(); + f.change({ structure: 1 }); + const result = await f.controller.run( + { + id: "timeline-key", + method: "tool.press", + params: { + session_id: "shared", + context_id: synced.context_id, + key: "ArrowRight", + ref: "e4", + depends_on: ["e4"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: "ArrowRight" }), + ); + expect(result).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); +}); + +it("revokes actions on media source changes even without DOM revision changes", async () => { + const f = await fixture({ sync: false }); + const media = { + id: "video", + src: null, + currentSrc: "one.webm", + srcObject: null, + sources: [{ id: "source", src: "one.webm", type: "video/webm", media: "" }], + }; + f.change({ media: [media] }, false); + const synced = await f.command("sync"); + // A browser source selection can change currentSrc without a DOM mutation. + const rev = f.readFrames.mock.results.at(-1)!.value; + const frames = await rev; + f.change( + { media: [{ ...media, currentSrc: "two.webm" }], revision: frames.get(0)!.revision }, + false, + ); + expect(await f.act(synced.context_id)).toMatchObject({ + message: "The media source changed; synchronize and replan", + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("requires a fresh sync when a media source changes during observation", async () => { + const f = await fixture({ sync: false }); + f.change( + { + media: [ + { id: "video", src: "one.webm", currentSrc: "one.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + vi.mocked(handleSnapshot).mockImplementationOnce(async () => { + f.change( + { + media: [ + { id: "video", src: "two.webm", currentSrc: "two.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + expect(await f.command("sync")).toMatchObject({ + message: "Page changed while synchronizing; sync again after it settles", + }); +}); + +it("does not revoke native input already started when that input switches the media", async () => { + const f = await fixture(); + expect( + await f.act(f.first.context_id, async (runner) => { + await runner.send(7, "Input.dispatchMouseEvent", { type: "mousePressed" }); + f.change( + { + media: [ + { id: "next", src: "two.webm", currentSrc: "two.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + await runner.send(7, "Input.dispatchMouseEvent", { type: "mouseReleased" }); + return { ok: true }; + }), + ).toEqual({ ok: true }); +}); + +it.each([ + "focus", + "media", +])("consumes a context refused for %s even if the page restores the old state", async (kind) => { + const f = await fixture(); + f.change( + kind === "focus" + ? { focus: "other" } + : { + media: [ + { + id: "video", + src: "other.webm", + currentSrc: "other.webm", + srcObject: null, + sources: [], + }, + ], + }, + false, + ); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + data: { effect_state: "none" }, + }); + f.change({ focus: undefined, media: [] }, false); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + message: "This action requires the context returned by the latest cooperate sync", + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it.each([ + "focus", + "media", +])("refreshes semantic evidence when %s changes before a delta sync", async (kind) => { + vi.mocked(handleSnapshot).mockResolvedValue({ + text: "Pause", + ref_count: 1, + tab_id: 7, + truncated: false, + }); + const f = await fixture(); + vi.mocked(handleSnapshot).mockClear(); + f.change( + kind === "focus" + ? { focus: "pause", focusIsDocument: false } + : { + media: [{ id: "video", src: null, currentSrc: "two.webm", srcObject: null, sources: [] }], + }, + false, + ); + expect(await f.command("sync")).toHaveProperty("observation"); + expect(handleSnapshot).toHaveBeenCalledOnce(); +}); + +it("rejects a document that becomes a specific keyboard recipient without changing its identity", async () => { + const f = await fixture(); + f.change({ focusIsDocument: false }, false); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + message: "The keyboard focus changed; synchronize before pressing a key", + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 4366b7a8..aed05249 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -14,6 +14,7 @@ import { contentChanges, type ObservationEvidence, observationEvidence, + observedKeyboardTargets, observesTarget, sameObservationEvidence, } from "./observation-evidence"; @@ -138,14 +139,40 @@ function quote(text: string): string { const points = Array.from(text); return JSON.stringify(points.length > 40 ? `${points.slice(0, 39).join("")}…` : text); } -function sameFields(a: Frames, b: Frames): boolean { +function sameFields(a: Frames, b: Frames, ignoreMediaProgress = false): boolean { + const stableFields = (frame: FrameState | undefined) => + frame?.fields.map((field) => + ignoreMediaProgress && field.kind === "media_progress" + ? { ...field, value: undefined } + : field, + ); return ( a.size === b.size && [...a].every( - ([id, frame]) => JSON.stringify(frame.fields) === JSON.stringify(b.get(id)?.fields), + ([id, frame]) => + JSON.stringify(stableFields(frame)) === JSON.stringify(stableFields(b.get(id))), ) ); } +function sameFocus(a: Frames, b: Frames): boolean { + return ( + a.size === b.size && + [...a].every(([id, frame]) => { + const other = b.get(id); + return frame.focus === other?.focus && frame.focusIsDocument === other?.focusIsDocument; + }) + ); +} + +function sameMedia(a: Frames, b: Frames): boolean { + return ( + a.size === b.size && + [...a].every( + ([id, frame]) => JSON.stringify(frame.media ?? []) === JSON.stringify(b.get(id)?.media ?? []), + ) + ); +} + function failure(message: string, effect: "none" | "unknown" = "none"): RpcError { return { code: "permission_denied", @@ -576,7 +603,9 @@ export class CooperationController implements ControlAuthority { baseline && [...before.values()].every((frame) => frame.fieldsComplete !== false) && baseline.refs === session.refStore.revision && - sameFrames(baseline.frames, before, "structure"); + sameFrames(baseline.frames, before, "structure") && + sameFocus(baseline.frames, before) && + sameMedia(baseline.frames, before); // A delta updates the model's field values. Refresh private semantic // evidence too, without replacing refs or repeating page text to the model. if (delta && !sameFields(baseline.frames, before)) @@ -608,7 +637,8 @@ export class CooperationController implements ControlAuthority { signal.aborted || state.epoch !== epoch || !sameFrames(before, after, "human") || - !sameFields(before, after) || + !sameFields(before, after, true) || + !sameMedia(before, after) || (delta && !sameFrames(before, after, "structure")) || this.humanActive(state) || state.held @@ -686,7 +716,7 @@ export class CooperationController implements ControlAuthority { }), state: { ...this.view(state.tabId), mode: "ready" }, guidance: - "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or selector always check the whole page exactly. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", + "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Keys without a ref or selector can use explicit dependencies, while their actual focused region stays protected; uncertain focus and other untargeted actions keep the exact whole-page check. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", }; } finally { state.working = false; @@ -1013,13 +1043,32 @@ export class CooperationController implements ControlAuthority { if (denied) return denied.message; if (!settledNavigation && state.unavailable) return "Page interaction coverage changed; wait and synchronize"; + if ( + req.method === "tool.press" && + !nativeInputStarted && + !params.ref && + !params.selector && + context && + !sameFocus(context.frames, state.frames) + ) + return "The keyboard focus changed; synchronize before pressing a key"; + if ( + !settledNavigation && + !nativeInputStarted && + context && + !sameMedia(context.frames, state.frames) + ) + return "The media source changed; synchronize and replan"; if (!settledNavigation && state.epoch !== epoch) return "The observed page or control state changed; synchronize"; }; const assert = () => { signal.throwIfAborted(); const reason = interruption(); - if (reason) throw new Error((revoked = reason)); + if (reason) { + if (context && state.context === context) state.context = undefined; + throw new Error((revoked = reason)); + } }; const verifyEvidence = async () => { if (!context || !verifiedFrames) throw new Error((revoked = "Synchronize before acting")); @@ -1050,10 +1099,9 @@ export class CooperationController implements ControlAuthority { ); } // Learned text is tolerated only around a resolved target present in both - // observations, whose region can be protected. A target-less key press or - // script acts on whatever is focused, and a selector may resolve outside - // the observation (for example inside aria-hidden content); both keep the - // exact whole-page check. + // observations, whose region can be protected. Untargeted actions never + // learn text. Keys with explicit dependencies additionally protect their + // actual focus; other untargeted actions keep the exact whole-page check. const anchored = actionTarget !== undefined && observesTarget(context.evidence, actionTarget) && @@ -1064,12 +1112,37 @@ export class CooperationController implements ControlAuthority { tolerant && state.volatile?.documents === pages ? state.volatile.keys : undefined; // The target's own semantic region and every declared dependency stay // exact; sameObservationEvidence only relaxes broad layout around a target. + const keyboardRegionMatches = () => { + if (actionTarget) return true; + // Only keyboard input has an implicit recipient. An untargeted script + // still requires the exact whole page, as before scoped-key support. + if (req.method !== "tool.press") + return sameObservationEvidence(context.evidence!, evidence!); + const recipients = observedKeyboardTargets(context.evidence!); + const current = observedKeyboardTargets(evidence!); + const specificFrames = [...context.frames.values()].filter( + (frame) => frame.focusIsDocument !== true, + ); + if (!specificFrames.length && !recipients.length && !current.length) return true; + // A single, stable, observed recipient gets its own exact region. Missing + // AX focus, hidden targets and ambiguous frame/shadow focus stay strict. + if ( + specificFrames.length === 1 && + recipients.length === 1 && + current.length === 1 && + recipients[0].backendNodeId === current[0].backendNodeId && + recipients[0].frameId === current[0].frameId + ) + return sameObservationEvidence(context.evidence!, evidence!, recipients[0]); + return sameObservationEvidence(context.evidence!, evidence!); + }; const matches = (keys?: ReadonlySet) => { if (evidenceTargets) { - const [own, ...dependencies] = evidenceTargets; return ( - sameObservationEvidence(context.evidence!, evidence!, own, keys) && - dependencies.every((dependency) => + keyboardRegionMatches() && + (!actionTarget || + sameObservationEvidence(context.evidence!, evidence!, actionTarget, keys)) && + evidenceTargets.every((dependency) => sameObservationEvidence(context.evidence!, evidence!, dependency), ) ); @@ -1113,7 +1186,7 @@ export class CooperationController implements ControlAuthority { ]; const [first] = reported; throw new Error( - (revoked = `Page text changed by itself (${quote(first.before)} → ${quote(first.after)}${reported.length > 1 ? ` and ${reported.length - 1} more` : ""}); synchronize and replan. After that sync, further changes to that text no longer block actions on a ref or selector elsewhere. They still block actions whose target shares a form, list, dialog, panel or other container with it, and actions without a ref or selector. If the decision relies on that text, declare it with --depends-on to keep it exact`), + (revoked = `Page text changed by itself (${quote(first.before)} → ${quote(first.after)}${reported.length > 1 ? ` and ${reported.length - 1} more` : ""}); synchronize and replan. After that sync, further changes to that text no longer block actions on a ref or selector elsewhere. They still block actions whose target shares a form, list, dialog, panel or other container with it, and actions without a ref, selector or explicit dependencies. If the decision relies on that text, declare it with --depends-on to keep it exact`), ); } throw new Error((revoked = "The action's observed content changed; synchronize and replan")); @@ -1138,7 +1211,6 @@ export class CooperationController implements ControlAuthority { ); if (isRpcError(resolved)) return resolved; actionTarget = resolved; - if (params.depends_on?.length) evidenceTargets = [resolved]; } if (params.depends_on !== undefined) { if ( @@ -1152,7 +1224,10 @@ export class CooperationController implements ControlAuthority { code: "invalid_params", message: "depends_on requires at most 16 observed refs/selectors", }; - if (evidenceTargets) + if (params.depends_on.length) { + // Dependencies constrain the decision even when a key has no target. + // Keep them separate from the target so learned text never relaxes them. + evidenceTargets = []; for (const dependency of params.depends_on) { const resolved = await resolveActionTarget( guardedRunner(cdp), @@ -1164,6 +1239,7 @@ export class CooperationController implements ControlAuthority { if (isRpcError(resolved)) return resolved; evidenceTargets.push(resolved); } + } } await verifyEvidence(); state.context = undefined; // Consume before execution, including failed/unknown replies. diff --git a/apps/extension/src/cooperation/media-progress.ts b/apps/extension/src/cooperation/media-progress.ts new file mode 100644 index 00000000..072205e6 --- /dev/null +++ b/apps/extension/src/cooperation/media-progress.ts @@ -0,0 +1,37 @@ +/** Range inputs also represent business data. Only explicitly named timelines + * with an unambiguous media association may tolerate incidental progress. */ +export interface PlaybackProgressControl { + tag: string; + role?: string; + name?: string; + attrs?: Readonly>; + inForm: boolean; +} + +const timelineName = + /^(?:(?:playback|video|audio|media)\s+(?:progress|position|time(?:\s+scrubber)?|timeline)|seek(?:\s+(?:bar|slider))?|time scrubber|(?:播放|视频|音频)(?:进度|时间|位置|时间轴))$/i; + +/** Explicit aria-controls wins; otherwise require exactly one media element. + * Named/submitted controls, form members and unknown labels stay protected. */ +export function playbackProgressMedia( + control: PlaybackProgressControl, + media: readonly T[], +): T | undefined { + const attrs = control.attrs ?? {}; + if (control.inForm || attrs.form !== undefined || attrs.name?.trim()) return; + const nativeRange = control.tag === "input" && attrs.type?.toLowerCase() === "range"; + if ( + !nativeRange && + control.tag !== "progress" && + !["slider", "progressbar"].includes(control.role?.toLowerCase() ?? "") + ) + return; + if (!timelineName.test(control.name?.trim().replace(/\s+/g, " ") ?? "")) return; + const controls = attrs["aria-controls"]?.trim().split(/\s+/).filter(Boolean); + if (controls?.length) { + if (controls.length !== 1) return; + const matches = media.filter((item) => item.id === controls[0]); + return matches.length === 1 ? matches[0] : undefined; + } + return media.length === 1 ? media[0] : undefined; +} diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index 00b496ec..1f91d89a 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -298,3 +298,172 @@ describe("learned text around a target inside
or without a region", () => expect(allowed(flat, 2, 3)).toBe(true); }); }); + +const mediaScene = () => [ + ...scene(), + { ...node(6, 1, "Video", "Preview"), tag: "video", attrs: { id: "preview", src: "clip.webm" } }, + { + ...node(7, 1, "slider", "Playback progress"), + tag: "input", + inForm: false, + value: "1", + attrs: { type: "range" }, + }, +]; + +it("tolerates only the value of an outside media timeline for a declared scope", () => { + const before = mediaScene(); + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(true); + expect(sameObservationEvidence({ nodes: before }, { nodes: after })).toBe(false); + expect( + sameObservationEvidence( + { nodes: before }, + { nodes: after }, + { backendNodeId: 7, frameId: "frame" }, + ), + ).toBe(false); +}); + +it.each([ + "business slider", + "business text", + "named input", + "form member", + "external form", + "missing media", + "ambiguous media", + "different frame", + "wrong relation", +])("protects %s despite video playback", (kind) => { + const before = mediaScene(); + const progress = before[6]; + if (kind === "business slider") progress.name = "Order amount"; + if (kind === "business text") { + progress.role = "textbox"; + progress.attrs = { type: "text" }; + } + if (kind === "named input") progress.attrs!.name = "amount"; + if (kind === "form member") { + progress.inForm = true; + before.push(node(8, 1, "form", "Order")); + progress.domAncestorIds = [8, 1]; + } + if (kind === "external form") progress.attrs!.form = "order"; + if (kind === "missing media") before[5].tag = "div"; + if (kind === "ambiguous media") before.push({ ...node(8, 1, "Video"), tag: "video" }); + if (kind === "different frame") before[5].frameId = "other"; + if (kind === "wrong relation") progress.attrs!["aria-controls"] = "order"; + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it("supports an explicit media relation when more than one player exists", () => { + const before = mediaScene(); + before.push({ ...node(8, 1, "Video"), tag: "video", attrs: { id: "other" } }); + before[6].attrs!["aria-controls"] = "preview"; + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(true); +}); + +it.each([ + "identity", + "label", + "disabled", + "removed", + "media identity", + "media source", + "business value", +])("retains %s evidence while ignoring timeline ticks", (kind) => { + const before = mediaScene(); + before.push({ ...node(8, 1, "textbox", "Order total"), value: "100" }); + const after = structuredClone(before); + after[6].value = "2"; + if (kind === "identity") after[6].backendNodeId = 70; + if (kind === "label") after[6].name = "Order amount"; + if (kind === "disabled") after[6].disabled = true; + if (kind === "removed") after.splice(6, 1); + if (kind === "media identity") after[5].backendNodeId = 60; + if (kind === "media source") after[5].attrs!.src = "other.webm"; + if (kind === "business value") after[7].value = "200"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it.each([ + "min", + "max", + "step", + "aria-valuemin", + "aria-valuemax", +])("protects timeline %s while its value advances", (attribute) => { + const before = mediaScene(); + const after = structuredClone(before); + after[6].value = "2"; + after[6].attrs![attribute] = "10"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it("never learns media values or relaxes an explicitly observed timeline", () => { + const before = { nodes: mediaScene() }; + const after = structuredClone(before); + after.nodes[6].value = "2"; + const learned = new Set(["frame:7"]); + expect(contentChanges(before, after)).toBeUndefined(); + expect(sameObservationEvidence(before, after, undefined, learned)).toBe(false); + expect( + sameObservationEvidence(before, after, { backendNodeId: 7, frameId: "frame" }, learned), + ).toBe(false); + expect(sameObservationEvidence(before, after, target, learned)).toBe(true); +}); + +it("allows incidental progress within the pause button's region, but keeps declared containers exact", () => { + const before = { nodes: mediaScene() }; + before.nodes[1].tag = "section"; + before.nodes[1].role = "region"; + before.nodes[5].parentId = 2; + before.nodes[6].parentId = 2; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, target)).toBe(true); + expect(sameObservationEvidence(before, after, { backendNodeId: 7, frameId: "frame" })).toBe( + false, + ); + expect(sameObservationEvidence(before, after, { backendNodeId: 2, frameId: "frame" })).toBe( + false, + ); + expect(sameObservationEvidence(before, after)).toBe(false); + after.nodes[2].value = "new order"; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); + +it("keeps explicitly declared DOM ancestors exact even when an AX parent differs", () => { + const before = { nodes: mediaScene() }; + before.nodes.push(node(8, 1, "section", "Player")); + before.nodes[6].domAncestorIds = [8, 1]; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, { backendNodeId: 8, frameId: "frame" })).toBe( + false, + ); +}); + +it.each([ + true, + undefined, +])("keeps media-labelled controls exact when native form membership is %s", (inForm) => { + const before = { nodes: mediaScene() }; + before.nodes[6].inForm = inForm; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); + +it("rejects a change in form membership even when the timeline value is unchanged", () => { + const before = { nodes: mediaScene() }; + const after = structuredClone(before); + after.nodes[6].inForm = true; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index 0e3a7a15..3ea00052 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -1,4 +1,5 @@ import type { VomNode, VomScene } from "@browser-skill/vom"; +import { playbackProgressMedia } from "./media-progress"; /** Reuse the observation's semantic nodes. CSS classes and pixel positions are * deliberately absent; the interaction layer rechecks the live hit target. */ @@ -33,6 +34,12 @@ const semanticAttributes = [ "aria-pressed", "aria-disabled", "aria-hidden", + "aria-controls", + "aria-valuemin", + "aria-valuemax", + "min", + "max", + "step", "readonly", "required", ]; @@ -57,6 +64,20 @@ export function observesTarget(evidence: ObservationEvidence, target: EvidenceTa return evidence.nodes.some((node) => isTarget(node, target)); } +/** Specific keyboard recipients visible to the browser's accessibility tree. + * Document/iframe focus is not a reliable substitute for the actual recipient. */ +export function observedKeyboardTargets(evidence: ObservationEvidence): EvidenceTarget[] { + return evidence.nodes + .filter( + (node) => + node.focused && + node.backendNodeId !== undefined && + !["body", "html", "iframe", "frame"].includes(node.tag) && + node.role !== "RootWebArea", + ) + .map((node) => ({ backendNodeId: node.backendNodeId!, frameId: node.frameId })); +} + const positioned = (node: VomNode) => ["absolute", "fixed"].includes(node.position); // Form state, value-bearing widgets and modal layers are never page activity. const protectedState = (node: VomNode) => @@ -232,6 +253,30 @@ function signature( learnedText.set(holder, set); } } + // A semantic region may include both Pause and the timeline. Only a direct + // dependency on the timeline, its player, or a containing element makes its + // value exact; sharing the inferred region with a button does not. + // Keep the original nodes intact: media controls cannot become learned text. + const media = nodes.filter((node) => node.tag === "video" || node.tag === "audio"); + const ignoredProgress = new Map(); + const explicitlyObserved = (node: VomNode) => + !!anchor && (tree.within(node, anchor) || (node.domAncestorIds ?? []).includes(anchor.id)); + if (target) + for (const node of nodes) { + if (explicitlyObserved(node) || node.value === undefined) continue; + const owner = playbackProgressMedia( + { + ...node, + // Only native ownership or complete raw DOM evidence can establish + // that a control is form-free. Pruned ancestors are not evidence. + inForm: node.inForm !== false, + }, + media + .filter((item) => item.frameId === node.frameId) + .map((item) => ({ id: item.attrs?.id, node: item })), + ); + if (owner && !explicitlyObserved(owner.node)) ignoredProgress.set(node, owner.node); + } // Identity, structure, state and visibility of learned nodes remain checked. const facts = (node: VomNode, relax: boolean) => { const tolerate = relax && tolerated(node); @@ -246,7 +291,8 @@ function signature( ? withoutLearned(node.name, learnedText.get(node)) : node.name, tolerate ? undefined : node.text, - node.value, + ignoredProgress.has(node) ? undefined : node.value, + node.inForm, node.checked, node.disabled, node.inert, @@ -266,6 +312,8 @@ function signature( .filter((node) => node.value !== undefined || node.checked !== undefined) .map((node) => facts(node, false)), nodes.filter((node) => node.modal).map((node) => facts(node, false)), + // Replacing the player or its source still invalidates the observation. + [...new Set(ignoredProgress.values())].map((node) => [...facts(node, false), node.attrs?.src]), ]); } @@ -276,7 +324,8 @@ function signature( * wherever it appears as whole words. Without a target this applies to the * whole page. With a target it never applies inside the target's own region, * as described in signature(). Form values, checked states and modal layers - * stay exact everywhere. + * stay exact everywhere except positively identified, non-submitted media + * progress values outside the explicitly observed target subtree. */ export function sameObservationEvidence( before: ObservationEvidence, diff --git a/apps/extension/src/cooperation/types.ts b/apps/extension/src/cooperation/types.ts index 22416754..9416a75c 100644 --- a/apps/extension/src/cooperation/types.ts +++ b/apps/extension/src/cooperation/types.ts @@ -12,6 +12,8 @@ export interface FieldChange { id?: string; label: string; value: string; + /** A positively identified, non-submitted media timeline. */ + kind?: "media_progress"; } export interface HumanActivity { @@ -30,8 +32,23 @@ export interface HumanActivity { context?: string; } +/** Media identity and source selection, excluding ordinary playback progress. */ +export interface MediaState { + id: string; + src: string | null; + currentSrc: string; + srcObject: string | null; + tracks?: string[]; + sources: { id: string; src: string; type: string; media: string }[]; +} + export interface FrameState { document: string; + /** Collector-local identity of the deep active element in this document. */ + focus?: string; + /** True only for the ordinary document focus, without a specific recipient. */ + focusIsDocument?: boolean; + media?: MediaState[]; revision: number; structure: number; human: number; diff --git a/apps/extension/src/tools/vom/__tests__/form-state.test.ts b/apps/extension/src/tools/vom/__tests__/form-state.test.ts index febf141b..fdf45dc9 100644 --- a/apps/extension/src/tools/vom/__tests__/form-state.test.ts +++ b/apps/extension/src/tools/vom/__tests__/form-state.test.ts @@ -329,3 +329,127 @@ describe("form state identity", () => { expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); }); }); + +it("captures actual native form membership independently of the form's accessibility role", async () => { + const doc = document.implementation.createHTMLDocument("ownership"); + doc.body.innerHTML = + '
'; + const { cdp, nodes } = await fakeCdp( + [[11, 12, 13].map((id) => ({ id, type: "range", value: "0" }))], + (params) => { + const entries = new Function("document", `return ${params.expression}`)(doc) as [ + HTMLInputElement, + string, + ][]; + return reply(entries.map(([element, json]) => [Number(element.id), JSON.parse(json)])); + }, + ); + await enrichFormControlStates(cdp, 7, nodes); + expect(nodes[0].filter((n) => n.tag === "input").map((n) => n.inForm)).toEqual([ + true, + false, + true, + ]); +}); + +it.each([ + true, + false, + undefined, +])("reads missing range ownership by backend identity, preserving unknown results (%s)", async (inForm) => { + const nodes = await decodeControls(snapshot([[{ id: 11, type: "range", value: "0" }]])); + const send = vi.fn(async (_tabId, method) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: "closed-shadow-range" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: inForm } }; + return {}; + }); + await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + expect(send).toHaveBeenCalledWith( + 7, + "DOM.resolveNode", + expect.objectContaining({ backendNodeId: 11 }), + ); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(inForm); + expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); +}); + +it("bounds missing range-owner reads and keeps the uninspected remainder unknown", async () => { + const nodes = await decodeControls( + snapshot([Array.from({ length: 65 }, (_, i) => ({ id: i + 1, type: "range", value: "0" }))]), + ); + const send = vi.fn(async (_tabId, method, params) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: String(params.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") return { result: { value: false } }; + return {}; + }); + await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + expect(send.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(64); + expect(nodes[0].find((n) => n.backendNodeId === 64)?.inForm).toBe(false); + expect(nodes[0].find((n) => n.backendNodeId === 65)?.inForm).toBeUndefined(); +}); + +it("drains in-flight range reads before releasing objects after a terminal error", async () => { + const nodes = await decodeControls( + snapshot([ + [ + { id: 11, type: "range", value: "0" }, + { id: 12, type: "range", value: "0" }, + ], + ]), + ); + let rejectFirst!: (reason: Error) => void; + let resolveSecond!: (value: unknown) => void; + const send = vi.fn(async (_tabId, method, params) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") + return new Promise((resolve, reject) => { + if (params.backendNodeId === 11) rejectFirst = reject; + else resolveSecond = resolve; + }); + return {}; + }); + let settled = false; + const result = enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes).catch( + (error) => { + settled = true; + return error; + }, + ); + await vi.waitFor(() => expect(resolveSecond).toBeTypeOf("function")); + const error = Object.assign(new Error("renderer timed out"), { name: "CdpReadTimeoutError" }); + rejectFirst(error); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(settled).toBe(false); + expect(send).not.toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); + resolveSecond({ object: { objectId: "second" } }); + expect(await result).toBe(error); + expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); + expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); +}); + +it("keeps fallback owner reads with identical backend ids scoped to their OOPIF target", async () => { + const sendToTarget = vi.fn(async (target, method) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: "range" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: target.sessionId === "owned-frame" } }; + return {}; + }); + const send = vi.fn(async () => { + throw new Error("unexpected root target"); + }); + const cdp = { send, sendToTarget } as unknown as CdpRunner; + for (const sessionId of ["owned-frame", "free-frame"]) { + const nodes = await decodeControls(snapshot([[{ id: 11, type: "range", value: "0" }]])); + await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(sessionId === "owned-frame"); + expect(sendToTarget).toHaveBeenCalledWith( + { tabId: 7, sessionId }, + "Runtime.callFunctionOn", + expect.objectContaining({ objectId: "range" }), + ); + } + expect(send).not.toHaveBeenCalled(); +}); diff --git a/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts new file mode 100644 index 00000000..77a08d49 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { normalizeDocument } from "../normalize"; +import { buildSemanticVomScene } from "../semantic-graph"; +import { BASIC_SNAPSHOT, type SnapshotProfile, VISUAL_SNAPSHOT } from "../snapshot"; + +interface Spec { + id: number; + parent?: number; + tag: string; + attrs?: Record; +} + +async function normalize(input: Spec[], profile: SnapshotProfile = BASIC_SNAPSHOT) { + const strings: string[] = []; + const str = (value: string) => strings.push(value) - 1; + return normalizeDocument( + { + nodes: { + backendNodeId: input.map((node) => node.id), + nodeType: input.map((node) => + node.tag === "#document" ? 9 : node.tag === "#document-fragment" ? 11 : 1, + ), + nodeName: input.map((node) => str(node.tag)), + parentIndex: input.map((node) => { + if (node.parent === undefined) return -1; + const index = input.findIndex((parent) => parent.id === node.parent); + return index >= 0 ? index : input.length + 99; + }), + attributes: input.map((node) => + Object.entries(node.attrs ?? {}).flatMap(([key, value]) => [str(key), str(value)]), + ), + }, + layout: { + nodeIndex: input.map((_, i) => i), + bounds: input.map(() => [0, 0, 100, 30]), + }, + }, + strings, + { + target: { tabId: 1 }, + frameId: "main", + ownerFrameBackendNodeId: null, + projection: null, + coordinates: null, + }, + undefined, + profile, + ); +} + +const roots: Spec[] = [ + { id: 1, tag: "#document" }, + { id: 2, parent: 1, tag: "html" }, +]; +const slider = { role: "slider", "aria-label": "Playback progress", "aria-valuenow": "0" }; + +describe("custom progress form membership", () => { + it.each([ + BASIC_SNAPSHOT, + VISUAL_SNAPSHOT, + ])("preserves form containment before document and semantic pruning: %j", async (profile) => { + const normalized = await normalize( + [ + ...roots, + { id: 3, parent: 2, tag: "form", attrs: { role: "none" } }, + { id: 4, parent: 3, tag: "div", attrs: slider }, + { id: 5, parent: 2, tag: "div", attrs: slider }, + { id: 6, parent: 99, tag: "div", attrs: slider }, + ], + profile, + ); + expect(normalized.nodes.some((node) => node.tag === "#document")).toBe(false); + expect(normalized.index.nodes.get(4)?.inForm).toBe(true); + expect(normalized.index.nodes.get(5)?.inForm).toBe(false); + expect(normalized.index.nodes.get(6)?.inForm).toBeUndefined(); + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + { + frameId: "main", + contextScopeId: "main", + target: { tabId: 1 }, + axNodes: [], + domNodes: normalized.nodes, + }, + ], + }); + expect(scene.nodes.some((node) => node.tag === "form")).toBe(false); + for (const [id, membership] of [ + [4, true], + [5, false], + [6, undefined], + ] as const) + expect(scene.nodes.find((node) => node.backendNodeId === id)?.inForm).toBe(membership); + }); + + it.each(["slider", "progressbar"])("retains raw shadow ancestry for %s widgets", async (role) => { + const normalized = await normalize([ + ...roots, + { id: 3, parent: 2, tag: "form", attrs: { role: "presentation" } }, + { id: 4, parent: 3, tag: "#document-fragment" }, + { id: 5, parent: 4, tag: "div", attrs: { ...slider, role } }, + { id: 6, parent: 2, tag: "#document-fragment" }, + { id: 7, parent: 6, tag: "div", attrs: { ...slider, role } }, + { id: 8, parent: 3, tag: "progress" }, + { id: 9, parent: 2, tag: "progress" }, + ]); + expect([5, 7, 8, 9].map((id) => normalized.index.nodes.get(id)?.inForm)).toEqual([ + true, + false, + true, + false, + ]); + }); + + it("does not infer native ownership from physical containment", async () => { + const normalized = await normalize([ + ...roots, + { id: 3, parent: 2, tag: "form" }, + { id: 4, parent: 3, tag: "input", attrs: { type: "range" } }, + { id: 5, parent: 2, tag: "input", attrs: { type: "range", form: "external" } }, + ]); + for (const id of [4, 5]) expect(normalized.index.nodes.get(id)?.inForm).toBeUndefined(); + }); + + it("keeps disconnected roots and cyclic ancestry unknown", async () => { + const normalized = await normalize([ + { id: 1, tag: "html" }, + { id: 2, parent: 1, tag: "div", attrs: slider }, + { id: 3, parent: 4, tag: "div", attrs: slider }, + { id: 4, parent: 3, tag: "div", attrs: slider }, + { id: 5, parent: 3, tag: "div", attrs: slider }, + { id: 6, parent: 99, tag: "#document" }, + { id: 7, parent: 6, tag: "div", attrs: slider }, + ]); + for (const id of [2, 3, 4, 5, 7]) + expect(normalized.index.nodes.get(id)?.inForm).toBeUndefined(); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index 85452385..d1b76af9 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -954,3 +954,61 @@ describe("semantic VOM graph", () => { expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id); }); }); + +it("retains browser focus on semantic nodes without trusting a DOM focused attribute", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "buy", + backendDOMNodeId: 2, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Buy" }, + properties: [{ name: "focused", value: { value: true } }], + }, + { + nodeId: "pause", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Pause" }, + properties: [{ name: "focused", value: { value: false } }], + }, + ], + [dom(2, null, "button"), dom(3, null, "button", { focused: "true" })], + ), + ], + }); + expect(scene.nodes.find((node) => node.backendNodeId === 2)?.focused).toBe(true); + expect(scene.nodes.find((node) => node.backendNodeId === 3)?.focused).toBeUndefined(); +}); + +it.each([ + "none", + "presentation", +])("retains native form membership when the %s form is omitted", (role) => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [], + [ + dom(1, null, "main"), + dom(2, 1, "form", { role }), + { + ...dom(3, 2, "input", { type: "range", "aria-label": "Playback progress" }), + formValue: "0", + inForm: true, + }, + ], + ), + ], + }); + expect(scene.nodes.some((n) => n.tag === "form")).toBe(false); + expect(scene.nodes.find((n) => n.backendNodeId === 3)?.inForm).toBe(true); +}); diff --git a/apps/extension/src/tools/vom/capture-types.ts b/apps/extension/src/tools/vom/capture-types.ts index 74910bf0..bfd7abea 100644 --- a/apps/extension/src/tools/vom/capture-types.ts +++ b/apps/extension/src/tools/vom/capture-types.ts @@ -32,6 +32,9 @@ export interface CapturedNode { */ rendered?: boolean; textContent?: string; + /** Native form ownership, or raw containment for custom progress widgets; + * absent when unavailable. Never infer this from the projected semantic tree. */ + inForm?: boolean; formState?: "empty" | "filled" | "default"; formValue?: string; /** Live native checked state, independent of the submitted form value. */ diff --git a/apps/extension/src/tools/vom/form-capture.ts b/apps/extension/src/tools/vom/form-capture.ts index ff1735fc..36e51099 100644 --- a/apps/extension/src/tools/vom/form-capture.ts +++ b/apps/extension/src/tools/vom/form-capture.ts @@ -6,8 +6,10 @@ interface RuntimeEvaluateReply { result?: { deepSerializedValue?: DeepSerializedValue }; } const MAX_FORM_ENRICH_CONTROLS = 250; +const MAX_RANGE_FORM_READS = 64; interface CapturedFormState { + inForm?: boolean; value?: string; defaultValue?: string; placeholder?: string; @@ -37,6 +39,7 @@ function formStateBatchExpression(maxControls: number): string { state, sensitive, placeholder, + inForm: el.form !== null, ...(sensitive ? {} : { value: rawValue, defaultValue }), }; }; @@ -80,6 +83,7 @@ function formStatesByBackendId(result: RuntimeEvaluateReply): Map { + const pending = groups + .flat() + .filter( + (node) => + node.tag === "input" && + node.attrs.type?.toLowerCase() === "range" && + node.inForm === undefined, + ) + .slice(0, MAX_RANGE_FORM_READS); + let cursor = 0; + let failed = false; + let failure: unknown; + await Promise.all( + Array.from({ length: Math.min(4, pending.length) }, async () => { + while (cursor < pending.length && !failed && !signal?.aborted) { + const node = pending[cursor++]; + try { + throwIfAborted(signal); + const resolved = await cdp.send<{ object?: { objectId?: string } }>( + tabId, + "DOM.resolveNode", + { + backendNodeId: node.backendNodeId, + objectGroup, + }, + ); + throwIfAborted(signal); + if (failed) return; + if (!resolved.object?.objectId) continue; + const result = await cdp.send<{ + result?: { value?: unknown }; + exceptionDetails?: unknown; + }>(tabId, "Runtime.callFunctionOn", { + objectId: resolved.object.objectId, + functionDeclaration: "function() { return this.form !== null; }", + returnByValue: true, + throwOnSideEffect: true, + }); + throwIfAborted(signal); + if (!result.exceptionDetails && typeof result.result?.value === "boolean") + node.inForm = result.result.value; + } catch (error) { + if (isCaptureTerminalError(error)) { + if (!failed) failure = error; + failed = true; + } + // Unknown ownership stays protected; never guess from missing ancestors. + } + } + }), + ); + throwIfAborted(signal); + if (failed) throw failure; +} + export async function enrichFormControlStates( cdp: CdpRunner, tabId: number, @@ -143,6 +213,7 @@ export async function enrichFormControlStates( for (const nodes of frameNodeGroups) { applyFormStates(nodes, states); } + await enrichRangeFormMembership(cdp, tabId, frameNodeGroups, objectGroup, signal); return true; } catch (error) { if (isCaptureTerminalError(error)) throw error; diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 3bc5a775..0e331e79 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -51,6 +51,68 @@ export interface NormalizedDocument { documentElementBackendNodeId?: number; } +/** Custom progress widgets have no native .form owner. Preserve their raw + * containment before document/shadow roots and transparent forms are omitted. + * Only a complete path to a document can establish that a widget is form-free. */ +async function captureCustomProgressFormMembership( + index: DocumentIndex, + signal?: AbortSignal, +): Promise { + const checkpoint = createCaptureCheckpoint(signal); + const membership = new Map(); + let work = 0; + for (const node of index.nodes.values()) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (["input", "textarea", "select"].includes(node.tag)) continue; + if ( + node.tag !== "progress" && + !node.attrs.role + ?.toLowerCase() + .split(/\s+/) + .some((role) => ["slider", "progressbar"].includes(role)) + ) + continue; + const path: number[] = []; + const seen = new Set(); + let current: NodeFacts | undefined = node; + let inForm: boolean | undefined; + while (current) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (membership.has(current.backendNodeId)) { + inForm = membership.get(current.backendNodeId); + break; + } + if (seen.has(current.backendNodeId)) break; + seen.add(current.backendNodeId); + path.push(current.backendNodeId); + if (current.tag === "form") { + inForm = true; + break; + } + if (current.parentMissing) break; + if (current.parentBackendNodeId === null) { + if (current.nodeType === 9) inForm = false; + break; + } + current = index.nodes.get(current.parentBackendNodeId); + } + for (const id of path) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + membership.set(id, inForm); + } + node.inForm = inForm; + } +} + /** Interpret one document using a supplied projection. No live reads or frame scheduling. */ export async function normalizeDocument( doc: SnapshotDocument, @@ -88,6 +150,7 @@ export async function normalizeDocument( }); } const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts); + await captureCustomProgressFormMembership(index, signal); const document: NormalizedDocument = { nodes: nodes.filter( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 4de56b8a..81e7a7dd 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -486,6 +486,7 @@ export function resolveSemanticGraph( if (isSensitive) delete attrs.value; const frame = graph.frames.get(node.frameId); const tag = node.dom?.tag.toLowerCase() ?? ""; + const inForm = node.dom?.inForm; const vom: ResolvedSemanticNode["vom"] = { tag, rect: node.dom?.rect ?? null, @@ -497,7 +498,11 @@ export function resolveSemanticGraph( ...(role ? { role } : {}), ...(name ? { name } : {}), ...(value !== undefined ? { value } : {}), + ...(inForm !== undefined ? { inForm } : {}), ...(checked !== undefined ? { checked } : {}), + ...(node.ax?.ignored !== true && axProperty(node.ax, "focused") === "true" + ? { focused: true } + : {}), ...(clean(node.dom?.formPlaceholder ?? attrs.placeholder) ? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) } : {}), diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 79172692..c23b2357 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -231,8 +231,7 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, - ...(profile.includeVisualFacts && - (dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null)) + ...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null) ? { parentMissing: true } : {}), diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index bf01819f..1a11b6c1 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -40,7 +40,8 @@ set is uncertain, omit it and retain whole-page checking. Text that changes by i sync and action (a clock, a counter) is reported once with the before/after text. After you sync and replan, further changes to that text no longer block actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or -selector, such as a bare key press, always check the whole page exactly. Added, removed or +selector never learn incidental text; a bare key press checks the whole page exactly unless +explicit dependencies narrow its decision regions. Added, removed or re-rendered elements and text without its own accessible node (such as `aria-hidden` content) keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an @@ -52,6 +53,19 @@ A regular coordination result is not a failed browser attempt. Use wait then syn the next action from the actual result. Use request-help only for an actual human requirement such as login or a captcha. Explicit user stop always ends the task. +On continuously updating pages, declare the observed decision regions instead of repeatedly +retrying a whole-page action. `press` honors dependencies even without a focus target; specify +a target when the key must be delivered there. Untargeted keys reject a changed focus and +protect the actual focused region even if the declared dependencies are elsewhere. Unknown +or ambiguous focus keeps whole-page checking; untargeted scripts also keep that strict check. +A clearly labelled media timeline with an unambiguous video/audio association may advance +beside a pause button, including in the same player region. Directly observing the timeline, +its media element or a containing element keeps its value exact. Submitted inputs, form members, +ordinary sliders and other business values remain protected. Media identity, child `` +declarations and the selected source stay exact. Form ownership remains protected even when +a form is omitted from the observation. Unknown media controls or unavailable form-ownership +evidence retain ordinary checks. + ## Scripts and file transfers Scripts, upload/download, active hover observation, full-page screenshots, device emulation diff --git a/evals/browser/cases/regression/cooperation-video/README.md b/evals/browser/cases/regression/cooperation-video/README.md new file mode 100644 index 00000000..37dd0907 --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/README.md @@ -0,0 +1,56 @@ +# Continuous video progress and cooperative actions + +This regression runs the actual CLI, daemon, extension and an isolated Chromium profile. +The fixture loops a bundled synthetic WebM in a real video element. No remote media, live +recording, or existing browser profile is required. A separate CDP connection checks decoded +frames, received key events and the final paused state independently of the CLI response. + +Build and run from the repository root, using a cooperative CLI build: + +```sh +BSK_DAEMON_WS_URL=ws://127.0.0.1:52931 pnpm --filter @browser-skill/extension build +BSK_COOP_CLI=/path/to/bsk \ +BSK_COOP_CHROME=/path/to/chromium \ +node evals/browser/cases/regression/cooperation-video/run.mjs +``` + +`BSK_COOP_PORT` must match the built extension's endpoint and must not be the normal daemon +port 52800. `BSK_COOP_EXTENSION` overrides the built extension directory. Chrome for Testing +uses `Extensions.loadUnpacked`; use `BSK_COOP_LOAD_CDP=0` with builds that support the load +extension command-line flags instead. `BSK_COOP_ARTIFACTS` selects the result directory. + +The 53 cases cover continuously advancing progress during sync (including incremental sync), +dependencies without a focus target, ordinary business sliders, submitted timelines, +form-owned timelines, simultaneous video and order changes, and a programmatic focus change. +They also combine PR #497's learned-text behavior with the media fix: a learned price becomes +exact again when it belongs to the current target or an explicit dependency, bare keys remain +strict, a refused context cannot be replayed, and an explicitly observed timeline stays exact. +Review regressions cover price changes around the actual keyboard focus despite unrelated +dependencies, child ``/effective source changes, and a pause button beside its labelled +timeline in one player region. Controls declare the timeline or whole player, focus the timeline, +or change a business field in the same player. Focus changes before a delta sync must refresh +semantic evidence, and a document becoming an explicit keyboard recipient revokes the key. +Closed shadow-root players also allow progress but reject a source change with or without +a simultaneous timeline update. Form-ownership cases keep the timeline's accessible name +while removing the form from the semantic tree (`role=none`, `role=presentation`, and +`display:contents`). They cover native form owners, external `form` associations, open and +closed shadow roots, custom progress widgets, and formless positive controls. The oracle +asserts the actual form owner and accessible name before changing the value, so losing a +label cannot accidentally make a negative case pass. +Positive cases require one received Space event +and a paused video. Negative cases require no key event, no simulated purchase, no pause call and +`effect_state: none`. Every created session, daemon and temporary browser profile is cleaned up. + +A dependency must still cover the action's actual decision. A key without a target +or dependencies keeps the exact whole-page check. Targeted actions retain PR #497's bounded +text learning after a refusal and a fresh sync. Implicit keyboard recipients have their own +protected region; uncertain focus falls back to the whole page. Only positively identified, +non-submitted media progress may change incidentally; explicitly declaring the timeline, +its media element or a containing element protects that value. Ordinary fields/sliders and +unknown media controls keep the conservative behavior. + +The two-second VP8 test clip can be regenerated with FFmpeg: + +```sh +ffmpeg -f lavfi -i testsrc2=size=160x90:rate=24 -t 2 -c:v libvpx -b:v 120k -an preview.webm +``` diff --git a/evals/browser/cases/regression/cooperation-video/fixture.html b/evals/browser/cases/regression/cooperation-video/fixture.html new file mode 100644 index 00000000..69a41eeb --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/fixture.html @@ -0,0 +1,157 @@ + + +BrowserSkill looping video regression + +
+

Looping preview

+

Local regression fixture. Space or the Pause button pauses the video.

+ +
+ +
+ +
+ diff --git a/evals/browser/cases/regression/cooperation-video/preview.webm b/evals/browser/cases/regression/cooperation-video/preview.webm new file mode 100644 index 00000000..02f550dc Binary files /dev/null and b/evals/browser/cases/regression/cooperation-video/preview.webm differ diff --git a/evals/browser/cases/regression/cooperation-video/run.mjs b/evals/browser/cases/regression/cooperation-video/run.mjs new file mode 100644 index 00000000..ac8b8b9c --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/run.mjs @@ -0,0 +1,873 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { withChrome } from "../snapshot-coordinates/chrome.mjs"; + +const directory = dirname(fileURLToPath(import.meta.url)); +const cli = resolve(process.env.BSK_COOP_CLI ?? "target/debug/bsk"); +const executable = process.env.BSK_COOP_CHROME; +assert(executable, "Set BSK_COOP_CHROME to an isolated Chromium executable"); +const port = Number(process.env.BSK_COOP_PORT ?? 52931); +assert(Number.isInteger(port) && port > 0 && port !== 52800); +const extensionPath = resolve(process.env.BSK_COOP_EXTENSION ?? "apps/extension/dist/chrome-mv3"); +assert( + (await readFile(join(extensionPath, "background.js"), "utf8")).includes(`ws://127.0.0.1:${port}`), + "Build the test extension with BSK_DAEMON_WS_URL matching BSK_COOP_PORT", +); +const artifacts = resolve( + process.env.BSK_COOP_ARTIFACTS ?? "evals/browser/results/cooperation-video", +); +await mkdir(artifacts, { recursive: true }); +const runtime = await mkdtemp(join(tmpdir(), "bsk-video-live-")); +const env = { ...process.env, BSK_HOME: runtime, BSK_AUTO_START: "0", BSK_AUTO_UPDATE: "off" }; +const exec = promisify(execFile); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const commands = []; +const results = []; +let daemon; +let daemonOutput = ""; +let browserVersion; +const bsk = async (args) => { + const start = performance.now(); + let output, stderr, exitCode; + try { + const result = await exec(cli, [...args, "--json"], { + env, + timeout: 35000, + maxBuffer: 2 ** 22, + }); + output = result.stdout; + stderr = result.stderr; + exitCode = 0; + } catch (error) { + output = error.stdout || ""; + stderr = error.stderr || error.message; + exitCode = error.code; + } + let result; + try { + result = JSON.parse(output); + } catch { + result = { stdout: output, stderr }; + } + const entry = { + args, + exitCode, + durationMs: Math.round(performance.now() - start), + result, + stderr, + }; + commands.push(entry); + await appendFile(join(artifacts, "commands.jsonl"), JSON.stringify(entry) + "\n"); + return entry; +}; +const ok = async (args) => { + const reply = await bsk(args); + assert.equal(reply.exitCode, 0, JSON.stringify(reply)); + return reply.result; +}; +const poll = async (read, label, timeout = 15000) => { + const until = performance.now() + timeout; + while (performance.now() < until) { + const value = await read(); + if (value) return value; + await delay(100); + } + throw new Error("Timed out: " + label); +}; +const html = await readFile(join(directory, "fixture.html")); +const clip = await readFile(join(directory, "preview.webm")); +const server = createServer((req, res) => { + if (req.url.startsWith("/preview.webm")) { + res.setHeader("Content-Type", "video/webm"); + res.setHeader("Content-Length", clip.length); + res.end(clip); + return; + } + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(html); +}); +const cases = [ + { name: "video_pixels_only", variant: "pixels", target: false, dependencies: false, allow: true }, + { + name: "timer_text_untargeted_space", + variant: "text", + target: false, + dependencies: false, + allow: false, + }, + { + name: "timer_text_targeted_scoped_space", + variant: "text", + target: true, + dependencies: true, + allow: true, + }, + { + name: "timer_text_dependencies_without_target", + variant: "text", + target: false, + dependencies: true, + allow: true, + }, + { + name: "progress_range_targeted_scoped_space", + variant: "range", + target: true, + dependencies: true, + allow: true, + }, + { + name: "cosmetic_class_untargeted_space", + variant: "cosmetic", + target: false, + dependencies: false, + allow: true, + }, + { + name: "continuous_progress_sync_and_space", + variant: "range", + target: true, + dependencies: true, + allow: true, + continuous: true, + }, + { + name: "progress_dependencies_without_target", + variant: "range", + target: false, + dependencies: true, + allow: true, + }, + { + name: "business_range_still_blocked", + variant: "business-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "submitted_timeline_still_blocked", + variant: "submitted-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "form_timeline_still_blocked", + variant: "form-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "progress_and_business_field_still_blocked", + variant: "combined", + target: true, + dependencies: true, + allow: false, + }, + { + name: "programmatic_focus_still_blocked", + variant: "focus", + target: false, + dependencies: true, + allow: false, + focus: true, + }, + { + name: "business_field_targeted_scoped_space", + variant: "field", + target: true, + dependencies: true, + allow: false, + }, + { + name: "learned_price_current_target_blocked", + variant: "learned-price", + learn: true, + target: true, + targetSelector: "#buy", + allow: false, + }, + { + name: "learned_price_declared_dependency_blocked", + variant: "learned-price", + learn: true, + target: true, + dependencies: ["#buy"], + allow: false, + }, + { + name: "learned_price_bare_key_blocked", + variant: "learned-price", + learn: true, + target: false, + allow: false, + }, + { + name: "learned_price_untargeted_dependency_blocked", + variant: "learned-price", + learn: true, + target: false, + dependencies: ["#buy"], + allow: false, + }, + { + name: "refused_context_requires_sync", + variant: "learned-price", + learn: true, + retryRefused: true, + target: true, + allow: true, + }, + { + name: "explicit_timeline_dependency_still_blocked", + variant: "range", + target: true, + dependencies: ["#progress"], + allow: false, + }, + { + name: "focused_buy_price_change_unrelated_dependency", + variant: "focused-price", + key: "Enter", + target: false, + dependencies: ["#pause"], + allow: false, + manual: true, + }, + { + name: "focused_buy_price_change_no_dependency", + variant: "focused-price", + key: "Enter", + target: false, + dependencies: false, + allow: false, + manual: true, + }, + { + name: "focused_buy_price_change_explicit_target", + variant: "focused-price", + key: "Enter", + target: true, + targetSelector: "#buy", + dependencies: ["#pause"], + allow: false, + manual: true, + }, + { + name: "child_source_changes_with_progress", + variant: "source-range", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "child_source_changes_without_progress", + variant: "source-only", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "video_src_changes_with_progress", + variant: "src-range", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "progress_and_pause_in_same_player_region", + variant: "same-region", + target: true, + dependencies: ["#pause"], + allow: true, + }, + { + name: "same_player_business_field_still_blocked", + variant: "same-region-field", + target: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "same_player_declared_timeline_still_blocked", + variant: "same-region", + target: true, + dependencies: ["#progress"], + allow: false, + }, + { + name: "same_player_declared_container_still_blocked", + variant: "same-region", + target: true, + dependencies: ["#player"], + allow: false, + }, + { + name: "focused_pause_in_same_player_allows_progress", + variant: "same-region", + target: false, + focusBefore: "#pause", + dependencies: ["#pause"], + allow: true, + }, + { + name: "focused_timeline_still_exact", + variant: "same-region", + target: false, + focusBefore: "#progress", + dependencies: ["#pause"], + allow: false, + }, + { + name: "focused_pause_after_delta_sync_allows_progress", + variant: "same-region", + target: false, + dependencies: ["#pause"], + allow: true, + focusAfterFirstSync: "#pause", + }, + { + name: "closed_shadow_media_progress_is_incidental", + variant: "range", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: true, + manual: true, + }, + { + name: "closed_shadow_child_source_changes_with_progress", + variant: "source-range", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "closed_shadow_child_source_changes_without_progress", + variant: "source-only", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "body_becomes_specific_recipient_is_blocked", + variant: "body-recipient", + target: false, + dependencies: ["#pause"], + allow: false, + focus: true, + }, + { + name: "form_role_none_progress_still_blocked", + formRole: "none", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_presentation_progress_still_blocked", + formRole: "presentation", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_display_contents_progress_still_blocked", + formRole: "none", + formContents: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "external_form_owner_progress_still_blocked", + formRole: "external", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_none_named_progress_still_blocked", + formRole: "none", + named: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_none_explicit_timeline_still_blocked", + formRole: "none", + dependencies: ["#progress"], + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + allow: false, + }, + { + name: "form_role_none_whole_page_still_blocked", + formRole: "none", + dependencies: false, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + allow: false, + }, + { + name: "form_role_none_business_label_still_blocked", + formRole: "none", + businessLabel: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "non_form_role_none_progress_allowed", + formRole: "none", + nonForm: true, + allow: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "open_shadow_form_progress_still_blocked", + formRole: "none", + controlShadow: "open", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "closed_shadow_form_progress_still_blocked", + formRole: "none", + controlShadow: "closed", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "closed_shadow_continuous_progress_allowed", + controlShadow: "closed", + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "open_shadow_continuous_progress_allowed", + controlShadow: "open", + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "shadow_host_form_does_not_own_timeline", + controlShadow: "closed", + shadowHostForm: true, + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "custom_progress_in_pruned_form_still_blocked", + formRole: "none", + customProgress: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "custom_non_form_progress_allowed", + customProgress: true, + allow: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + }, +]; +try { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const base = `http://127.0.0.1:${server.address().port}`; + daemon = spawn(cli, ["daemon", "start", "--foreground", "--port", String(port)], { + env, + stdio: ["ignore", "pipe", "pipe"], + }); + daemon.stdout.on("data", (chunk) => { + daemonOutput += chunk; + }); + daemon.stderr.on("data", (chunk) => { + daemonOutput += chunk; + }); + await poll(async () => (await bsk(["status"])).exitCode === 0, "isolated daemon"); + await withChrome( + { + executable, + extensionPath, + extensionViaCdp: process.env.BSK_COOP_LOAD_CDP !== "0", + deviceScale: 1, + zoom: 1, + headless: true, + softwareRendering: true, + startupTimeout: 30000, + extraArgs: ["--autoplay-policy=no-user-gesture-required"], + }, + async (send) => { + browserVersion = await send("Browser.getVersion"); + const browsers = await poll(async () => { + const reply = await ok(["browsers"]); + return reply.length && reply; + }, "test extension connection"); + assert.equal(browsers.length, 1, "Only the isolated test browser should use this daemon"); + for (const scenario of cases) { + console.log("CASE " + scenario.name); + const { session_id: session } = await ok([ + "session", + "start", + "--browser", + browsers[0].instance_id, + ]); + const scope = ["--session", session]; + let oracle; + try { + const query = new URLSearchParams({ + variant: scenario.variant, + case: scenario.name, + interval: String(scenario.continuous ? 10 : 80), + shadow: scenario.closedShadow ? "closed" : "none", + }); + for (const key of [ + "formRole", + "formContents", + "named", + "businessLabel", + "nonForm", + "controlShadow", + "customProgress", + "shadowHostForm", + ]) + if (scenario[key] !== undefined) query.set(key, String(scenario[key])); + const url = `${base}/?${query}`; + await ok(["navigate", url, ...scope]); + const target = (await send("Target.getTargets")).targetInfos.find( + (target) => target.url === url, + ); + assert(target, "Test tab must exist"); + ({ sessionId: oracle } = await send("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + })); + await send("Runtime.enable", {}, oracle); + await send("Page.bringToFront", {}, oracle); + const read = async (expression) => { + const reply = await send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + oracle, + ); + assert(!reply.exceptionDetails, JSON.stringify(reply.exceptionDetails)); + return reply.result.value; + }; + let playback; + try { + await poll(async () => { + playback = await read( + "({ready:window.fixtureReady,error:window.fixtureError,report:window.report?.()})", + ); + assert(!playback.error, playback.error); + return playback.ready && playback.report.decodedFrames >= 3; + }, "actual video playback"); + } catch (error) { + await writeFile( + join(artifacts, scenario.name + ".playback-error.json"), + JSON.stringify(playback, null, 2), + ); + throw error; + } + if (scenario.focusBefore) + await read( + `document.querySelector(${JSON.stringify(scenario.focusBefore)}).focus(); true`, + ); + let learning; + if (scenario.learn) { + const initial = await ok(["cooperate", "sync", ...scope]); + // Keep node identities stable and tick exactly once per observation. + await read("window.tickNoise(); true"); + const learnArgs = [ + "press", + "Space", + ...scope, + "--context", + initial.context_id, + "--selector", + "#pause", + ]; + const refused = await bsk(learnArgs); + assert.notEqual(refused.exitCode, 0); + assert.match(refused.result.message, /changed by itself/); + assert.equal(refused.result.data?.effect_state, "none"); + learning = { refused }; + if (scenario.retryRefused) { + learning.retry = await bsk(learnArgs); + assert.notEqual(learning.retry.exitCode, 0); + assert.equal( + learning.retry.result.message, + "This action requires the context returned by the latest cooperate sync", + ); + assert.equal(learning.retry.result.data?.effect_state, "none"); + } + const report = await read("window.report()"); + assert.equal(report.keydowns, 0); + assert.equal(report.paused, false); + } + if (scenario.continuous) { + await read("window.startNoise(); true"); + await delay(100); + } + let sync = await ok(["cooperate", "sync", ...scope]); + if (scenario.continuous) { + for (let i = 0; i < 2; i++) { + await delay(100); + sync = await ok(["cooperate", "sync", ...scope]); + } + } + if (scenario.focusAfterFirstSync) { + await read( + `document.querySelector(${JSON.stringify(scenario.focusAfterFirstSync)}).focus(); true`, + ); + sync = await ok(["cooperate", "sync", ...scope]); + assert(sync.observation, "A new recipient must refresh semantic focus evidence"); + } + const before = await read("window.report()"); + assert.equal(before.paused, false); + if (scenario.ownershipCheck || scenario.variant === "form-range") { + assert.equal( + before.progressLabel, + scenario.businessLabel ? "Order amount" : "Playback progress", + "The regression must keep the media-like accessible name", + ); + assert.equal( + before.formOwner, + scenario.variant === "form-range" || (scenario.formRole && !scenario.nonForm) + ? "order-form" + : null, + ); + if (scenario.ownershipCheck) + assert.equal(before.controlName, scenario.named ? "start_time" : ""); + assert.equal(before.progressShadow, !!scenario.controlShadow); + } + // Start page-origin updates after the observation, before the proposed input. + await read( + scenario.learn || scenario.manual + ? "window.tickNoise(); true" + : "window.startNoise(); true", + ); + let progressReport; + let sourceBaseline; + let atAction; + try { + atAction = await poll(async () => { + progressReport = await read("window.report()"); + // A reload resets the native frame counter. Establish a new + // baseline only after currentSrc changes, then require more + // decoded frames from that same source on a later read. + if (scenario.source) { + if (progressReport.currentSrc === before.currentSrc || !progressReport.currentSrc) + return; + if (sourceBaseline?.currentSrc !== progressReport.currentSrc) { + sourceBaseline = progressReport; + return; + } + } + const advanced = + progressReport.decodedFrames > (sourceBaseline ?? before).decodedFrames; + return advanced && progressReport.ticks > before.ticks && progressReport; + }, "actual video frames and page updates before input"); + } catch (error) { + await writeFile( + join(artifacts, scenario.name + ".progress-error.json"), + JSON.stringify({ before, current: progressReport }, null, 2), + ); + throw error; + } + if (scenario.source) assert.notEqual(atAction.currentSrc, before.currentSrc); + assert(atAction.ticks > before.ticks, "Page updates must run before the input"); + if (atAction.progress !== undefined) assert.notEqual(atAction.progress, before.progress); + if (scenario.variant === "text") assert.notEqual(atAction.clock, before.clock); + if (["field", "combined", "same-region-field"].includes(scenario.variant)) + assert.notEqual(atAction.order, before.order); + if (scenario.learn || scenario.variant === "focused-price") + assert.notEqual(atAction.price, before.price); + if (scenario.focus && scenario.variant !== "body-recipient") + assert.notEqual(atAction.focused, before.focused); + const wait = await ok(["cooperate", "wait", ...scope, "--timeout-ms", "0"]); + const action = await bsk([ + "press", + scenario.key ?? "Space", + ...scope, + "--context", + sync.context_id, + ...(scenario.target ? ["--selector", scenario.targetSelector ?? "#pause"] : []), + ...(scenario.dependencies + ? (Array.isArray(scenario.dependencies) ? scenario.dependencies : ["#pause"]).flatMap( + (dependency) => ["--depends-on", dependency], + ) + : []), + ]); + const after = await read("window.report()"); + const evidence = { + ...scenario, + session, + learning, + before, + atAction, + sourceBaseline, + coordinationReady: wait.ready, + action, + after, + }; + results.push(evidence); + await writeFile( + join(artifacts, scenario.name + ".json"), + JSON.stringify(evidence, null, 2), + ); + if (scenario.name === "timer_text_untargeted_space") { + // Diagnostic only: all input and page-state assertions still run. + try { + const screenshot = await send("Page.captureScreenshot", { format: "png" }, oracle); + await writeFile( + join(artifacts, "blocked-preview.png"), + Buffer.from(screenshot.data, "base64"), + ); + } catch (error) { + await writeFile(join(artifacts, "blocked-preview-error.txt"), String(error)); + } + } + assert.equal(wait.ready, true, "No person should be holding control"); + assert.equal(after.purchases, 0, "No simulated purchase may execute"); + if (scenario.allow) { + assert.equal(action.exitCode, 0, JSON.stringify(action)); + assert.equal(after.keydowns, 1); + assert.equal(after.paused, true); + } else { + assert.notEqual(action.exitCode, 0, JSON.stringify(action)); + assert.equal( + action.result.message, + scenario.focus + ? "The keyboard focus changed; synchronize before pressing a key" + : scenario.source + ? "The media source changed; synchronize and replan" + : "The action's observed content changed; synchronize and replan", + ); + assert.equal(action.result.data?.effect_state, "none"); + assert.equal(after.keydowns, 0); + assert.equal(after.pauseCalls, 0); + assert.equal(after.paused, false); + } + console.log( + JSON.stringify({ + name: scenario.name, + exit: action.exitCode, + keydowns: after.keydowns, + paused: after.paused, + elapsedMs: action.durationMs, + }), + ); + } finally { + if (oracle) await send("Target.detachFromTarget", { sessionId: oracle }).catch(() => {}); + await ok(["session", "stop", session]); + } + } + }, + ); + await writeFile( + join(artifacts, "result.json"), + JSON.stringify({ passed: true, browserVersion, cases: results }, null, 2), + ); + console.log(`PASS ${results.length} real-browser cases; artifacts=${artifacts}`); +} catch (error) { + await writeFile( + join(artifacts, "result.json"), + JSON.stringify( + { passed: false, browserVersion, error: String(error.stack || error), cases: results }, + null, + 2, + ), + ); + throw error; +} finally { + await writeFile(join(artifacts, "daemon.log"), daemonOutput); + if (daemon && daemon.exitCode === null && daemon.signalCode === null) { + const exited = once(daemon, "exit"); + daemon.kill("SIGTERM"); + const force = setTimeout(() => daemon.kill("SIGKILL"), 5000); + await exited; + clearTimeout(force); + } + await new Promise((resolve) => server.close(resolve)); + await rm(runtime, { recursive: true, force: true }); +} diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index ad368b99..9831fb53 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -34,7 +34,8 @@ set is uncertain, omit it and retain whole-page checking. Text that changes by i sync and action (a clock, a counter) is reported once with the before/after text. After you sync and replan, further changes to that text no longer block actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or -selector, such as a bare key press, always check the whole page exactly. Added, removed or +selector never learn incidental text; a bare key press checks the whole page exactly unless +explicit dependencies narrow its decision regions. Added, removed or re-rendered elements and text without its own accessible node (such as `aria-hidden` content) keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an @@ -46,6 +47,19 @@ A regular coordination result is not a failed browser attempt. Use wait then syn the next action from the actual result. Use request-help only for an actual human requirement such as login or a captcha. Explicit user stop always ends the task. +On continuously updating pages, declare the observed decision regions instead of repeatedly +retrying a whole-page action. `press` honors dependencies even without a focus target; specify +a target when the key must be delivered there. Untargeted keys reject a changed focus and +protect the actual focused region even if the declared dependencies are elsewhere. Unknown +or ambiguous focus keeps whole-page checking; untargeted scripts also keep that strict check. +A clearly labelled media timeline with an unambiguous video/audio association may advance +beside a pause button, including in the same player region. Directly observing the timeline, +its media element or a containing element keeps its value exact. Submitted inputs, form members, +ordinary sliders and other business values remain protected. Media identity, child `` +declarations and the selected source stay exact. Form ownership remains protected even when +a form is omitted from the observation. Unknown media controls or unavailable form-ownership +evidence retain ordinary checks. + ## Scripts and file transfers Scripts, upload/download, active hover observation, full-page screenshots, device emulation diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 2b6855ac..50d7a58c 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -34,7 +34,12 @@ export interface VomNode { value?: string; /** Checked state is separate from the control's submitted value. */ checked?: boolean | "mixed"; + /** Browser accessibility focus, used to protect implicit keyboard recipients. */ + focused?: boolean; placeholder?: string; + /** Native form ownership, or raw DOM form containment for custom widgets. + * Absence means unknown, never proof that a control is outside a form. */ + inForm?: boolean; inputState?: "empty" | "filled" | "default" | "unknown"; href?: string; // hostname of external link target; omitted for same-origin links text?: string;