From 3f05d05dd89c1755dd0a9d3004441b20391e82a8 Mon Sep 17 00:00:00 2001 From: Zhang GH <5351707+iuyo5678@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:49:53 +0800 Subject: [PATCH 1/3] fix(cooperation): preserve actions around incidental media progress --- .../__tests__/cooperation-capture.test.ts | 49 ++ .../src/content/cooperation-capture.ts | 40 +- .../src/cooperation/controller.test.ts | 296 ++++++++++- apps/extension/src/cooperation/controller.ts | 39 +- .../src/cooperation/media-progress.ts | 37 ++ .../cooperation/observation-evidence.test.ts | 118 +++++ .../src/cooperation/observation-evidence.ts | 35 +- apps/extension/src/cooperation/types.ts | 4 + .../bsk-cli/skill/references/cooperation.md | 11 +- .../regression/cooperation-video/README.md | 41 ++ .../regression/cooperation-video/fixture.html | 87 ++++ .../regression/cooperation-video/preview.webm | Bin 0 -> 31545 bytes .../regression/cooperation-video/run.mjs | 476 ++++++++++++++++++ .../skill/references/cooperation.md | 11 +- 14 files changed, 1228 insertions(+), 16 deletions(-) create mode 100644 apps/extension/src/cooperation/media-progress.ts create mode 100644 evals/browser/cases/regression/cooperation-video/README.md create mode 100644 evals/browser/cases/regression/cooperation-video/fixture.html create mode 100644 evals/browser/cases/regression/cooperation-video/preview.webm create mode 100644 evals/browser/cases/regression/cooperation-video/run.mjs diff --git a/apps/extension/src/content/__tests__/cooperation-capture.test.ts b/apps/extension/src/content/__tests__/cooperation-capture.test.ts index e67f0c18..abb981ad 100644 --- a/apps/extension/src/content/__tests__/cooperation-capture.test.ts +++ b/apps/extension/src/content/__tests__/cooperation-capture.test.ts @@ -404,3 +404,52 @@ it("announces renderer readiness once after the collection listener exists", () expect(chrome.runtime.sendMessage).toHaveBeenCalledWith({ kind: COOPERATION_READY }); expect(read()).toHaveProperty("document"); }); + +it("classifies a non-submitted media timeline without masking business fields", () => { + document.body.innerHTML = + ''; + const read = collector(); + const before = read()!; + expect(before.fields[0]).toMatchObject({ value: "1", kind: "media_progress" }); + expect(before.fields.slice(1).every((field) => field.kind === undefined)).toBe(true); + const progress = document.querySelector("#timeline")!; + progress.value = "2"; + expect(read()!.fields[0]).toMatchObject({ value: "2", kind: "media_progress" }); + progress.name = "order_amount"; + expect(read()!.fields[0].kind).toBeUndefined(); +}); + +it.each([ + "form", + "external", + "ambiguous", + "missing", +])("does not classify a %s timeline as incidental", (kind) => { + document.body.innerHTML = + ''; + const timeline = document.querySelector("#timeline")!; + if (kind === "form") { + const form = document.createElement("form"); + document.body.append(form); + form.append(timeline); + } + if (kind === "external") { + const form = document.createElement("form"); + form.id = "order"; + document.body.append(form); + timeline.setAttribute("form", "order"); + } + if (kind === "ambiguous") document.body.append(document.createElement("video")); + if (kind === "missing") document.querySelector("video")!.remove(); + expect(collector()()!.fields[0].kind).toBeUndefined(); +}); + +it("reports programmatic focus changes even without a human event", () => { + document.body.innerHTML = ''; + const read = collector(); + const first = read()!.focus; + document.querySelector("#order")!.focus(); + const current = read()!; + expect(current.focus).not.toBe(first); + expect(current.human).toBe(0); +}); diff --git a/apps/extension/src/content/cooperation-capture.ts b/apps/extension/src/content/cooperation-capture.ts index 77cf2c99..9bf8aa18 100644 --- a/apps/extension/src/content/cooperation-capture.ts +++ b/apps/extension/src/content/cooperation-capture.ts @@ -1,3 +1,4 @@ +import { playbackProgressMedia } from "@/cooperation/media-progress"; import { COOPERATION_ARM, COOPERATION_DISABLE, @@ -119,12 +120,16 @@ export function attachCooperationCapture() { }; const read = () => { mutations(observer.takeRecords()); + let active = document.activeElement; + while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement; + state.focus = active ? id(active) : undefined; if ([...roots].some((root) => root instanceof ShadowRoot && !root.host.isConnected)) { observer.disconnect(); roots.clear(); } for (const element of fields.keys()) if (!element.isConnected) fields.delete(element); let fieldCount = 0; + const mediaByRoot = new Map(); state.fieldsComplete = true; for (const target of controls(document)) { if ( @@ -140,6 +145,34 @@ export function attachCooperationCapture() { : (target.textContent ?? ""); const value = fullValue.slice(0, 500); const name = label(target); + let mediaProgress = false; + if (target instanceof HTMLInputElement && target.type === "range") { + const root = target.getRootNode() as Document | ShadowRoot; + let media = mediaByRoot.get(root); + if (!media) { + media = [...root.querySelectorAll("video,audio")].map((element) => ({ id: element.id })); + mediaByRoot.set(root, media); + } + const labelledBy = (target.getAttribute("aria-labelledby") ?? "") + .split(/\s+/) + .filter(Boolean) + .map((id) => root.getElementById(id)?.textContent ?? "") + .join(" "); + const labels = [...new Set(target.labels ?? [])] + .map((label) => label.textContent) + .join(" "); + mediaProgress = !!playbackProgressMedia( + { + tag: "input", + name: target.getAttribute("aria-label") || labelledBy || labels, + attrs: Object.fromEntries( + [...target.attributes].map((attr) => [attr.name, attr.value]), + ), + inForm: target.form !== null, + }, + media, + ); + } fieldCount++; if (fieldCount > MAX_CHANGES || fullValue.length > 500) state.fieldsComplete = false; const sampled = sampledValues.get(target); @@ -147,7 +180,12 @@ export function attachCooperationCapture() { sampledValues.set(target, fullValue); const old = fields.get(target); if (fields.size < MAX_CHANGES || old) - fields.set(target, { id: id(target), label: name, value }); + fields.set(target, { + id: id(target), + label: name, + value, + ...(mediaProgress ? { kind: "media_progress" as const } : {}), + }); } for (const element of fields.keys()) if (!element.isConnected) fields.delete(element); state.busy = composing || pointer || keys.size > 0; diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 335d748a..266e4e10 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -1,4 +1,4 @@ -import type { VomScene } from "@browser-skill/vom"; +import type { VomNode, VomScene } from "@browser-skill/vom"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { SessionManager } from "@/session-manager/manager"; @@ -860,6 +860,26 @@ describe("shared-tab dispatch fences", () => { expect(f.native).not.toHaveBeenCalled(); }); + it.each([ + ["e5"], + ["e4", "e5"], + ["e5", "e4"], + ])("keeps every untargeted key dependency exact after learning: %j", async (...dependsOn) => { + live((tick) => scene(`10:00:0${tick}`)); + refs(); + const f = await fixture(); + expect((await run(f, f.first.context_id, { ref: "e4" })).message).toContain( + "changed by itself", + ); + expect( + await run(f, (await f.command("sync")).context_id, { + key: "Space", + dependsOn, + }), + ).toMatchObject({ message: changed, data: { effect_state: "none" } }); + expect(f.native).not.toHaveBeenCalled(); + }); + it("does not let a refused context retry with newly learned tolerance", async () => { live((tick) => scene(`10:00:0${tick}`)); const f = await fixture(); @@ -2161,3 +2181,277 @@ describe("default collaboration preserves the original unattended lifetime", () expect(f.sessionActivity).toHaveBeenLastCalledWith("shared", false); }); }); + +describe("looping preview action guards", () => { + type PreviewNode = Pick; + const previewScene = (seconds: number, range = false): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: ( + [ + { id: 1, parentId: null, tag: "main", role: "main", name: "" }, + { id: 2, parentId: 1, tag: "form", role: "form", name: "Player controls" }, + { id: 4, parentId: 2, tag: "button", role: "button", name: "Pause" }, + { + id: 6, + parentId: 1, + tag: "video", + role: "Video", + name: "Preview", + attrs: { id: "video", src: "clip.webm" }, + }, + range + ? { + id: 5, + parentId: 1, + tag: "input", + role: "slider", + name: "Playback progress", + value: String(seconds), + attrs: { type: "range" }, + } + : { id: 5, parentId: 1, tag: "aside", role: "region", name: `Playback time: ${seconds}` }, + ] as PreviewNode[] + ).map((node) => ({ + ...node, + backendNodeId: node.id, + frameId: "frame", + rect: { x: 0, y: 0, w: 20, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + })), + }); + it.each([ + { + name: "untargeted Space is rejected by unrelated timer text", + ref: false, + deps: false, + range: false, + changes: true, + allowed: false, + }, + { + name: "targeted Space with declared dependencies permits outside timer text", + ref: true, + deps: true, + range: false, + changes: true, + allowed: true, + }, + { + name: "untargeted Space honors its declared dependencies", + ref: false, + deps: true, + range: false, + changes: true, + allowed: true, + }, + { + name: "outside playback range does not revoke a scoped action", + ref: true, + deps: true, + range: true, + changes: true, + allowed: true, + }, + { + name: "revision changes with unchanged semantics permit Space", + ref: false, + deps: false, + range: false, + changes: false, + allowed: true, + }, + ])("$name", async ({ ref, deps, range, changes, allowed }) => { + vi.mocked(handleSnapshot).mockImplementationOnce(async (_m, _p, options) => { + options!.onScene?.(previewScene(1, range)); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + vi.mocked(captureVomObservation).mockImplementation(async (_c, _t, _u, options) => { + options?.onScene?.(previewScene(changes ? 2 : 1, range)); + return {} as Awaited>; + }); + const f = await fixture(); + f.change({ structure: 1 }); + // A ready coordination state does not mean page semantics stopped changing. + expect(await f.command("wait")).toMatchObject({ ready: true }); + const result = await f.controller.run( + { + id: "preview-space", + method: "tool.press", + params: { + session_id: "shared", + tab_id: 7, + context_id: f.first.context_id, + key: "Space", + ...(ref ? { ref: "e4" } : {}), + ...(deps ? { depends_on: ["e4"] } : {}), + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }), + ); + if (allowed) { + expect(result).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); + } else { + expect(result).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + } + if (deps) expect(resolveActionTarget).toHaveBeenCalled(); + else if (!ref) expect(resolveActionTarget).not.toHaveBeenCalled(); + }); +}); + +it.each([ + undefined, + "media_progress" as const, +])("sync tolerates only classified media progress: %s", async (kind) => { + const f = await fixture({ sync: false }); + const field = { + id: "timeline", + label: "Playback progress", + value: "1", + ...(kind ? { kind } : {}), + }; + f.change({ fields: [field] }, false); + vi.mocked(handleSnapshot).mockImplementationOnce(async () => { + f.change({ fields: [{ ...field, value: "2" }] }, false); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + const result = await f.command("sync"); + if (kind) expect(result.context_id).toEqual(expect.any(String)); + else + expect(result).toMatchObject({ + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); +}); + +it("rejects untargeted key dependencies that cannot be resolved", async () => { + const f = await fixture(); + vi.mocked(resolveActionTarget).mockResolvedValueOnce({ + code: "not_found", + message: "Dependency disappeared", + }); + const result = await f.controller.run( + { + id: "key", + method: "tool.press", + params: { + session_id: "shared", + context_id: f.first.context_id, + key: "Space", + depends_on: ["#gone"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }), + ); + expect(result).toMatchObject({ code: "not_found" }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("does not retarget an untargeted key when page code changes focus", async () => { + const f = await fixture(); + const result = await f.act( + f.first.context_id, + async (runner) => { + f.change({ focus: "order-amount" }, false); + return runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }); + }, + "tool.press", + ); + expect(result).toMatchObject({ + message: "The keyboard focus changed; synchronize before pressing a key", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("allows focus changes caused by a key that has already started", async () => { + const f = await fixture(); + const result = await f.act( + f.first.context_id, + async (runner) => { + await runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: " " }); + f.change({ focus: "next-field" }, false); + await runner.send(7, "Input.dispatchKeyEvent", { type: "char", key: " ", text: " " }); + await runner.send(7, "Input.dispatchKeyEvent", { type: "keyUp", key: " " }); + return { ok: true }; + }, + "tool.press", + ); + expect(result).toEqual({ ok: true }); + expect(f.native).toHaveBeenCalledTimes(3); +}); + +it("refreshes timeline evidence on a delta when that value is an explicit decision dependency", async () => { + const scene = (value: string): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: [ + { + id: 4, + parentId: null, + backendNodeId: 4, + frameId: "frame", + tag: "input", + role: "slider", + name: "Playback progress", + value, + attrs: { type: "range" }, + rect: { x: 0, y: 0, w: 100, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + const f = await fixture({ sync: false }); + const field = { + id: "timeline", + label: "Playback progress", + kind: "media_progress" as const, + value: "1", + }; + f.change({ fields: [field] }, false); + vi.mocked(handleSnapshot).mockImplementationOnce(async (_m, _p, deps) => { + deps!.onScene?.(scene("1")); + return { text: "Playback progress 1", ref_count: 1, tab_id: 7, truncated: false }; + }); + await f.command("sync"); + f.change({ fields: [{ ...field, value: "2" }] }, false); + vi.mocked(captureVomObservation).mockImplementation(async (_c, _t, _u, options) => { + options?.onScene?.(scene("2")); + return {} as Awaited>; + }); + const synced = await f.command("sync"); + expect(synced).toMatchObject({ + kind: "delta", + changes: [expect.objectContaining({ value: "2" })], + }); + expect(captureVomObservation).toHaveBeenCalledOnce(); + f.change({ structure: 1 }); + const result = await f.controller.run( + { + id: "timeline-key", + method: "tool.press", + params: { + session_id: "shared", + context_id: synced.context_id, + key: "ArrowRight", + ref: "e4", + depends_on: ["e4"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: "ArrowRight" }), + ); + expect(result).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); +}); diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 4366b7a8..5c48a30c 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -138,11 +138,18 @@ function quote(text: string): string { const points = Array.from(text); return JSON.stringify(points.length > 40 ? `${points.slice(0, 39).join("")}…` : text); } -function sameFields(a: Frames, b: Frames): boolean { +function sameFields(a: Frames, b: Frames, ignoreMediaProgress = false): boolean { + const stableFields = (frame: FrameState | undefined) => + frame?.fields.map((field) => + ignoreMediaProgress && field.kind === "media_progress" + ? { ...field, value: undefined } + : field, + ); return ( a.size === b.size && [...a].every( - ([id, frame]) => JSON.stringify(frame.fields) === JSON.stringify(b.get(id)?.fields), + ([id, frame]) => + JSON.stringify(stableFields(frame)) === JSON.stringify(stableFields(b.get(id))), ) ); } @@ -608,7 +615,7 @@ export class CooperationController implements ControlAuthority { signal.aborted || state.epoch !== epoch || !sameFrames(before, after, "human") || - !sameFields(before, after) || + !sameFields(before, after, true) || (delta && !sameFrames(before, after, "structure")) || this.humanActive(state) || state.held @@ -1013,6 +1020,15 @@ export class CooperationController implements ControlAuthority { if (denied) return denied.message; if (!settledNavigation && state.unavailable) return "Page interaction coverage changed; wait and synchronize"; + if ( + req.method === "tool.press" && + !nativeInputStarted && + !params.ref && + !params.selector && + context && + [...context.frames].some(([id, frame]) => frame.focus !== state.frames.get(id)?.focus) + ) + return "The keyboard focus changed; synchronize before pressing a key"; if (!settledNavigation && state.epoch !== epoch) return "The observed page or control state changed; synchronize"; }; @@ -1053,7 +1069,7 @@ export class CooperationController implements ControlAuthority { // observations, whose region can be protected. A target-less key press or // script acts on whatever is focused, and a selector may resolve outside // the observation (for example inside aria-hidden content); both keep the - // exact whole-page check. + // exact whole-page check unless explicit dependencies narrow the decision. const anchored = actionTarget !== undefined && observesTarget(context.evidence, actionTarget) && @@ -1066,10 +1082,10 @@ export class CooperationController implements ControlAuthority { // exact; sameObservationEvidence only relaxes broad layout around a target. const matches = (keys?: ReadonlySet) => { if (evidenceTargets) { - const [own, ...dependencies] = evidenceTargets; return ( - sameObservationEvidence(context.evidence!, evidence!, own, keys) && - dependencies.every((dependency) => + (!actionTarget || + sameObservationEvidence(context.evidence!, evidence!, actionTarget, keys)) && + evidenceTargets.every((dependency) => sameObservationEvidence(context.evidence!, evidence!, dependency), ) ); @@ -1113,7 +1129,7 @@ export class CooperationController implements ControlAuthority { ]; const [first] = reported; throw new Error( - (revoked = `Page text changed by itself (${quote(first.before)} → ${quote(first.after)}${reported.length > 1 ? ` and ${reported.length - 1} more` : ""}); synchronize and replan. After that sync, further changes to that text no longer block actions on a ref or selector elsewhere. They still block actions whose target shares a form, list, dialog, panel or other container with it, and actions without a ref or selector. If the decision relies on that text, declare it with --depends-on to keep it exact`), + (revoked = `Page text changed by itself (${quote(first.before)} → ${quote(first.after)}${reported.length > 1 ? ` and ${reported.length - 1} more` : ""}); synchronize and replan. After that sync, further changes to that text no longer block actions on a ref or selector elsewhere. They still block actions whose target shares a form, list, dialog, panel or other container with it, and actions without a ref, selector or explicit dependencies. If the decision relies on that text, declare it with --depends-on to keep it exact`), ); } throw new Error((revoked = "The action's observed content changed; synchronize and replan")); @@ -1138,7 +1154,6 @@ export class CooperationController implements ControlAuthority { ); if (isRpcError(resolved)) return resolved; actionTarget = resolved; - if (params.depends_on?.length) evidenceTargets = [resolved]; } if (params.depends_on !== undefined) { if ( @@ -1152,7 +1167,10 @@ export class CooperationController implements ControlAuthority { code: "invalid_params", message: "depends_on requires at most 16 observed refs/selectors", }; - if (evidenceTargets) + if (params.depends_on.length) { + // Dependencies constrain the decision even when a key has no target. + // Keep them separate from the target so learned text never relaxes them. + evidenceTargets = []; for (const dependency of params.depends_on) { const resolved = await resolveActionTarget( guardedRunner(cdp), @@ -1164,6 +1182,7 @@ export class CooperationController implements ControlAuthority { if (isRpcError(resolved)) return resolved; evidenceTargets.push(resolved); } + } } await verifyEvidence(); state.context = undefined; // Consume before execution, including failed/unknown replies. diff --git a/apps/extension/src/cooperation/media-progress.ts b/apps/extension/src/cooperation/media-progress.ts new file mode 100644 index 00000000..072205e6 --- /dev/null +++ b/apps/extension/src/cooperation/media-progress.ts @@ -0,0 +1,37 @@ +/** Range inputs also represent business data. Only explicitly named timelines + * with an unambiguous media association may tolerate incidental progress. */ +export interface PlaybackProgressControl { + tag: string; + role?: string; + name?: string; + attrs?: Readonly>; + inForm: boolean; +} + +const timelineName = + /^(?:(?:playback|video|audio|media)\s+(?:progress|position|time(?:\s+scrubber)?|timeline)|seek(?:\s+(?:bar|slider))?|time scrubber|(?:播放|视频|音频)(?:进度|时间|位置|时间轴))$/i; + +/** Explicit aria-controls wins; otherwise require exactly one media element. + * Named/submitted controls, form members and unknown labels stay protected. */ +export function playbackProgressMedia( + control: PlaybackProgressControl, + media: readonly T[], +): T | undefined { + const attrs = control.attrs ?? {}; + if (control.inForm || attrs.form !== undefined || attrs.name?.trim()) return; + const nativeRange = control.tag === "input" && attrs.type?.toLowerCase() === "range"; + if ( + !nativeRange && + control.tag !== "progress" && + !["slider", "progressbar"].includes(control.role?.toLowerCase() ?? "") + ) + return; + if (!timelineName.test(control.name?.trim().replace(/\s+/g, " ") ?? "")) return; + const controls = attrs["aria-controls"]?.trim().split(/\s+/).filter(Boolean); + if (controls?.length) { + if (controls.length !== 1) return; + const matches = media.filter((item) => item.id === controls[0]); + return matches.length === 1 ? matches[0] : undefined; + } + return media.length === 1 ? media[0] : undefined; +} diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index 00b496ec..0d74ccb3 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -298,3 +298,121 @@ describe("learned text around a target inside
or without a region", () => expect(allowed(flat, 2, 3)).toBe(true); }); }); + +const mediaScene = () => [ + ...scene(), + { ...node(6, 1, "Video", "Preview"), tag: "video", attrs: { id: "preview", src: "clip.webm" } }, + { + ...node(7, 1, "slider", "Playback progress"), + tag: "input", + value: "1", + attrs: { type: "range" }, + }, +]; + +it("tolerates only the value of an outside media timeline for a declared scope", () => { + const before = mediaScene(); + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(true); + expect(sameObservationEvidence({ nodes: before }, { nodes: after })).toBe(false); + expect( + sameObservationEvidence( + { nodes: before }, + { nodes: after }, + { backendNodeId: 7, frameId: "frame" }, + ), + ).toBe(false); +}); + +it.each([ + "business slider", + "business text", + "named input", + "form member", + "external form", + "missing media", + "ambiguous media", + "different frame", + "wrong relation", +])("protects %s despite video playback", (kind) => { + const before = mediaScene(); + const progress = before[6]; + if (kind === "business slider") progress.name = "Order amount"; + if (kind === "business text") { + progress.role = "textbox"; + progress.attrs = { type: "text" }; + } + if (kind === "named input") progress.attrs!.name = "amount"; + if (kind === "form member") { + before.push(node(8, 1, "form", "Order")); + progress.domAncestorIds = [8, 1]; + } + if (kind === "external form") progress.attrs!.form = "order"; + if (kind === "missing media") before[5].tag = "div"; + if (kind === "ambiguous media") before.push({ ...node(8, 1, "Video"), tag: "video" }); + if (kind === "different frame") before[5].frameId = "other"; + if (kind === "wrong relation") progress.attrs!["aria-controls"] = "order"; + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it("supports an explicit media relation when more than one player exists", () => { + const before = mediaScene(); + before.push({ ...node(8, 1, "Video"), tag: "video", attrs: { id: "other" } }); + before[6].attrs!["aria-controls"] = "preview"; + const after = structuredClone(before); + after[6].value = "2"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(true); +}); + +it.each([ + "identity", + "label", + "disabled", + "removed", + "media identity", + "media source", + "business value", +])("retains %s evidence while ignoring timeline ticks", (kind) => { + const before = mediaScene(); + before.push({ ...node(8, 1, "textbox", "Order total"), value: "100" }); + const after = structuredClone(before); + after[6].value = "2"; + if (kind === "identity") after[6].backendNodeId = 70; + if (kind === "label") after[6].name = "Order amount"; + if (kind === "disabled") after[6].disabled = true; + if (kind === "removed") after.splice(6, 1); + if (kind === "media identity") after[5].backendNodeId = 60; + if (kind === "media source") after[5].attrs!.src = "other.webm"; + if (kind === "business value") after[7].value = "200"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it.each([ + "min", + "max", + "step", + "aria-valuemin", + "aria-valuemax", +])("protects timeline %s while its value advances", (attribute) => { + const before = mediaScene(); + const after = structuredClone(before); + after[6].value = "2"; + after[6].attrs![attribute] = "10"; + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); +}); + +it("never learns media values or relaxes an explicitly observed timeline", () => { + const before = { nodes: mediaScene() }; + const after = structuredClone(before); + after.nodes[6].value = "2"; + const learned = new Set(["frame:7"]); + expect(contentChanges(before, after)).toBeUndefined(); + expect(sameObservationEvidence(before, after, undefined, learned)).toBe(false); + expect( + sameObservationEvidence(before, after, { backendNodeId: 7, frameId: "frame" }, learned), + ).toBe(false); + expect(sameObservationEvidence(before, after, target, learned)).toBe(true); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index 0e3a7a15..a00ab9cf 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -1,4 +1,5 @@ import type { VomNode, VomScene } from "@browser-skill/vom"; +import { playbackProgressMedia } from "./media-progress"; /** Reuse the observation's semantic nodes. CSS classes and pixel positions are * deliberately absent; the interaction layer rechecks the live hit target. */ @@ -33,6 +34,12 @@ const semanticAttributes = [ "aria-pressed", "aria-disabled", "aria-hidden", + "aria-controls", + "aria-valuemin", + "aria-valuemax", + "min", + "max", + "step", "readonly", "required", ]; @@ -232,6 +239,27 @@ function signature( learnedText.set(holder, set); } } + // Only progress values outside the protected region may be incidental. + // Keep the original nodes intact: media controls cannot become learned text. + const media = nodes.filter((node) => node.tag === "video" || node.tag === "audio"); + const forms = new Set(nodes.filter((node) => node.tag === "form").map((node) => node.id)); + const ignoredProgress = new Map(); + if (target) + for (const node of nodes) { + if (inside(node) || node.value === undefined) continue; + const owner = playbackProgressMedia( + { + ...node, + inForm: + tree.ancestors(node).some((ancestor) => ancestor.tag === "form") || + (node.domAncestorIds ?? []).some((id) => forms.has(id)), + }, + media + .filter((item) => item.frameId === node.frameId) + .map((item) => ({ id: item.attrs?.id, node: item })), + ); + if (owner) ignoredProgress.set(node, owner.node); + } // Identity, structure, state and visibility of learned nodes remain checked. const facts = (node: VomNode, relax: boolean) => { const tolerate = relax && tolerated(node); @@ -246,7 +274,7 @@ function signature( ? withoutLearned(node.name, learnedText.get(node)) : node.name, tolerate ? undefined : node.text, - node.value, + ignoredProgress.has(node) ? undefined : node.value, node.checked, node.disabled, node.inert, @@ -266,6 +294,8 @@ function signature( .filter((node) => node.value !== undefined || node.checked !== undefined) .map((node) => facts(node, false)), nodes.filter((node) => node.modal).map((node) => facts(node, false)), + // Replacing the player or its source still invalidates the observation. + [...new Set(ignoredProgress.values())].map((node) => [...facts(node, false), node.attrs?.src]), ]); } @@ -276,7 +306,8 @@ function signature( * wherever it appears as whole words. Without a target this applies to the * whole page. With a target it never applies inside the target's own region, * as described in signature(). Form values, checked states and modal layers - * stay exact everywhere. + * stay exact everywhere except positively identified, non-submitted media + * progress values outside the protected region of a scoped comparison. */ export function sameObservationEvidence( before: ObservationEvidence, diff --git a/apps/extension/src/cooperation/types.ts b/apps/extension/src/cooperation/types.ts index 22416754..9cd0fc2c 100644 --- a/apps/extension/src/cooperation/types.ts +++ b/apps/extension/src/cooperation/types.ts @@ -12,6 +12,8 @@ export interface FieldChange { id?: string; label: string; value: string; + /** A positively identified, non-submitted media timeline. */ + kind?: "media_progress"; } export interface HumanActivity { @@ -32,6 +34,8 @@ export interface HumanActivity { export interface FrameState { document: string; + /** Collector-local identity of the deep active element in this document. */ + focus?: string; revision: number; structure: number; human: number; diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index bf01819f..184c9dfc 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -40,7 +40,8 @@ set is uncertain, omit it and retain whole-page checking. Text that changes by i sync and action (a clock, a counter) is reported once with the before/after text. After you sync and replan, further changes to that text no longer block actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or -selector, such as a bare key press, always check the whole page exactly. Added, removed or +selector never learn incidental text; a bare key press checks the whole page exactly unless +explicit dependencies narrow its decision regions. Added, removed or re-rendered elements and text without its own accessible node (such as `aria-hidden` content) keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an @@ -52,6 +53,14 @@ A regular coordination result is not a failed browser attempt. Use wait then syn the next action from the actual result. Use request-help only for an actual human requirement such as login or a captcha. Explicit user stop always ends the task. +On continuously updating pages, declare the observed decision regions instead of repeatedly +retrying a whole-page action. `press` honors dependencies even without a focus target; specify +a target when the key must be delivered there. Untargeted keys reject a changed focus. A +clearly labelled media timeline outside all declared regions may advance without revoking the +action when it is unambiguously associated with a video/audio element. Submitted inputs, form members, +ordinary sliders, and other business values remain protected; declaring the timeline itself +as a dependency protects its value too. Unknown media controls retain the ordinary checks. + ## Scripts and file transfers Scripts, upload/download, active hover observation, full-page screenshots, device emulation diff --git a/evals/browser/cases/regression/cooperation-video/README.md b/evals/browser/cases/regression/cooperation-video/README.md new file mode 100644 index 00000000..80fc5306 --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/README.md @@ -0,0 +1,41 @@ +# Continuous video progress and cooperative actions + +This regression runs the actual CLI, daemon, extension and an isolated Chromium profile. +The fixture loops a bundled synthetic WebM in a real video element. No remote media, live +recording, or existing browser profile is required. A separate CDP connection checks decoded +frames, received key events and the final paused state independently of the CLI response. + +Build and run from the repository root, using a cooperative CLI build: + +```sh +BSK_DAEMON_WS_URL=ws://127.0.0.1:52931 pnpm --filter @browser-skill/extension build +BSK_COOP_CLI=/path/to/bsk \ +BSK_COOP_CHROME=/path/to/chromium \ +node evals/browser/cases/regression/cooperation-video/run.mjs +``` + +`BSK_COOP_PORT` must match the built extension's endpoint and must not be the normal daemon +port 52800. `BSK_COOP_EXTENSION` overrides the built extension directory. Chrome for Testing +uses `Extensions.loadUnpacked`; use `BSK_COOP_LOAD_CDP=0` with builds that support the load +extension command-line flags instead. `BSK_COOP_ARTIFACTS` selects the result directory. + +The 20 cases cover continuously advancing progress during sync (including incremental sync), +dependencies without a focus target, ordinary business sliders, submitted timelines, +form-owned timelines, simultaneous video and order changes, and a programmatic focus change. +They also combine PR #497's learned-text behavior with the media fix: a learned price becomes +exact again when it belongs to the current target or an explicit dependency, bare keys remain +strict, a refused context cannot be replayed, and an explicitly observed timeline stays exact. Positive cases require one received +Space event and a paused video. Negative cases require no Space event, no pause call and +`effect_state: none`. Every created session, daemon and temporary browser profile is cleaned up. + +A dependency must still cover the action's actual decision. A key without a target +or dependencies keeps the exact whole-page check. Targeted actions retain PR #497's bounded +text learning after a refusal and a fresh sync. Only positively +identified, non-submitted media progress outside all declared regions may change incidentally; +ordinary fields/sliders and unknown media controls keep the conservative behavior. + +The two-second VP8 test clip can be regenerated with FFmpeg: + +```sh +ffmpeg -f lavfi -i testsrc2=size=160x90:rate=24 -t 2 -c:v libvpx -b:v 120k -an preview.webm +``` diff --git a/evals/browser/cases/regression/cooperation-video/fixture.html b/evals/browser/cases/regression/cooperation-video/fixture.html new file mode 100644 index 00000000..d96a6350 --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/fixture.html @@ -0,0 +1,87 @@ + + +BrowserSkill looping video regression + +
+

Looping preview

+

Local regression fixture. Space or the Pause button pauses the video.

+ +
+ +
+ +
+ diff --git a/evals/browser/cases/regression/cooperation-video/preview.webm b/evals/browser/cases/regression/cooperation-video/preview.webm new file mode 100644 index 0000000000000000000000000000000000000000..02f550dc432f9c367c0bcde98bed0bea3a5c7eea GIT binary patch literal 31545 zcmcG!gc;prC*#NF$)2bV;mqh_tjc2qGaZ z^{(GvJkRq7yytW8#GNxU=bo8!X3m*m&{!7HDHP;UcYLty2gZiy`8e4qz+RUf_1N1v9tC1!%%j5cdRLH_;y<)cS=PH3cC-y4m>I1PO@19c5! z^|=4H|3{C$JE0Su{!dSCefj@4b{d9yCk*vsGXJP&yYJx`;2ZQGoOpFJV--_v1AX6tY8zP(~_sUEewDJtplT-Cx<_f8^~uR;6U2!SixtC6dlOUO0s z;${c-6ZYMHy|7LB^EEttDdg4CFj}c5T}&gT+4H~x!iDyI)=X9orK~HS{o#4m;)tGh>abqr%5cq# z;cLdr_IMxojoyXX1>~CYQs6@RPy6)VdE143|4rca#kIjDPc`ZD_O=_1tETJo6Wq&3 z=d{N@+qrWQV-e0bH#c6t7l&Zedq~=4ma*;L9|XFs#qM~zNspPb;RqfqcaQ)Qj}j`w}*QjB-%k38gP(@nxWMDZ-A zJj6cqIGz6Z3K3j4{SFUp^TrB~oyohEn)Fqo#B}|s2Yz>Om5SHWb_92-H1qCt#_a)q z=2tIiL^Pd!flu{uF?qVqJ6Y?sDstJ_foZO6TO!_S3Ri{&_%3lozf<`3dn*A=><-g$+(@G)x_7Wpo7@+cKX=+7Hbf3Jue;c;5O`Xa*`gb>wa? zvli!kWlg+ydk%@z-;|6#cxJ=$mdkUsGR>v zG3md38F&63Ixp0!{WYRUVN)7xu^f)o9vh0Nz$f6DFB;^M7r!mU)%z&+dHVEnN{y;$ zLeU*gYnDIJNPF^uedaCEdsBhUbwqV(1)+J^00q*K-=K^;=O89t4M zDeV$FPq3qO9A$p1P7_W3%}w0V<_NWDwL2+HmNFN4m82Z6@-XVdTO1Lzs=G zN*7_>it^OcXE~Dd)Ldg*Qsl+5RZC-h`rueW`OjO?%Wxxm0ZyY@zSl%6_T&gQQvNO! zbN4HwlV5MNaH3qqsMXmR141M2_We9{8Yc+$dTioAQb+6eEo6UUkx#H%D>qHgXNuV2 zDDRxo)7QE!W%ccB-Mgi1?!>DsV(fnQ;JL)nb&=#fPEK}RN26^8K2evh7v;|=^LN{< zKXd!9JZ?A-VykvYw=f!_7Z6-sPlF+hi$-6x(YY+iNd<{2AiJ+3Lc14je@cnJQtG6| z?UOKb&WnF~;p3S9#R_^rNvyz6$4> z1o1t*ap6^lYp>Y&9Ol>~6%Vbv`!la(Bh$k)C5Y>a*38?g%-g=Qs|LB>z4lvye88<2 z@~4##u9>1?4R6#nmjA&=i_Y2ll7HLa_M?u6oMIX#j$fXK*^d?Lvs^g(i{ru^Id}YO zk}?ba7CbpY^za@tyUq?1{PKsJ&%eo%XU$%}MaZOM{?rG4B9SSy_GTSwwKnD^n8B3I zShhpl`G_@uP-6I{D(&uid>ZSt8t>9OqR%PO?y%nJ^^!yFM#kNpO^8KP_rhtVe-s1J zM9rXa%>@;acZWbJUVA*Gia!tWNrwKdfz`CjB=*B@g&Hv~E^@^o;-w+7Cz zrZ{$fGl!MKP6Zc1e!mZqwwra5QoVy97g@O%MEMfJp0Q7?z}U{rYx|Z8?J^sKpYTHI zb;QQ~A~x(9mFL8YlBkh820Xt9;YH{Wgdno>9hv2^pst!(WQuV94&AKKp0lakSF{jY zL7I$9A2gv&^eb&mX&!m9Nn4mL2eDu@fAU+1dyk)_z7PgYt7{vZA^j!Oe*%H@X@zl8A z(lW+fN%?3>;&H9fcV;WA_c z$W_vI)yQD{w#;?u0K}VeGHS9;0as|sPg`KLPh&TFOvQQW-{u*XY~G`y^PVD|z>J-p zyMQv=jh-yX6Aei`ff5Ze%oXoA%#GmQ{-jB$W3(n7NL}Qm>teCE|Ka`!MddI1B1f@Z z!NG~^pya?zY$H7)NEYByd)%Qz;Bn@gakCFd|1SMn{idzekrqdo@sLVur3hFJT%R=O z32|aiN5(((t+o>fbZ|QEsh$F^t8+K?oYOLLf(!qYiH7eEcb%E~$tglT8Qzjd16895<^!Hea~7al$Mt&I|YVg*`d=1TX(ygqLGQn5!47iTk>=gFz1 zYJLtiRtvDK6rSZ)B4;$f zWi6}raKWJaap?#=jbC#W9%$b0^m^oOIkSd8?&W}}!L7$+c#{Z;`&t0^^Ocm;^mK({ zdDz@G9FT_mD!P}>*`3>pwc@PJa0TDVJktA>!T|@cF45-&%*S5Fg<%E3!X{g#t}WcT zzEF0%@ISMvIX$$czqOQp=fbUbo3KB>{(ywi0RaH11b_ey%mJhp5a%)Jz@hh6xL(JX zy7SyO#c5&`;~M#|kkT@&&18(TlPwu40PgxhZTF=_O!bZwRw(J~uAe`pAcax{w~y~B?b;SIiD_?74P&j((Q z<$DRwd;?ZlsurRyXQ;HU-t+;!&kY`iz(XbN%{=~}MYrx|&Rr3=+3&6`@4o*0MQof* zaj)fN{Gp2e@<5+Yp?67$LpWgra}yh|2-#%*8A8kpfM7e@(MjE*A|0)LGu<+@HM$pf z(A=twx_{`_`U;VvB?p2D!`=Zze(I(z*#iYycz|S15^7_X*WRg;n(o}I7FWKdu$Q_& zt&HvV7#Jegf+7D2KV~?fm6+iG57=oKQCsod2sdAroE@rO16f1i-xV>)*jj?wot$`z z0YapzI#obP9QMU=H*cS!eUq$J=>8^4Gmhw}>&EI8WI~kOE9XE(!>2j39C)Q>AnHHP z4cl^GUg1V0_y#lVoPyA7$XDFuQkARd^eQUXEr_2jucuce(cJ z&;|Gb^Jv9rEUbOub%+xUEOMo^>UN`Rn6 zM~&8N1tsWq6JIT{W>0;{;Ujd&0QR*dRRfX)4}kfOC`bU3M+IsNfX2yk=d#0Ni((e{ zN6WJAf9Ap4$o7mS{gsg~WYk!s2n=^obl=-CT0u05x*Gd-dSThY7`poeV! zB3izP1gNiYa+YL4{ZASbDt-#a51bKnMwzP_{TGFhQuCGpXLFu6+{Yy7igskz`jP?p zy|=f?3tn(8pZDq%K&jEXB+PfH;Z8lLzsYd5I&%Pog@`WxAM0yTzyy#J13a=zK!=;d zdQ*t%>l(QH(6h4sA_BGLN*|#|dn+B$Wbx>aU3Bf7c7q_n^X4!tVxv87wr|3m3FhK8 zzb_o#WSu+=zZ+T6zY@^7+F{q-u6?#Hp+M|6X@JAZfW!DuUV#~2ReKK`Vmn|157=iq zibg8)yzR0Jln=1(L^A&=1<-;Rma;FFZcRs83xr!&7Ms${T)OMeKN90#O>^f5ZkI)? z9;sw7h)j~{9Zk?TBa9Q9!ugzL9#y`%*PQz(VZO%CBpmfkN=B<2&ea0TS2gk)hJmW6 zBE=%eQ-Jwh0CQ6U(6kEAwM&Yv@b}%bjyC+aeq?BV*7eyREl@m80ovA1H)sHuBBRq4 zq-p7zUkV!PDT(QzMv>t17840^^UR6fIosZ32(+$n#vcfL@P61y{SK#6m$3MP{xutp z&Y}pg%L7g?OGcnFu=B;JQe!?eaaoesf*=aK!=Vzr>hgC{Lo<0BzRcVCsXwO)i-a*C zGuE8&H*rKy8C+58*L%{TJ#DZsUq61B&9gHvW$cy8+okrt6u9M^?nu}=4iZ@RONt>e z!`6oaVIY+d5R;t&mM7-Hqtr50Ld@!IQ?w(5|gNAj+WRjDl; zqS__qkL;$`$%*qK>A!X5%-COZh`r7j(zMGa4De;z^7Xv;`H^p)EAzh1J>$qY&@c;o z_sTJd&J!7@C7a`37Bfp#f#8n1H-bHJhw)G3+pqT57Xg;feK*2o-jQp6;`Zc(!EuyC znp)%|;6|0?7rs@Njk2LFYa*%lfU|maJl(^uKhdD;;H+pxq|#PSds4J%Bg*%ERHRcR zw4D(Qs-~;VSUz5rBOPptUF+g6{&l@H7bUR!g2jH4lQh&p9smU+&Xbkd>GTPfC5j48 z0I+&&GLZn};UMT#|A?!WS}iB^JvaBy6GyUaL=%>FK4GMN*I`Uri4NF8|Al+JZMCv? zSH~XviNFhcN5Gc7Fc~*=qbA4|bJoGB8sIS*Yc%>s`c6|Gc8x+I74M*8m;dCLV-JiL zS_?VNVOEJ1Ic5#Z;_$r&gh+U7Ka<2vlNOUD);bMlrs0K1{*%BA6et0HM1YWNAJ9SD zZ`cjSd^L9G7Qaq}!KXJfsU31QG7qX-i`$y%zlz%h5skQNOz2cduQyP|-H}iY60{{1 zGpN3j!zQ`-LUn~UUjL0A-1}9N)hK~uXyEc`o`ZRmho(>T@#-Q3u)uXXJ^*KxdMefD z`%@-L#re-4)spF=xL}ncA8H@vL{<$P4C)OuYrGMKwJ$A~1njJYnijWiYa0%eSo2~N zd&;S|T)?Q4hL)P7;2nPi#iwB&$UN6zF6O!Q^jdeFd29UEH|vf&e1Y#QyR&%==Q^qU z&_mt3j|427nELrX`V|S3e>FhT@!K^Rc;*KfYDnFo^w+ip7SDi6-d#t-eP@I=0suhs z3_I9xm+jaxD+CytX5y0F=6L-(4;WHekbiP};Ih+GvaDlLvIK ztaFq_UrI0>J$l@8M6OoXHmd<)e%yPe6nG{;1C!kIO=s&BV_$yp<{43HVq&7co|Rg- zDS(7ypSFHHB0{#%_+gV@MTH$*~4*K}zv}GvD4LAokvRVG7omX;wXu)ZunnYyZx6^j7*e%Se z;H%X2pUd^MCm^8lq6#G)GLLDHgy;~BX(oxCO%OI_2yyIMRA{7v?*|_7PTdKUEyCg! zf|`CJgyEm4QRjtP-`+WK$>4NSWoO`@aSL-nN#2h+9@wz`#TBtG&Rb~r0UjA0+WfRw znlxZ_fq)D}UOp!u@pD;o+Vk>l2i3K*SachsY-x0@8QT0yP9C z@TD9)86=*}y1cL@se=lwq7XvYqxMyJQv7jGqW^e~;T?jYJYokpwR?3ux%m9=jUyN|uI|uM+fhkzg_nOc2f^WjyVehg%ra$FV#0e;*;CzAvxK~qOhVwVaOvgIY}%n7 zPR3i|fPmsz%(tz2sKx+K(|e`&gpj6-J^*`P@oxqtpuk^{N(_j|b^ud{#9Z>5qp6E@ zi?e>YbhnI=r)1t}roFxN75#?p# z>@k-QPE%{Z-#G-py+>-@|1o=Q4&R|+rJCTGfclbwS8rs&5EwRKXcbri5gETCY1|4S zKTfwOpogN`5hl^IN;tIV&C!2hb3)Ot4|53X=H~(T8OhHN)ox@#Y#2HDFTe}>pA-Nc zUy=VOL;NqTcg<(`e?I*uh1(;jB^4bsL}K!|A|^E@)m@=e)7_(~Lw7nIvhi_RkQoVJ zN*GKWG$#0Op!tUUpX9$7!pSPRr=W_4g!ZMA3|OohQpx@~6#wysNoo8GcB@Q> z!tMX72A(0UFu>kd0{FX?!MGFjFu!#8?ck4h5+6}zSv%3=|6izGklvr^27NCxpsS)J zam&+~BHtE;)#H`O&JO(XN~Qcw_`|%p;Hy~>DE@}@)kC*hp%3BB98H=Wp5KVrrKDx$i8UGugoOrez zE?17b?K-1l6GGC00&8~mYtHfh&tf8Z7!kM<)IlA>Lax<^0{Q80?AU*9-!YdsOgQ_q zk6GJlFTbb$Pdon=TWVQhj+#KBrUMGX2}J1ugiL`7!NsO5B1wA~es;oonfSW~zJf4HO;L@(hWj%=*5AyQ5VmDnvDwv{NA z(1|@4a9+(|A)uDMjk}5)MWr>*{UeMpvVdlpu%V)=NbbH*sPo$>ew$^=G}$|8xg*)k zmHI0OhkR;-t=@;7@16aat(i9pZf|+Z^x;`Dm$Qgkc(b`CkXtBb6bG>5Nc7m4=vC}d z2q3nOH~f=Q8iHeVcQ<6PL<#zako2EGIzJ5O;96Q)FwDvlJ~{oLBRr0H*bU>PB7CVt}j>Q)nOB*B{@vZs(ax0Lqh#zN=i*1%KAJE><~0EB^nc8yWm!V0*+2cxQV8mC zjy)Fh=Z$rFP$HiQo32(j0`-+rP2MFY7iZ=P@U~PUxEd5P2|`qI6Td(i`oHM-0we4+ zwEn%z^xM0v4EH~!OyLkXYr zK_Yx<%ZVnrs{cEs=N2-!L}wa2?XtDQBlnj$_(ah|@?D3V<{Dc^WvT3IzXVB;L`hYvrN8Hw%@FZhmt>Uhz67 zztnh>A29Y1kqmWA$+%*b>tpr0>Zj2)wV*I>Rp(X?3&--06J5N{Gk^l(e1$LCB}t9; z#=LbEccd%`)C5sZCrWuVzFmPAPx+aaA{l=S4sAdCxNVmy90cg|viMi=7qz+=+bgUb zj%8+W97}llX}8H1(1wU1%lrQo_~YL8rnm)W7($GYzo$70vEfS+BCxQLubr-#%>W-5*+ zR@ow@Px?adix8XAGVZjRPHV zzH1SV{FYid{g}4?{d2n$0+9E2FzTdmkGuPxvAm}IKZX6j;P&nUuBk@MYaE0aV|rnD zkAnfut?8$mCIb5178*z$5<`)lCGsy_~&i$dl%9t z5Z7C<73(QB%F`yK{&ZSi?(;6b+KG$lh?-0lKKwC`pe5bUxk@aOgbnrvkvg%dL+sI`hX?-M}diK%tgN3KNSgoDmNL02v3>#Z=qfdvO6n}4n8ul5EE&0y;!+HsP?t&G^%)7qZaQlFI=dQF+qplb^Tb8cEiA!t=^@Xmb zSEp-dx1)VaMvld4(-^!Yo;Vjggi~#997>H?s~=QUMiTQ*EiT{8$;pwNzk%XdH_NR+ z9DXxCBP~*Tv1f8E%z3vlZ;mz%4q4rRVKIur^D-<>r2;W)jEV;eqW8~~!6)Ygbl7Hb zIYiV4Y}y*$tRBl)f8~_!IBtZ+w;X+Coo zk5sgYRXi?)Fq+(|YZBs%E|$Sw$6bD``oh-U#5a#7f?5tT9yT|4k9Rlh4@D&oF2?#s!NS8G=be6cEh@N<_`S|9-pb)&Lfu`)^Y|Zm5Nok#*I7;9g zWNvJLrQz1ZJ#33Fy=<6C26+ZpkRL42R%ndFofPwhB0eKI8Yu2{k%5-?l^Ab5f(ZH( zgGea(nav7W^Sp?k(xps80VJU;x%Vr_*`Pa|Hbw9&oF1&qV3$)0Kuv)SXh@Pp0MQ=4 zH#|qDg*&YruGIoTMyf6v$Z(CixmECi*?+vYLO}w7C?ST|Y=92GqPys!O<`b?V)whP z`QlYpMH@uwr#~NyhntMAf@$jJ3IP9Wn%f=B36EDQW#)|eaeu@o@JQ@zP_^P85DQ^% zPq!_w$e1+ULcD#}ASk7IlvNE>xN8H(^4Lx8J{I!azp-Np63$__a&N63$6Q!RXh%{# zZql#frXbAtGAaMI(Uu0Q;?82%3k{USU-E!GA(YzPFR`Cu_1Cli3<%0WCC-(2s4Ae(4ZAXqT?Nf));yAzkFFpkZ z^yZr;-*?PM1-HLkx&Qa?NZ@;-*8yRr9Uq=q5^pB-sGc61us5EI?1(Hxj<5`R&R@+o z2fG+^F@94{k)a>g7q|O(V@Qh=O`DhTGTC3qmVxt2)dAK*OL>{<%zcoX> zLB!S1xXk_BAIf=k16Uohv@Dp@?+g3_vg+tdrEzvP@FpoV#mMNvhwlj@NHq&z z0Nkeh0Y#Pi_ecdejyi^+fVbQW0rI%>P0;^x{|nGen73gH@FDFQ*lfC+BZ%Bgs!~Cm z4Wr`e?2ydSq=FtLc~3pR*urw210U0wQ||w*`JV)t7VO3OWzsoh#7tv9jqfSto5zaZ znUo56D-ibV7z{FrR|4w;fdkT~Y5z|}Y6$Y5=)6Wj2L7oj3UXdR2iRmDN1Qb^%>VpQ z5}L?e0d7^2lh2HP8;-p0ARfFo5xu|FM*5jWvvFLLmv&(ao$TFT;%zy3b*+B3;*?dq zU-2lBnhG!DwM|*i%FB`Q9M_93-t>4CDBhW;lOU#dUgSq4Hnl2&!COxfpU`e0T3s}l zH#zDSF16hBY9GdjKAqZt_z;PZ~A`#EY6?MeCT_GiY_eLxkIx)<1E&X@7(B7e?)JY zo{{xkLoT0EL2;{e7MmX$uSVE4IcDvLGuXLb7}`26Vd&G?BpIxplm$83$=tjQl4u$% zD)b#DoDJ{|7}~=*`&2`4=V$$Kq}^+Pxi6Cinyh;Zz!v)qkouX5aOih@j{+jwx69dt z$AWGfkU2*HH!7z>-=fD>`0ZNw44r`n^>nr!P>sEr#y_^O_!Q0Pj@Qc~`j;hdh4Z&u z!W-+HOL3C`>BM@m>G=aNU9HhVa}&Uh*I2E9fxhPw+x*fcfIy)!U*#m%v}8#_{N6q2My zq2WNFd!#Z$m`m0{)`$SWQ3jA!YzTW+Jwfei*mVj`5eQLMPGpR6Y2fyQs^)+H!|C=*pK zw&uA(nV0?SHh(m8pDkcaO!{Yk zZ8tc{{Q+=(g9Pvd4u<>mpHsbV&FB4?e*T_+mJpk4;U~kcv9kaC{1+p;XIMCiQoao0 z>bNr2bf099Pq40Nue>4|g@llrcp*WSN}!Ct@@Q8NcH$@&t^PMPfg5qL24~Pc03Kat zYZkRUruEe>Cf#`$nM*)HS^l{~2+0Ni$=sPx+yQl$j{oYt9~x=gO(uam(4FTPNWsb#W-;^b`xI_kTWXg*A-euUeW?9 zYJcHat<8@1PZUd^6_@Ztt;FyYU=A81W#!00`t)+NFEBCY{}6 z46=D$0&YZ}I0{t_w{B6Z**tkz|JiLdIRi-!$Ls(+GJ-QX*w}D*F$$I>tmCcbm`N)S79S9O)?f!S3P1hUzndwv_^1CcV{r$Fk^s{GjK$bR2gMirfD9Cm z^L|Jp`1IiwZ31ke!8++I$5)Azm9@jD)MRNQ54#Gt!R*m~VX>dB;nDmiu1Z_`q8$;Y&kW71i12sgB zKD%f5Coez9KP?^T?U8bG<{{LPB1lv)zc=;Qh!a~e@%UA!dM=?9OC3(ZqAPu!+9m)F zlW6F-5%SfwqV{kW%-=#uB5$FU>d2Tne{JwGcuQxv%HD~Tnc4oF(n)sv+4GN9>_+Qr zVRw+sSlrlmgGmNoB012M%buxNGC#>i0PdF&C<;8W7Sn}D0ja9E~xz2ZsDDbZp@tUlbm6&u6C zD{^=jldIN>XKfCgH|v$8+f~L?%=@cG)rpvcNJnLxSc9Q=Kz`!6uUm4ZXjDfGDME^U zfL=QARvb4acXwD*o9D{10C$Gct96VF%f_G^H_T=bW^Urp6Jf#^?Um)le<<=O@(@hKQ{Oj5OhywyS^*AWlQUJdrmA#mlds> zNJ#kn{CBm3SQHi?C%yzb$6J7>S&Kb>h5zk0(L49}kaiM52(BDzB-YaLhug%mW!Co% zxrS57Bk>(mMU_Y8%WG&I?$%iJ7?X6V*5jm2W4V03o5~dTZJ&%H!Dk zaP_KZzJHN%s`J{4=?g`bOCxk}3h;>X4xH9zB;gbinXOVdT!-4FPKyl>sM(jz0$W9q9|(mZ#(@p8fYu6M`)_~GS&(*@5lG}%rl zDlOQADYt3I*tc-5^C|Oe3#5dO)&kW~R`%%({TXI#sl@dKcnbTA?}!ELS&0nkc#mtL zq1DPW+)C`KSP>@E_+$g{!+*NY0qY69COO}dRS1bq*NYJC zO6}rsd`@^&)4e2WLPAm`?bxZ#*jlrdAwpC$I}pyR|MFN)Ih3R#-H0l0Ol69U+?iy& z`OOpiRc%8}%We{KmFsK5Zq4%iCG+o#O_NAbNa?X`*>73%%8@oi^7BA>hFcdIs^nWP z64lBFyBs=5EYF&i(H*rYfg2<=J92}Jcm@f`3PuoM5y56X4e){p@lGK4P}xkVQ=Z}N zdEI!NGTLNhWa|}Gtw5v_+o1)+>^rE|e(zIg5zPN9G_FsnNi=Qh=+he0z=u6Y-X6JQ z1~pOL@zGZ!-^lrv{*<#6P5P(ZndKO1_z4PhRq5PQAflXvBw7j&U0>JhB(AW(Z~;J_ zHvhnJ8wr>QmO!~tE7r4D<6`A}LPKnP;#3rdy)FI8+FpGuGW*KC;)vzbThP7ap_gQq zp3MS7N=6H<5}0HAUJkTw4-M#|;r! z>|(Xp;=oa5lFmJ{w)=DLX4PC(((5YKB+1EQOT%{?g@iC%*b?o7B|#(&vLo_KW@y%^ z102XLAYb4FE2bb^`6LvJM~`)pE-6_IBd$X?<0FOnk8b5h+&65E4CWTL)soCh$;)_{ zx(WZR1;WZsB^F0+J>^X{R7b1x?4&zt5^BZPd$!kpTJW%aFm*FEc})9p zRh%l3SfEjU^YZyLjB!w~iX-34CKQgR>QTO%{inyIHCS3Ide-aX?eXU~h2&Eb4*qO% zM48BohC_uFA4QG7cju41mS9SZ2&lFE!~W44S0({%Gf{hfN-;~JU6VjQG$aOR#C+t- z{=apa5(Uiyegyx$XJUZ|RMJhp=^p5CGl_3pisySWYjf?9pIVFHd9V6`k@v?@zvIp> zr5WPMg?DRo$;@nmv*7PXPewjPW4N+`f_8yXvi~y9AyY@(99yn}cYdce?00>fB}zB@6FC_4A;~kY z(B*qRyOpt@U9gTKFMBw<(^)PRe(T?=+*fs`3ztb(gPb!}bZH`$2>5x%+(WD3=3X&BY~E#bA4kIcHJiK8x0#AV|r z)gLKBhpV@vzDOeea-#Dk$!5Fz-2ADtD+=zo8zvqJwuUSHBJOrboyOBl>e)(y6U{ej zZU^yOn=xH(5^;1lkx^oH8D>yvR8~h9)U^Q0Hp9bo6Q0_dH8j&mpDMdAHRU0N(h@?u ze*HV~ssHQU*xtZV#In3;Xcv^UU#N7bLb;;wO-1!6{|wGj;{Rp?u5sl}8yM7Pp?p zmVjK$w@6QiYjx_TPy~mC$QlVXUINf3U7iTp^19!~yjJbomShwpEJrG8q?UM(oW6t4 z8{8DnBtJn`J&SJ&?5O+KTmiX5bp?)hWQr>qD-)2&%|{J9-YuBr<`*{sG-FG6w+Wuo zSZnbGpq}otLa9{a_Vr0D$c#ii0B;`tf2Kbi1>OC3D~#+vW;!eIHxpUU6#6|$ajm`> zB|N*xcFdx2_Oz1~+(dnY$xkd zepNB5aj>*o*fk|DM&q58s6AX!j#J9+mR1$*bZ|R0F|}2M@3H!vef(Lttk{><5c0A& zZp!$pXHxM!d zzQt#a6Oegy=&4{Xk$Ztk(GQDj1E*0{FjrEknh(niII`byZrdh z*bgpA)zVDKF$Q;e$=Gsy>t~YFPj2=VYI=qAMKn8twC3bfB~s660@kqQTV$B&2l9!o z$>b%}A~USFt|j7XUZnd0g9d9;{J1S0svfGH^=!7ma)l6_ATk0p?+|Yad(S34X0?FK zM>b}wU(Flo1PXQCeuV}g4wteGp7`cDqpP=<#PJdU-S*4=XCX~oA+3${q?2xM0S?V7q7T_O^iyFClj~n zLu+2^+`lxZkT{Z9(rKdbW5MFeyZc8YcSd{Aj7ZK`*|%zVUOU9{JKzmSdb1csYJqVK zNOEZEa(=`%+TS`D&xmr;Hj63lvNI3-M1Jf3YzN=-q!NXS`ymXLy8c*|WdlKqEnu1| zBf_{Kc&9MG7Z>q}5$`Ff@!|X9Ji>jtH)yWhXt9c}P5M~pSZ4`l@?Vj?qR!~n9fet` z%0@k9-#Yh@ouQW9v-p8Od?e`tUm= zVI?FF?-FCYGhhsVT9j^-vgPNhMI{jC6!#!zau*f&F5vt1w>tm)X54&-8%5Cibb6RY z#IV#np3Y4u7VEpaYBMJle~L!DB+!gu&C_G%KOcO20X>g&yR#_?0Jj8Y9oqp|rPw?v z_-1^~VwxCr4RGi-SS&wP^$hJp#6=jZ`OO#r7m^?|vRsP+;z&Jp&x%Jft-y}skq^bM z^3C=!>jVsaE80_kIEzRUPju0QC{B!^-=JV#|J*AC|4e%Pi92rscK<|u6#JAMCB!H` z+^g^VUd}|vKuC(g3~O)a3HLi;RZyMcAVi?#PTb<=$DpQgfv0;}65AHl&uJG)L{9O@ zd)Ab<>yhn-2iBv~l^9o^xpWFt@Uwv_

IWHInzvw&a$;vh=B*8;=Pu;p67U?IAXB zrsCb!AK|Gr!3F(TEI*!3=#>O1iN(&ayv_No-)M(NI^FQq{@M&17PP~Shz#@|gy6l+ z@+Jj1QU$~ zmr%NxAjuaC{^`U|bOW*za+79z{+g3EdL?rmRXA#0`D_(UAw>YO{Zez=RUR@6I7!#@ z$mZPJIGK3~j2fKDf0RWm^tthS|M)d+ez@Q88*m_4AJgC>6jc7&B&bFG&37vy?QF(l zK`cpH;}toQrVn@WSs>)ogZ4G~$JtV&YPcjeehG51k&j-+2XC2eKXis(CV;DYpeY!) z^J0(Cz@kX!Pgwo}4&s+G1K?T3!g|ntix72v>zx{ZJ9I41+>Axz&i*WZd?i=>4|}53 zek?f785DU-g90M41aQ+r^vK~}BlLjKn=n?(C0Z*9)EP3%0BkM5l?Nc|b^rk=MzA6P zM(fX1!Y~tz^!J1aV*tP_uR52zcjui9hS5npQZV|%nhl_)g(p`1(n+|V(-aWb>H*1p zuPb-uzhx2EF@jKMq4^)Bq&@X2qd$)AM?j%QFACNMQt1FTnKKoFS>gG}^SO$e1^x%} z9@%l4$()&&Q^stI7c;j>jq_PVFA^eWSC#ccUWbmW88EAmw4e9sdQf+&FT1E@hS}KC zFz>+HQYYnzsvih3O|_p1j{bbKC^XoOJoWad^@&?BmmEalTPHw|D95do=?%X1GtQGs z*1rE(`$5Z8Vz@0md^149R)R8XzC<&cF-6DTjapzM>(rq^b|v^!fn!>WRdJ~KAQDQnO_xeCl|PfmT&TOpYH|=KRQq4 z33>>LFQcvcO7i(=?MXeY^gxN$gZ$m#bR1&GWlA#F*F(Ym&Qtzk_jh~_2yZXBHxz{( z6_(+=;JMEoS*fUOejg{qvt5w#E*=)4@*<1+6!}lIB+hM>4^oknC18VXsr6#e3GnwcE|UdiX@!=FMN&V%<4vfqJ-`cJ_71yQKUnaYzP&cBtQ@1Z66i~~ zcvH}SAsqfLRL>Boo$nHaKR=9EGB^tnd-*1-P08Ru_UJ674Wu~yzz)pqT9p&&3En|4 z4J}9ewC^8eJgtL$8mgpz<@B0Qr%P~7C0a4>(++^G*d%c|z)K81Uv;cb5{w&-Ecnzr zuE<@}*Znz8c=K@nsq*VL64oSP1)Rg2$+u#Vs_y>1XR8tBB}tR6qnmk!L)sy>iF)~; zk|beqXV2u!BWrSA#9#2g&f)>A)a|t~VdFwOvnriooiWq#jZB&k)x^HC|qeTQJ7 z?N?@ry+9pmYDxBJ>?P`;8&D->S!$BKaUr}Ki@9*~D+zvSQZ(n4l38oTdF+V<82kC~ zkO#K}u>d2^abO$&V%aBGhMMV%5Ho?*`i!z!YJ1VPKx$6OkfZpA`(okN6u-}dn}Tn! zpl{oK4wbo6bW+c2@Xs&K)91v~f1AV- z$S9P=T}PExG18P%H8vk0M^K|BA z;uyZTp0;?s&mh50i5~VzB+oD)uk6Y7DqY0#@i4=a^-4oVJcq1_3Y}aR8$l_b>{Va? zM)keV6=7J}1TpB1VULv7$b{3`j}{ptDt=RIdV+iY_5sv&JUBzOx~S~2#Ili?UPkTf z^ws2ZIPJ}J_&b`)_eMV#BRBMmBtd^|>h5ySewmL8s1Q8O+Y;VpR_$f=hwo|6@A~MM znnK$&aGE1~tUjuC!PT$CPYc6}Vg}r7dxm54;F=BGi zx|%FX&s&&kMGcpGKidj!hS*sbiLFNxCYIDZZk0u{y;&SzwP#jAdzuK4}a zW~C$NNBzKl7%{aQdNmS#jmAgsu0M=cz4+-2>$dI9GjG%$ev)l-Yq3N=x)!*7pM`8 z&Qi4+|C}K>VD{nt?}u4cXAwnu%z7^kD$nuVxTo5&C@cL0HReox_ZfjaISsqp?k$aN zAs}_^wC-gMwTr3B`slXPSb6?n2be@U`m(sX;MkZXy_+t&?v&$o&T}J36CIR(G^+eQ zdpHRopItJ-K~mQlf*7X(ff3ITew&7Z8<{8RLSPjmU&5zgSV;iLLxcgyY7s6^0MxVf zZ~YYiF6jdhyJrYEDyLMg5}P)3o%*^Lql&IwClDTLc}*{$P)oZ|((G3+30plTDwc?( z|BvO0Bw(d{P8kw^AI1F8Xrz|W$@|8aA+0khIipVsPjB2Jpp@|*g}*YIi(^K0cZtnX z_9I`1KX!R!4ZamMZ<&36+3k(Ff9ctiKL)rIGb}!yuJHL<|1M1d2oCS{YU!Olnqi&Y zoNYQ|>>ySmT>~@c2>j%EZLHs6lw#BO8@24^?mB6gU%WWeH9Gn?Tyyo7-e|r{Y739m zallBn&w+hA=@LclPi|Fj(TRJy>$gQAgP{gru0T9M_;}VDyQN;ZkWdjDRQ!ly+;|oy z6rQ48K^R(#%8AKEu%FKZ1y!(lN?LlFLfYB{nsCr@zKjZdCh*~5?qP)5vRXla$Y9+w zY0G1NHk*f7L^pUMj%Wzre8`;8F^OIhk!Oz$P$=n+jeCasPewUfyK74r#qKaYk{uM* z>&booco)BtakZlE(T?pQg^;RX2}%=A_bPw*)-S;w>$Y(}u#R ziRAMr-U_jw*;aRp1kdx$w{@%63r)qfnw*!rQi>8Fmi=dC@Ee zz=|f1kpN5({+CMw7>rsh%06cVoY7R4`or+{$esYk0s4IaH3~xj;4F;Hm;`oC-+w!6 z08=UZ>z(5891yplgG}I=9$5srO|n-oUuB&Q4;n$4 z$(4|-IpuPKqZZp22l0r2HjYAyl$xjC{_tmVck*bVjNsX2b{zK?%qHKOejMn4_^yY-Rf7aP1`Vy}i4Us~sTWEMha|zp)H~f$BMW@_(!7rtD8}o57fSlPizeOg zG<$d8!f*~p z!*s2=^R3=sq?|s=s5dw3k-W$l4v@WE`^E9e{oKiOt(sewoKJomit5_0eiw7G+P6Mrs&kTuNqHVFz)=t`O=zv zx#Ux6zs+RNL?+gDAWOX(G;(fbZqgGduu{YrwyKHDi*r`Tj3t!7TaN=k67rIX0U(2w z6d(CrOMsaWkRc4bYA?fUzJy6p$^OBD4jTf51k4>U!Z>&<9`KRS)ldxmr7X5iJ{<5e z?6aW5W=g-sLr4gBD!%WgwWn<`Ob*d%HIOa{U<5y*d5aMBkCPM^Xzj@v@2eH8CBUkl z;Bu30MRp?m5c&~tF_vm8N@{2jg@EO{4<%-V0JI>9Oos4RRH3Rj5!rmKdc@GNLTasV zl*Z^1o$rU`PXYzkgwk9^H@=n@Plcv6_tCo~c-i|uvlH)hG@118Kz@;apUDv9OqRXU za*a;;k;w(Vz);^-9uj3SB)*1G&rIIFw6M;@AjEM_Q#=1P6#?eJ?k!e;kYh((wW!^d zFP2kGp)^)f=vK|TE76l$iHd5`{3zxmo}EEv_Dd|1w&x0=!f`EClxMT|B3y$q^C3gW_dV|9z7@Je15o>U%SOhgc z^jhYc6IW|q$`UO>pO4qRez0`A0 z;?cq^fAW8^Pdwg>Bw~?#6?>@%GW)Q?pE1aQ5^U9mlv&|KG&1n`5qQ!l+vkE(bM*+pm7$yg^gbS;`2JkH2Zb006w3%Hel5#lI%b%W$>7^MGkV9em(j6 z9_40Z&QsQS*{X@4I|2WA1Ct*$Cg}A;=s#1-{)Y~MPZ`FYPZ6xCgU(14_QD)vT z$^RDy`S)MO!BL1EtOmhEyE@~O>t*7nzw zO9xUCbOar<9G?Y7_g&+Bw3#Jh^g+IkZTsdv_t~NdeUI3#Uk|e8C~V|ph~cF@^fFz@ zdyj_2UxaQnBt>7Dvhcm2h3>)+Z(@|0C_hcCjqqd zCDRNRl<$z@Mk>8>N@Y6X=bIT%mYvO`{o$iM*P|oL(xAPjpy8v%zo`~HeNlk~DQS=5 z4A%iT?IPjSG+#zJ>mZhzz8F&dH`d=SOmtAV_wT@eTy87?Z)aK~e_4oHU2;Pi5y(w^ z|L;9~m9@s#L1RH4g3Tf?FMWP?!TlPwgXz1(&pBjRm8_FDf8S8Y+5T`mC``I_W#j5W z?8*Mg+Jp8q(s9~VDHB|Y;|%6y59X&;<_sq8i#=QbswBPW>XgvYsj1YJLfMx}koNdr z09OL1+hIR1CZcs5M)5#msWPYrxNmK{zGlpzVigfQw}AmbmD~y-(#Hp~ zWE+K~sh2k-wKBULtfIamVJ8n}#kTJGL%F~sgvlQH3?Anv!oBStR(6q_0FGDh0T77E z8l=CjS#WW|Isd|g`C2BXqOa%c_<5dkD!^`Q1}aLd=ykmwFx&gl@%BwugYAP0^u69w zS)P)BQECRB)ZJ54){n+ z1IR@p*;x(7=+z?QvR76F8W4Q*xY+*(FsaKWjL=pNm~#Z}Q-O8L18Fyq8kZ{FdgH$+ zFfedSDJywjjN)rUiP#Hef*nj`_6*G&D`f?QfhcxMeHPXO)&ag&pVJtG$- zg6YGQNT$J64T~S#&*|zZaBd6Bc?@p;lAyJ*xV?LQh1Lzr6MQY(3pk%Ky$ueZl{`2@ z%iI&K>+Sk|XGUEw=d7@xQ%(j#vVrQ9BK;e)Up#wq5`c=O+aA}}dIBKE?S2@LwDLAA z&-T3WFOYBM>Dbhsyeyp&@3_j%y9w-aV^kyQ?zYieunpZPf1c3AJeMgZ2~ZgER5$h? zcD*O>=AgH$csLy*o&vx{LMKDQiuhc$at>fW`oTcLngDp%BDvViC-g*s9&nRAHwLY{ z`LkA&ODeM9oiMva0wfqcK;HfpRz{&{**Jh?DIG-mjhSE;?PYLKhsDE13`gc+^5DF@ z$m#=%d&t>Lm|hv4a+m!rzU=Wov^@6N#^P*=k;k^9z5q5B;(rL_$vL^e$(~4ia~weW z)_sN5?xm1fw@jPjfNTz6!2y>{^J)lRo93ix5W_8`8~CA!KH^_$);JZW3rXb6Kjuk- zc~sAnmW8xgE+;2q_6ajhMB(GiFpTUtK+Jx|s)x-PHWLhE=Au%keUU7@GeS!uTm7Rw zZ?c#lmj~2o*|C78m%7O*;`lA7owqLN4Z< zWEyv4edMd+)HlBg)r{=Hyx?=}U`l2rgzYoslQ>Wgw zB^dyr)bi^>agmMETX`Q}*_SYT%MUHP$JgDeaWF4u88s^i;W?-FgW}s2x5*gLe+U9DiZuWR*qOf z2WE7-+1H)hHT0jJS=ywp*OO`^M7;wjS2n(UD9}6SNTG*)=9O;c>#`C8V8X2$C_Krw49Cp26el@>O0>RkLZ`F zr$WUz`QH_}f4tORK%o9?i))o8#NbZV=*YID>K(pYY`AWk>zG+{Lsl>B&ok?QrK`)U2qm4wLwS2R37@H)4s-V|e z@s#~s3~NUW(e?BGhPav&>kMjOxk>y6`Gu(lx)+ctlNEp=0!)MS6JowfNeltvY61`= z0IETn3&~1@;A?9cu=ztzNdmQ#&^%@KH%FAs!-b#^?q7UjWl|7Cr7|i*ru&NIFU~`+ z{tMtsBBre2;NeLEI9UiSn2Um7NFC$-FC56oD6tIoS3mVnJZu+`udPGj$LcL=WY?fK zPC4Bs6p~yZ!8#AxaSr55BhE~99b^122>G?l0G%Sqypz#F1?4zB0z(Ob2N0l+QvEM^ z7HVcrBC7T;EdrerU`13NaLe8kYCzey+FPE$o(omqyF<{ot#6-cd8*?q*U|Dbd8~5B zXKtpE4xS|P<{^W#eygySYxZJ+9;Z>!-MwE9kI8e+CTPacH(qcji4;qJ@6$=jt^!}k zY&;2?x4T&>{EWA}nnQ0*yEg11u3!P}TBcSiTqHm4&RM(uzQO%`wKY#aO{FtBOrhcl z-hlE~JFbBiRRLh{1V`*Ch_S%^=JS5QzIWcKpVZDYEGSNwH?;^D{P6&NMi89Q0bU@G zc+Y7Nn)E?EI+>XvAqo#zX(5_W{Xui2R8yDtlJ5cl(GSw4W+7i30$D6i*lO4ykP0an zNr(XCN|f4k!>HQG-DmYtjn+7`Fe&xy99V60_}B> z2Hr~?3Q^n!s!Gph)kdv}w_3C66h2^a_)UJ1o6dz?(ZxaXx2AjLcnRH?^0L2oi9l+uzzkOdkUf=E2H=VXA zbpn26%7L~J1UTaqvd3|w-=UHZXobUkOG02lF;Iu$fFt5bDJ_MgyGBoWsEJQr#P zWvH?K7u4M56m?O_f%sMM$C!`(zB?=~0%--3-|FDeB$^OKDoL^{3~qo1Lk4F$@^r2d zA7KhfHwZy;Aw~&Y+ll+FoIo4L%NzT3U^&WRWC{UW`(;Ad>9v;G*xyA9EH^ObyYgMo^Wem?~u?_i7n z6nJ-qRY@J{ZW$6hQuyA{m=f*lOdk4y3=g@Z?yAg;Zy#9Gx=LV89VG$i)-whicDafG zd0*6|DyQCc%Z5P57UDjL{2>(D!FH2G#T@e=T0vXufWnRt2}lw2B>{3BiWQ~|!a@$b zDui_%<`1b$Pt)!3O##?70)P?5YFamTwJsf&k)Q_mPk_@GDM8&F1_!+Wfu>3#w3lh# zC)L{)`ZQ>6;plte<1SmKH>>dQk?M74Nk~v))nDx9IpnX0qmQTD7oFvxZ*2I%`W=f* z9Doa?!vWdK&AQ({kqkM#0ExWZKF4EplZ=gXqZi8P$w8Zhr zS{TT#gB55@t9T}~yw$^g={%T+Stws`*)21CA#i_xx%A$8yPCaXszCGnzJHMCxie(> zZRUd;%BF?l_1L z6T}0UAwxocSV)dmWQvhf6Yn)H$%B_fj({kf&Yjg(5rA-5=4{O4FZtPV%;U6Ve!`LcjJaBX|ga@ zDcL0XR8|hJFWe>1zYp+7PKw$grct{Xykt33lqV12wX35UdX|knT|~$#97+qI+JP_@ zaw%1hrHKkYhZ$f#`Ec2ZO|V5BSodX}wIv_?A%e~$2Q)KOlIZkys-j;g5krnye39NOL$J}K&9r%B?}#utEoUTRrwD|qLCJ=mKVWU z#m5%P3`j;3pvKlA!LA{p5K;(-NPtbKX#e_3(5M`khmrhYH~3|a5g{nlG9;oeff_nU zfnvgVRnN#iRl^fDcS0CpK>ZGV!P7YCNL59<=Y$UpPdUBKZdz8VO~g>|Bk!*T4VtSQ zY>n)?bSVW&6cZzyVc@uMn2Y1m*q?JFlufb!d1y|N|`5C?cF}7 z-37mNPHbO|)k|vS10Quejf&JWOEdaU%np?sg--8t#VW=x*&Z$-eZLOug;ENuKc;WzwF<{SokR!vaQH+7QXHU5SZQ~7y$bsM$$uN&V%4CUY5-M zq9**>xeLg!v1-wiE9HsYrlWxMwKCv)HoA&rxZC2lYk)RXu z=3G3v)P;vjJWT;BLFpXYZR-NE0`jYQEP=K8;ndf!#AYOkx0#9JM+69S4JEbJ9L~k` z=MJ=LH`uT=&137{*M7$gH2{9f)~?s(pLs>yeP++*6bRDS;Q?;M3kaG;8qg~OnY4cw z6#1YGHWLinelk`Fbpii{^enqi-Ht#oMfHr7~3Zb>L+WL&)@5M{(#1ezZzAN+x$)O+}h)x-o_zS zUy5HPT5c@O)X6?SxY?^BBjVe8y5c}N_xZVdoAKwQF|(H!I8{o(RW}8r)&t6Y0IOlL z+;}$Bhy$q8@cdF#Xs9ws)cT~=DV0C^u~W&Af6D56yLEgc;@rKbBHLApANE|tPk6E~uzth-O-#sz5AiN+{-JPA-E81u_(eVWxxbuD zU_e?mlB0{NORAUiS{P~ityzd{>_@7f#Gn_w{nLWHHk;qtvbA(a1U?K252~=Nwq+Ft z_~x7C&?qi1ePMXc6%=L0I?YfPI4?=DWYASc5zA>Jd*SKfxyMNbWTSqr65m$?P9fS3 z{M-KU7`RjtBCsx~1OP&y(u4pCM&1nqrHPYv=ll8_+0J}_jLtuQmFEKe{~zF12iTjc z$9&CAdAE`bRY;pTCRa@Qyt-v{>3geju&9J+{Pk$RjHt@{15K5M;?H$9FRdY$mg_%g z@iI$$Tf*{CU1Z@)*^y_AL-=wCE(|w))o({O ztIDyteePyjdio>wqj|+2d4CrJoVv7+7+25iS}|4Ca{RJMc?t2PkgyO4SJPRCrmCSK z-yiP+QKCQbGb}Ci(dTZhC7rRq-$gvcLgVh?1G((|yX;XeVd(PVM^=`S!VVge{dPgL zT~fEUV9-7VQt^(YNiR~B6;SL?D8dRYI!$f-a`ny9&%NlpH{T54sh~HSyPiEhEI#pz zhX!N8-{~K_>#yJ_H~bIm9gaK@#$2p)^~d5VESeZUV07}+eia5a7^i3{YE$$`e;&A-nXoKV)|`_n4t!Y+?(+B zOxjfd+*9y?SDp$`1rw+~Vbbmq;LOUf_`y3_1d4a2v?z){tnnsr0pSR77ZN-NI(9nY zk~qOy6~N4t?!xY|NRxZC=0Rpwc(Od8(-NKM51sO=4wEq${fj=3pj82UXdwD%K(;kn zA+6*ON6h;;EW7J^k^-V-tgh`%OgCG;u;$yi?-%Z~#jH{0jbTVtS@Pb-QJwF zUFGG;++M%^ILaXNinIRXeibvZZ5oWW0f29)SY*eU1)R32wr&{3SDZMzE`>Ad&hksb ztDX;oYAL`CJ^l+Y{lC@)r=TF^S3q`;;AH;|?$7ck{B)jW7AhprM`lj2uufILxG(gh zo!Pe0Faa`<07NP_0^5@Yg??Y}b%%qE|CzWmLSizoA=SMZ_oTUbFB8K4FZ?y|a01CL zTVP896d$-Di|r~$w2oQ;`8_6hlZZ&5=+z8uOTjVGvxakpUrBQT zyhx85@B;Zx1V5Z1$dL}f4{xLa^Gg2Elc8AjAwb8{!a_b_Kt=)y5~>CvRu_E>$fPuA z1qi_Ey;uNB_czJ?#}D(+YDnkDxNL+S9Z{YY6f&c58Losr(n3{!B6{rD{)h5-mIFBP zqNd4z9L3w3Bo`cvcGIjkwyBHJT6=06*cOpKS$my^mW0N1uGNIgg;Gv&+Ng|MQ$~L_ z+)fJXm}g}Pp>@)GoAxs9k}BQymd;-}(MzlKvGGLCQ_Zu5k0V!QnxvHS$RCuuB`+!! zNxowo6rc1^IrBmYzFV33o&eZrKGM8?!J(NqD+-El1rc9SI{8W(qzJ#ZPtqYti<^7H zSPDvS092iVoJ3SoZ^`dC9o2Xph{|^>z!wvMbwQpu)`t92i-H=JV+N4lz#?jI^$_SE zFyU2nW7~tFLv&FV(9T3s9Dq$A9v`=lkvI-6dF%nc2rM)XtXa`OLFBQ3$_*_xen(pU zh4vOAl$tV;3>*{cud|r}m$%Ps%=1MWR9vPub~@`(H;u`Aj~$K-t2*4RUzML*ElscJ zf0Ss)@Nl=lhj4p$!nO~C&gzXR$UplsslB9EUxfRzZD1y$|LQV@*@ehwlu6|PsUl=a z0H`Zx^gYb4PquB9CeDheYq+V5otViJq&cgOMf|A(btiu~vD=yON~iVex=X|{S09Az zsVH^f_OwE#^S)V?M!=%rsDLcT>!ElsNNadyER6(J1_BiM(^CXMmi2Bc=$4rp{;xP> zM7s`x?M3IcvN{}r{O|z|elxPvpj;ru7FS#Rr?Mnv!LIk}HwQ!7P*dtO#I(kMm8(SM zxGt@#?$>WSW1?(`qBV(bXe7TRG?8!~Kf&3!gl7~!CGM@5QgBLH2+BjumLas>%Y>Si z`30~`Iv=AS5^yEq z;qH0U8M4UdnSJq%Xby_Ozsmg2Ltyw6LChFHLkcS&?4RgkA9-h|`t#iCe?=OpEeMRD z^ZpeFYa%iA8(X-yW46g5jou$}i%mDoK;=t22GuEi_H^WY9SimcVUF#4-mXY&9x@Ay`K!!j}$b}5gH9OB19HC7^-(6KgwjCKaF zqJM6om24V5`7qvPzq5?#XdU29L4Gg1NZC-Up<8Gxv%2<_knfxG@%qr6eFeQsDG0yQ zb+w07Pk5KFI5|DQK9}4nv1kc8+@Drko}7qAv-8VHN!8lld*YV$t@bJ0^`#iad)ct( zFIE)*N~F+v{LcGkG^GOxI!m`hn}5?(3zgoR+A?}o{oCg*iL>?Lyq90#Z?)+Nwct;e z$@x=zaPi~f5}s%YM0Vad9>Sm<4w(dctmgY=?+b5q}O2kEK|WA4<8e{z}g}ECPUKE3nb{ z@lXtR`RdjCX94mq%ud!!mGD5pli`l;AGTIU-qJW24cMX@mJWLb5IWlCGiq3{E%x16 z^qLmtVXOg`|3R7%uhy#dH8&qe{nn8000A`<9v}!I4sY?N2;(mfRcOuMs7ut7;Plxt z_0=BT&S)f8kNSR_FuOJ|yURbOxNH zmn!`X%+q;D>+}i7bb#I81@L9?G2|AdC(IJp@A!$EEOon1qvFpPB_lmOUWg^R~Yn8P<9JKt9?EGnSTtK(D))gBSaE&?f9h&8sL zE|_ajXfJvekV+lOze)ka;_l>Ff$@1*=o#M{B#^=T!Hy=hl;T!0v!bhjgzt|NQ?2Wt-;dnH8=Puow@!Z{M zowiu&#JQ!+sew32`$`M_A-3are6<)gm`jAcz-gOdhl4yd^ zx2`24fojhif;-N4FLPavz(yf6 z#m8r!%NVwvv>yKHe#HM4Pq_#v#V7l>=vctyIRAYo=55^}VpDJT+PwC^$^-@KP5vE& zHHFk{RM1U}sjGz~ecwPL!d{(R3K#FIZ4*8KO8e`yOAz^R2F0EU#%- zk~Lq%{vxO^t4Mf@xEpT;eFJHf0shQ0BpQ?01PP3?!aXFz*R7#r1QNIhAW>C|*4P_40)Qc%#|i-|UZENrkt_(jKLN<0gY8+7 zkJf$T3xgD$XZb!XZcH)qM)Z3C+n{8lfIuBlo9vJYw9n>oc&Cr7OrQKyb0-O3dKRH6ODXT+=Pq{3;e^bAK^u15I}cfMbgz!ml^~9@0|~9=eS%F);o`)ahsUm1UoK(H@#B{nV zZt3Kfwr$0>^Myaw_PL*;U91bIjy|D6F9qLv1m}Xi2nsdvpyScW0}UjrP@(-*<;ENw zxbj${b_SS5S$|?Jsi3nyCzG;@3l<`s4yU55orn(y9|C-QG{skp zx)c9$Zd~LsI9}z1MZuRL)iTwVq-7Sy)TH?5_TmUL zCQ0LJX!HHfcX|S{tsTrcu_hUj|G#=gJ7wsx_SiNKuByDO=5 z)JcB5YTIS?;Bi3lwY4%EDIMhArby}glQlkJ?$rsp+SZj0R(wMK8qcp;m?oD_5Z{d{ zJY;9AEIVtw5!#&o95!oeE>%o2Z!|lq3jU}sV9mSk`FX4W!1=&VOY7NEE&)3p*N3b2 zIxqtR=alSz0m{M{yvY-?gj11YenhT4Dqjl;G#A%urM2Xb&p_6b>1-6vA`gdld!PBnk28TI1q2We{iH;?E3FWn zyHB7?V4?}L8fkA4VxUJ~N#300>B|Rq()N^_5t%w>F$c8kGZ~71@ga0{BQ$=FLwh_2 z7}^_$2ah8^EnL6<{4~|m&+ty-`n~MYL)C}^#zokz@`8oimV?*w3&y#i^BuqA$mFn5 zq8lPJg-xxA)-G3V+pxOYfJn(T1; z5{D#doC~xEc_PSgAG8KrXb3q~)ib3k1JI?4LMg1?sn3rJV2<{+{j!%vhk9SS=viW% zBaECY@3=cKq&M;Nb}2uN00S+{tuO4Cfz=I{+SNTdI>?^=xy3ayu30RIpNIlTR)ONN}5^sabt* z&oKknNN9ai2<>1Gv!OFCibSS=ib#!HDWc6IGme}2Sco)O=)U|uS}cl9V?6&_O$tYY zbC+C@Wuv=OeZ<+xQOk#n@&fv<;{wImn?lrP9X_q-Mair*{eRzoMo1T z-cy%w9bN~Bg3Y-6l9fHP35_R1DEaM9t66(LKSVvAw3bk+4f}JLbo=VzWa8nEfSVu2 z@Lua`+tG!=i-T!R|J2OWF(#^hCpXUkS9=F{<>H?(=t_uo?tszaLW>A|G=??$*kKIK&Pj{(P z3afM$!f6haKWtn)c{!I>8Q8&`pXUJ(Aa}aAgC?HLs^G6|sGHv}HNz5?-bP0V1l^kJ z%)OVvbi>$6eUTYAb^}6(EGh8E&tss(&}7L_l>*K){|W!|)Er|sHdCHl5Ln4N5%K>3tM=&! literal 0 HcmV?d00001 diff --git a/evals/browser/cases/regression/cooperation-video/run.mjs b/evals/browser/cases/regression/cooperation-video/run.mjs new file mode 100644 index 00000000..e39973f6 --- /dev/null +++ b/evals/browser/cases/regression/cooperation-video/run.mjs @@ -0,0 +1,476 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { once } from "node:events"; +import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { withChrome } from "../snapshot-coordinates/chrome.mjs"; + +const directory = dirname(fileURLToPath(import.meta.url)); +const cli = resolve(process.env.BSK_COOP_CLI ?? "target/debug/bsk"); +const executable = process.env.BSK_COOP_CHROME; +assert(executable, "Set BSK_COOP_CHROME to an isolated Chromium executable"); +const port = Number(process.env.BSK_COOP_PORT ?? 52931); +assert(Number.isInteger(port) && port > 0 && port !== 52800); +const extensionPath = resolve(process.env.BSK_COOP_EXTENSION ?? "apps/extension/dist/chrome-mv3"); +assert( + (await readFile(join(extensionPath, "background.js"), "utf8")).includes(`ws://127.0.0.1:${port}`), + "Build the test extension with BSK_DAEMON_WS_URL matching BSK_COOP_PORT", +); +const artifacts = resolve( + process.env.BSK_COOP_ARTIFACTS ?? "evals/browser/results/cooperation-video", +); +await mkdir(artifacts, { recursive: true }); +const runtime = await mkdtemp(join(tmpdir(), "bsk-video-live-")); +const env = { ...process.env, BSK_HOME: runtime, BSK_AUTO_START: "0", BSK_AUTO_UPDATE: "off" }; +const exec = promisify(execFile); +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const commands = []; +const results = []; +let daemon; +let daemonOutput = ""; +let browserVersion; +const bsk = async (args) => { + const start = performance.now(); + let output, stderr, exitCode; + try { + const result = await exec(cli, [...args, "--json"], { + env, + timeout: 35000, + maxBuffer: 2 ** 22, + }); + output = result.stdout; + stderr = result.stderr; + exitCode = 0; + } catch (error) { + output = error.stdout || ""; + stderr = error.stderr || error.message; + exitCode = error.code; + } + let result; + try { + result = JSON.parse(output); + } catch { + result = { stdout: output, stderr }; + } + const entry = { + args, + exitCode, + durationMs: Math.round(performance.now() - start), + result, + stderr, + }; + commands.push(entry); + await appendFile(join(artifacts, "commands.jsonl"), JSON.stringify(entry) + "\n"); + return entry; +}; +const ok = async (args) => { + const reply = await bsk(args); + assert.equal(reply.exitCode, 0, JSON.stringify(reply)); + return reply.result; +}; +const poll = async (read, label, timeout = 15000) => { + const until = performance.now() + timeout; + while (performance.now() < until) { + const value = await read(); + if (value) return value; + await delay(100); + } + throw new Error("Timed out: " + label); +}; +const html = await readFile(join(directory, "fixture.html")); +const clip = await readFile(join(directory, "preview.webm")); +const server = createServer((req, res) => { + if (req.url === "/preview.webm") { + res.setHeader("Content-Type", "video/webm"); + res.setHeader("Content-Length", clip.length); + res.end(clip); + return; + } + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.end(html); +}); +const cases = [ + { name: "video_pixels_only", variant: "pixels", target: false, dependencies: false, allow: true }, + { + name: "timer_text_untargeted_space", + variant: "text", + target: false, + dependencies: false, + allow: false, + }, + { + name: "timer_text_targeted_scoped_space", + variant: "text", + target: true, + dependencies: true, + allow: true, + }, + { + name: "timer_text_dependencies_without_target", + variant: "text", + target: false, + dependencies: true, + allow: true, + }, + { + name: "progress_range_targeted_scoped_space", + variant: "range", + target: true, + dependencies: true, + allow: true, + }, + { + name: "cosmetic_class_untargeted_space", + variant: "cosmetic", + target: false, + dependencies: false, + allow: true, + }, + { + name: "continuous_progress_sync_and_space", + variant: "range", + target: true, + dependencies: true, + allow: true, + continuous: true, + }, + { + name: "progress_dependencies_without_target", + variant: "range", + target: false, + dependencies: true, + allow: true, + }, + { + name: "business_range_still_blocked", + variant: "business-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "submitted_timeline_still_blocked", + variant: "submitted-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "form_timeline_still_blocked", + variant: "form-range", + target: true, + dependencies: true, + allow: false, + }, + { + name: "progress_and_business_field_still_blocked", + variant: "combined", + target: true, + dependencies: true, + allow: false, + }, + { + name: "programmatic_focus_still_blocked", + variant: "focus", + target: false, + dependencies: true, + allow: false, + focus: true, + }, + { + name: "business_field_targeted_scoped_space", + variant: "field", + target: true, + dependencies: true, + allow: false, + }, + { + name: "learned_price_current_target_blocked", + variant: "learned-price", + learn: true, + target: true, + targetSelector: "#buy", + allow: false, + }, + { + name: "learned_price_declared_dependency_blocked", + variant: "learned-price", + learn: true, + target: true, + dependencies: ["#buy"], + allow: false, + }, + { + name: "learned_price_bare_key_blocked", + variant: "learned-price", + learn: true, + target: false, + allow: false, + }, + { + name: "learned_price_untargeted_dependency_blocked", + variant: "learned-price", + learn: true, + target: false, + dependencies: ["#buy"], + allow: false, + }, + { + name: "refused_context_requires_sync", + variant: "learned-price", + learn: true, + retryRefused: true, + target: true, + allow: true, + }, + { + name: "explicit_timeline_dependency_still_blocked", + variant: "range", + target: true, + dependencies: ["#progress"], + allow: false, + }, +]; +try { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const base = `http://127.0.0.1:${server.address().port}`; + daemon = spawn(cli, ["daemon", "start", "--foreground", "--port", String(port)], { + env, + stdio: ["ignore", "pipe", "pipe"], + }); + daemon.stdout.on("data", (chunk) => { + daemonOutput += chunk; + }); + daemon.stderr.on("data", (chunk) => { + daemonOutput += chunk; + }); + await poll(async () => (await bsk(["status"])).exitCode === 0, "isolated daemon"); + await withChrome( + { + executable, + extensionPath, + extensionViaCdp: process.env.BSK_COOP_LOAD_CDP !== "0", + deviceScale: 1, + zoom: 1, + headless: true, + softwareRendering: true, + startupTimeout: 30000, + extraArgs: ["--autoplay-policy=no-user-gesture-required"], + }, + async (send) => { + browserVersion = await send("Browser.getVersion"); + const browsers = await poll(async () => { + const reply = await ok(["browsers"]); + return reply.length && reply; + }, "test extension connection"); + assert.equal(browsers.length, 1, "Only the isolated test browser should use this daemon"); + for (const scenario of cases) { + console.log("CASE " + scenario.name); + const { session_id: session } = await ok([ + "session", + "start", + "--browser", + browsers[0].instance_id, + ]); + const scope = ["--session", session]; + let oracle; + try { + const url = `${base}/?variant=${scenario.variant}&case=${scenario.name}&interval=${scenario.continuous ? 10 : 80}`; + await ok(["navigate", url, ...scope]); + const target = (await send("Target.getTargets")).targetInfos.find( + (target) => target.url === url, + ); + assert(target, "Test tab must exist"); + ({ sessionId: oracle } = await send("Target.attachToTarget", { + targetId: target.targetId, + flatten: true, + })); + await send("Runtime.enable", {}, oracle); + await send("Page.bringToFront", {}, oracle); + const read = async (expression) => { + const reply = await send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + oracle, + ); + assert(!reply.exceptionDetails, JSON.stringify(reply.exceptionDetails)); + return reply.result.value; + }; + let playback; + try { + await poll(async () => { + playback = await read( + "({ready:window.fixtureReady,error:window.fixtureError,report:window.report?.()})", + ); + assert(!playback.error, playback.error); + return playback.ready && playback.report.decodedFrames >= 3; + }, "actual video playback"); + } catch (error) { + await writeFile( + join(artifacts, scenario.name + ".playback-error.json"), + JSON.stringify(playback, null, 2), + ); + throw error; + } + let learning; + if (scenario.learn) { + const initial = await ok(["cooperate", "sync", ...scope]); + // Keep node identities stable and tick exactly once per observation. + await read("window.tickNoise(); true"); + const learnArgs = [ + "press", + "Space", + ...scope, + "--context", + initial.context_id, + "--selector", + "#pause", + ]; + const refused = await bsk(learnArgs); + assert.notEqual(refused.exitCode, 0); + assert.match(refused.result.message, /changed by itself/); + assert.equal(refused.result.data?.effect_state, "none"); + learning = { refused }; + if (scenario.retryRefused) { + learning.retry = await bsk(learnArgs); + assert.notEqual(learning.retry.exitCode, 0); + assert.equal( + learning.retry.result.message, + "This action requires the context returned by the latest cooperate sync", + ); + assert.equal(learning.retry.result.data?.effect_state, "none"); + } + const report = await read("window.report()"); + assert.equal(report.keydowns, 0); + assert.equal(report.paused, false); + } + if (scenario.continuous) { + await read("window.startNoise(); true"); + await delay(100); + } + let sync = await ok(["cooperate", "sync", ...scope]); + if (scenario.continuous) { + for (let i = 0; i < 2; i++) { + await delay(100); + sync = await ok(["cooperate", "sync", ...scope]); + } + } + const before = await read("window.report()"); + assert.equal(before.paused, false); + // Start page-origin updates after the observation, before the proposed input. + await read(scenario.learn ? "window.tickNoise(); true" : "window.startNoise(); true"); + await delay(320); + const atAction = await read("window.report()"); + assert( + atAction.decodedFrames > before.decodedFrames, + "Video must actually decode frames", + ); + assert(atAction.ticks > before.ticks, "Page updates must run before the input"); + if (atAction.progress !== undefined) assert.notEqual(atAction.progress, before.progress); + if (scenario.variant === "text") assert.notEqual(atAction.clock, before.clock); + if (["field", "combined"].includes(scenario.variant)) + assert.notEqual(atAction.order, before.order); + if (scenario.learn) assert.notEqual(atAction.price, before.price); + if (scenario.focus) assert.notEqual(atAction.focused, before.focused); + const wait = await ok(["cooperate", "wait", ...scope, "--timeout-ms", "0"]); + const action = await bsk([ + "press", + "Space", + ...scope, + "--context", + sync.context_id, + ...(scenario.target ? ["--selector", scenario.targetSelector ?? "#pause"] : []), + ...(scenario.dependencies + ? (Array.isArray(scenario.dependencies) ? scenario.dependencies : ["#pause"]).flatMap( + (dependency) => ["--depends-on", dependency], + ) + : []), + ]); + const after = await read("window.report()"); + const evidence = { + ...scenario, + session, + learning, + before, + atAction, + coordinationReady: wait.ready, + action, + after, + }; + results.push(evidence); + await writeFile( + join(artifacts, scenario.name + ".json"), + JSON.stringify(evidence, null, 2), + ); + if (scenario.name === "timer_text_untargeted_space") { + const screenshot = await send("Page.captureScreenshot", { format: "png" }, oracle); + await writeFile( + join(artifacts, "blocked-preview.png"), + Buffer.from(screenshot.data, "base64"), + ); + } + assert.equal(wait.ready, true, "No person should be holding control"); + if (scenario.allow) { + assert.equal(action.exitCode, 0, JSON.stringify(action)); + assert.equal(after.keydowns, 1); + assert.equal(after.paused, true); + } else { + assert.notEqual(action.exitCode, 0, JSON.stringify(action)); + assert.equal( + action.result.message, + scenario.focus + ? "The keyboard focus changed; synchronize before pressing a key" + : "The action's observed content changed; synchronize and replan", + ); + assert.equal(action.result.data?.effect_state, "none"); + assert.equal(after.keydowns, 0); + assert.equal(after.pauseCalls, 0); + assert.equal(after.paused, false); + } + console.log( + JSON.stringify({ + name: scenario.name, + exit: action.exitCode, + keydowns: after.keydowns, + paused: after.paused, + elapsedMs: action.durationMs, + }), + ); + } finally { + if (oracle) await send("Target.detachFromTarget", { sessionId: oracle }).catch(() => {}); + await ok(["session", "stop", session]); + } + } + }, + ); + await writeFile( + join(artifacts, "result.json"), + JSON.stringify({ passed: true, browserVersion, cases: results }, null, 2), + ); + console.log(`PASS ${results.length} real-browser cases; artifacts=${artifacts}`); +} catch (error) { + await writeFile( + join(artifacts, "result.json"), + JSON.stringify( + { passed: false, browserVersion, error: String(error.stack || error), cases: results }, + null, + 2, + ), + ); + throw error; +} finally { + await writeFile(join(artifacts, "daemon.log"), daemonOutput); + if (daemon && daemon.exitCode === null && daemon.signalCode === null) { + const exited = once(daemon, "exit"); + daemon.kill("SIGTERM"); + const force = setTimeout(() => daemon.kill("SIGKILL"), 5000); + await exited; + clearTimeout(force); + } + await new Promise((resolve) => server.close(resolve)); + await rm(runtime, { recursive: true, force: true }); +} diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index ad368b99..2f5f1609 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -34,7 +34,8 @@ set is uncertain, omit it and retain whole-page checking. Text that changes by i sync and action (a clock, a counter) is reported once with the before/after text. After you sync and replan, further changes to that text no longer block actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or -selector, such as a bare key press, always check the whole page exactly. Added, removed or +selector never learn incidental text; a bare key press checks the whole page exactly unless +explicit dependencies narrow its decision regions. Added, removed or re-rendered elements and text without its own accessible node (such as `aria-hidden` content) keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an @@ -46,6 +47,14 @@ A regular coordination result is not a failed browser attempt. Use wait then syn the next action from the actual result. Use request-help only for an actual human requirement such as login or a captcha. Explicit user stop always ends the task. +On continuously updating pages, declare the observed decision regions instead of repeatedly +retrying a whole-page action. `press` honors dependencies even without a focus target; specify +a target when the key must be delivered there. Untargeted keys reject a changed focus. A +clearly labelled media timeline outside all declared regions may advance without revoking the +action when it is unambiguously associated with a video/audio element. Submitted inputs, form members, +ordinary sliders, and other business values remain protected; declaring the timeline itself +as a dependency protects its value too. Unknown media controls retain the ordinary checks. + ## Scripts and file transfers Scripts, upload/download, active hover observation, full-page screenshots, device emulation From d3b260d0647d629a3b715f216cd1e4e0e7d3e25f Mon Sep 17 00:00:00 2001 From: Zhang GH <5351707+iuyo5678@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:49:53 +0800 Subject: [PATCH 2/3] fix(cooperation): protect implicit focus and media source changes --- .../__tests__/cooperation-capture.test.ts | 60 +++++- .../src/content/cooperation-capture.ts | 60 +++++- .../src/cooperation/controller.test.ts | 197 ++++++++++++++++++ apps/extension/src/cooperation/controller.ts | 73 ++++++- .../cooperation/observation-evidence.test.ts | 31 +++ .../src/cooperation/observation-evidence.ts | 26 ++- apps/extension/src/cooperation/types.ts | 13 ++ .../vom/__tests__/semantic-graph.test.ts | 31 +++ .../src/tools/vom/semantic-graph/resolve.ts | 3 + .../bsk-cli/skill/references/cooperation.md | 13 +- .../regression/cooperation-video/README.md | 22 +- .../regression/cooperation-video/fixture.html | 42 +++- .../regression/cooperation-video/run.mjs | 187 ++++++++++++++++- .../skill/references/cooperation.md | 13 +- packages/vom/src/types.ts | 2 + 15 files changed, 722 insertions(+), 51 deletions(-) diff --git a/apps/extension/src/content/__tests__/cooperation-capture.test.ts b/apps/extension/src/content/__tests__/cooperation-capture.test.ts index abb981ad..8f0b3e21 100644 --- a/apps/extension/src/content/__tests__/cooperation-capture.test.ts +++ b/apps/extension/src/content/__tests__/cooperation-capture.test.ts @@ -20,9 +20,10 @@ afterEach(() => { vi.restoreAllMocks(); }); -function collector() { +function collector(shadowRoot?: (element: HTMLElement) => ShadowRoot | null) { let receive!: (message: unknown, sender: unknown, reply: (result: unknown) => void) => void; vi.stubGlobal("chrome", { + dom: shadowRoot ? { openOrClosedShadowRoot: shadowRoot } : undefined, runtime: { sendMessage: vi.fn(async () => {}), onMessage: { @@ -453,3 +454,60 @@ it("reports programmatic focus changes even without a human event", () => { expect(current.focus).not.toBe(first); expect(current.human).toBe(0); }); + +it("records media sources independently of playback time, including runtime-only source changes", () => { + document.body.innerHTML = ''; + const video = document.querySelector("video")!; + const source = video.querySelector("source")!; + const read = collector(); + const first = read()!; + video.currentTime = 1; + expect(read()!.media).toEqual(first.media); + source.src = "two.webm"; + const changed = read()!; + expect(changed.media).not.toEqual(first.media); + expect(changed.revision).toBeGreaterThan(first.revision); + Object.defineProperty(video, "currentSrc", { configurable: true, value: "selected.webm" }); + const selected = read()!; + expect(selected.media![0].currentSrc).toBe("selected.webm"); + expect(selected.revision).toBeGreaterThan(changed.revision); + Object.defineProperty(video, "srcObject", { configurable: true, value: {} }); + const stream = read()!; + expect(stream.media![0].srcObject).toEqual(expect.any(String)); + Object.defineProperty(video, "srcObject", { configurable: true, value: {} }); + expect(read()!.media![0].srcObject).not.toBe(stream.media![0].srcObject); +}); + +it("collects media and deep focus in open shadow roots", () => { + document.body.innerHTML = '

'; + const root = document.querySelector("#host")!.attachShadow({ mode: "open" }); + root.innerHTML = ''; + const read = collector(); + expect(read()!.focusIsDocument).toBe(true); + root.querySelector("button")!.focus(); + const focused = read()!; + expect(focused.focusIsDocument).toBe(false); + expect(focused.media).toHaveLength(1); + root.querySelector("source")!.src = "two.ogg"; + expect(read()!.media).not.toEqual(focused.media); +}); + +it("collects source and recipient changes inside closed shadow roots", () => { + document.body.innerHTML = '
'; + const host = document.querySelector("#host")!; + const root = host.attachShadow({ mode: "closed" }); + root.innerHTML = + ''; + const read = collector((element) => (element === host ? root : null)); + const buttons = root.querySelectorAll("button"); + buttons[0].focus(); + const first = read()!; + expect(host.shadowRoot).toBeNull(); + expect(first.media).toHaveLength(1); + buttons[1].focus(); + root.querySelector("source")!.src = "two.webm"; + const changed = read()!; + expect(changed.focus).not.toBe(first.focus); + expect(changed.media).not.toEqual(first.media); + expect(changed.revision).toBeGreaterThan(first.revision); +}); diff --git a/apps/extension/src/content/cooperation-capture.ts b/apps/extension/src/content/cooperation-capture.ts index 9bf8aa18..98fa1a81 100644 --- a/apps/extension/src/content/cooperation-capture.ts +++ b/apps/extension/src/content/cooperation-capture.ts @@ -43,9 +43,9 @@ export function attachCooperationCapture() { const keys = new Set(); const fields = new Map(); const sampledValues = new WeakMap(); - const ids = new WeakMap(); + const ids = new WeakMap(); let nextId = 0; - const id = (element: Element) => { + const id = (element: object) => { let value = ids.get(element); if (!value) ids.set(element, (value = String(++nextId))); return value; @@ -102,6 +102,20 @@ export function attachCooperationCapture() { }; const observer = new MutationObserver(mutations); const roots = new Set(); + // Page JS cannot see closed roots, but the extension can. Observation also + // traverses these roots, so source and focus guards must cover the same DOM. + const shadowRoot = (element: Element): ShadowRoot | null => + element.shadowRoot ?? + (element instanceof HTMLElement ? (chrome.dom?.openOrClosedShadowRoot(element) ?? null) : null); + const activeElement = () => { + let active = document.activeElement; + while (active) { + const nested = shadowRoot(active)?.activeElement; + if (!nested) break; + active = nested; + } + return active; + }; const controls = function* (root: Document | ShadowRoot): Generator { if (!roots.has(root)) { observer.observe(root, { @@ -115,14 +129,17 @@ export function attachCooperationCapture() { for (const element of root.querySelectorAll("*")) { if (overlay(element)) continue; if (element.matches("input,textarea,select,[contenteditable=true]")) yield element; - if (element.shadowRoot) yield* controls(element.shadowRoot); + const nested = shadowRoot(element); + if (nested) yield* controls(nested); } }; const read = () => { mutations(observer.takeRecords()); - let active = document.activeElement; - while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement; + const active = activeElement(); state.focus = active ? id(active) : undefined; + state.focusIsDocument = + (active === document.body || active === document.documentElement) && + !active?.matches('[tabindex],[role],[contenteditable]:not([contenteditable="false"])'); if ([...roots].some((root) => root instanceof ShadowRoot && !root.host.isConnected)) { observer.disconnect(); roots.clear(); @@ -188,6 +205,36 @@ export function attachCooperationCapture() { }); } for (const element of fields.keys()) if (!element.isConnected) fields.delete(element); + const media = [...roots].flatMap((root) => + [...root.querySelectorAll("video,audio")].map((element) => { + const stream = element.srcObject; + return { + id: id(element), + src: element.getAttribute("src"), + currentSrc: element.currentSrc, + srcObject: stream ? id(stream) : null, + tracks: + typeof MediaStream !== "undefined" && stream instanceof MediaStream + ? stream + .getTracks() + .map((track) => track.id) + .sort() + : [], + sources: [...element.children] + .filter((child): child is HTMLSourceElement => child instanceof HTMLSourceElement) + .map((source) => ({ + id: id(source), + src: source.src, + type: source.type, + media: source.media, + })), + }; + }), + ); + // currentSrc/srcObject may change without a DOM mutation. Keep the source + // evidence fresh even when the page's mutation counter did not advance. + if (state.media && JSON.stringify(state.media) !== JSON.stringify(media)) state.revision++; + state.media = media; state.busy = composing || pointer || keys.size > 0; state.fields = [...fields.values()]; return { ...state }; @@ -257,8 +304,7 @@ export function attachCooperationCapture() { clearTimeout(focusTimer); focusTimer = setTimeout(() => { if (!enabled) return; - let active = document.activeElement; - while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement; + const active = activeElement(); if (state.editing) state.revision++; state.editing = state.editing && editable(active); composing = false; diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 266e4e10..12768c98 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -39,6 +39,7 @@ async function fixture(options: { url?: string; sync?: boolean } = {}) { let now = 10000; let frame: FrameState = { document: "doc-a", + focusIsDocument: true, revision: 0, structure: 0, human: 0, @@ -880,6 +881,60 @@ describe("shared-tab dispatch fences", () => { expect(f.native).not.toHaveBeenCalled(); }); + it.each([ + "focused", + "unobserved", + "ambiguous", + ])("protects %s keyboard recipients independently of declared dependencies", async (focus) => { + live( + page((tick) => [ + [1, null, "main"], + [2, 1, "form"], + [4, 2, "button", { name: "Pause" }], + [6, 1, "form"], + [7, 6, "span", { text: `$${100 + tick}` }], + [8, 6, "button", { name: "Buy", focused: focus !== "unobserved" }], + ...(focus === "ambiguous" + ? [[9, 1, "button", { name: "Other", focused: true }] as Spec] + : []), + ]), + ); + refs(); + const f = await fixture({ sync: false }); + f.change({ focus: "buy", focusIsDocument: false }, false); + const synced = await f.command("sync"); + expect(await run(f, synced.context_id, { key: "Enter", dependsOn: ["e4"] })).toMatchObject({ + message: changed, + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + }); + + it("keeps untargeted scripts strict even when they declare unrelated dependencies", async () => { + live((tick) => scene(`10:00:0${tick}`)); + refs(); + const f = await fixture(); + f.change({ structure: 100 }); + const result = await f.controller.run( + { + id: "script", + method: "tool.evaluate", + params: { + session_id: "shared", + tab_id: 7, + context_id: f.first.context_id, + expression: "buy()", + depends_on: ["e4"], + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Runtime.evaluate", { expression: "buy()" }), + ); + expect(result).toMatchObject({ message: changed, data: { effect_state: "none" } }); + expect(f.native).not.toHaveBeenCalled(); + }); + it("does not let a refused context retry with newly learned tolerance", async () => { live((tick) => scene(`10:00:0${tick}`)); const f = await fixture(); @@ -1026,6 +1081,13 @@ describe("shared-tab dispatch fences", () => { f.first.context_id, async (runner) => { await runner.send(7, "Page.navigate", { url: receipt.final_url }); + // The completed navigation may replace media before the old document + // disappears; that is its effect, not grounds to revoke its receipt. + f.change({ + media: [ + { id: "next", src: "next.webm", currentSrc: "next.webm", srcObject: null, sources: [] }, + ], + }); f.controller.invalidate(7); f.change({ document: "doc-b" }); return receipt; @@ -2455,3 +2517,138 @@ it("refreshes timeline evidence on a delta when that value is an explicit decisi expect(result).toEqual({}); expect(f.native).toHaveBeenCalledOnce(); }); + +it("revokes actions on media source changes even without DOM revision changes", async () => { + const f = await fixture({ sync: false }); + const media = { + id: "video", + src: null, + currentSrc: "one.webm", + srcObject: null, + sources: [{ id: "source", src: "one.webm", type: "video/webm", media: "" }], + }; + f.change({ media: [media] }, false); + const synced = await f.command("sync"); + // A browser source selection can change currentSrc without a DOM mutation. + const rev = f.readFrames.mock.results.at(-1)!.value; + const frames = await rev; + f.change( + { media: [{ ...media, currentSrc: "two.webm" }], revision: frames.get(0)!.revision }, + false, + ); + expect(await f.act(synced.context_id)).toMatchObject({ + message: "The media source changed; synchronize and replan", + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it("requires a fresh sync when a media source changes during observation", async () => { + const f = await fixture({ sync: false }); + f.change( + { + media: [ + { id: "video", src: "one.webm", currentSrc: "one.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + vi.mocked(handleSnapshot).mockImplementationOnce(async () => { + f.change( + { + media: [ + { id: "video", src: "two.webm", currentSrc: "two.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + return { text: "Pause", ref_count: 1, tab_id: 7, truncated: false }; + }); + expect(await f.command("sync")).toMatchObject({ + message: "Page changed while synchronizing; sync again after it settles", + }); +}); + +it("does not revoke native input already started when that input switches the media", async () => { + const f = await fixture(); + expect( + await f.act(f.first.context_id, async (runner) => { + await runner.send(7, "Input.dispatchMouseEvent", { type: "mousePressed" }); + f.change( + { + media: [ + { id: "next", src: "two.webm", currentSrc: "two.webm", srcObject: null, sources: [] }, + ], + }, + false, + ); + await runner.send(7, "Input.dispatchMouseEvent", { type: "mouseReleased" }); + return { ok: true }; + }), + ).toEqual({ ok: true }); +}); + +it.each([ + "focus", + "media", +])("consumes a context refused for %s even if the page restores the old state", async (kind) => { + const f = await fixture(); + f.change( + kind === "focus" + ? { focus: "other" } + : { + media: [ + { + id: "video", + src: "other.webm", + currentSrc: "other.webm", + srcObject: null, + sources: [], + }, + ], + }, + false, + ); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + data: { effect_state: "none" }, + }); + f.change({ focus: undefined, media: [] }, false); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + message: "This action requires the context returned by the latest cooperate sync", + }); + expect(f.native).not.toHaveBeenCalled(); +}); + +it.each([ + "focus", + "media", +])("refreshes semantic evidence when %s changes before a delta sync", async (kind) => { + vi.mocked(handleSnapshot).mockResolvedValue({ + text: "Pause", + ref_count: 1, + tab_id: 7, + truncated: false, + }); + const f = await fixture(); + vi.mocked(handleSnapshot).mockClear(); + f.change( + kind === "focus" + ? { focus: "pause", focusIsDocument: false } + : { + media: [{ id: "video", src: null, currentSrc: "two.webm", srcObject: null, sources: [] }], + }, + false, + ); + expect(await f.command("sync")).toHaveProperty("observation"); + expect(handleSnapshot).toHaveBeenCalledOnce(); +}); + +it("rejects a document that becomes a specific keyboard recipient without changing its identity", async () => { + const f = await fixture(); + f.change({ focusIsDocument: false }, false); + expect(await f.act(f.first.context_id, undefined, "tool.press")).toMatchObject({ + message: "The keyboard focus changed; synchronize before pressing a key", + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); +}); diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 5c48a30c..aed05249 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -14,6 +14,7 @@ import { contentChanges, type ObservationEvidence, observationEvidence, + observedKeyboardTargets, observesTarget, sameObservationEvidence, } from "./observation-evidence"; @@ -153,6 +154,25 @@ function sameFields(a: Frames, b: Frames, ignoreMediaProgress = false): boolean ) ); } +function sameFocus(a: Frames, b: Frames): boolean { + return ( + a.size === b.size && + [...a].every(([id, frame]) => { + const other = b.get(id); + return frame.focus === other?.focus && frame.focusIsDocument === other?.focusIsDocument; + }) + ); +} + +function sameMedia(a: Frames, b: Frames): boolean { + return ( + a.size === b.size && + [...a].every( + ([id, frame]) => JSON.stringify(frame.media ?? []) === JSON.stringify(b.get(id)?.media ?? []), + ) + ); +} + function failure(message: string, effect: "none" | "unknown" = "none"): RpcError { return { code: "permission_denied", @@ -583,7 +603,9 @@ export class CooperationController implements ControlAuthority { baseline && [...before.values()].every((frame) => frame.fieldsComplete !== false) && baseline.refs === session.refStore.revision && - sameFrames(baseline.frames, before, "structure"); + sameFrames(baseline.frames, before, "structure") && + sameFocus(baseline.frames, before) && + sameMedia(baseline.frames, before); // A delta updates the model's field values. Refresh private semantic // evidence too, without replacing refs or repeating page text to the model. if (delta && !sameFields(baseline.frames, before)) @@ -616,6 +638,7 @@ export class CooperationController implements ControlAuthority { state.epoch !== epoch || !sameFrames(before, after, "human") || !sameFields(before, after, true) || + !sameMedia(before, after) || (delta && !sameFrames(before, after, "structure")) || this.humanActive(state) || state.held @@ -693,7 +716,7 @@ export class CooperationController implements ControlAuthority { }), state: { ...this.view(state.tabId), mode: "ready" }, guidance: - "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or selector always check the whole page exactly. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", + "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Keys without a ref or selector can use explicit dependencies, while their actual focused region stays protected; uncertain focus and other untargeted actions keep the exact whole-page check. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", }; } finally { state.working = false; @@ -1026,16 +1049,26 @@ export class CooperationController implements ControlAuthority { !params.ref && !params.selector && context && - [...context.frames].some(([id, frame]) => frame.focus !== state.frames.get(id)?.focus) + !sameFocus(context.frames, state.frames) ) return "The keyboard focus changed; synchronize before pressing a key"; + if ( + !settledNavigation && + !nativeInputStarted && + context && + !sameMedia(context.frames, state.frames) + ) + return "The media source changed; synchronize and replan"; if (!settledNavigation && state.epoch !== epoch) return "The observed page or control state changed; synchronize"; }; const assert = () => { signal.throwIfAborted(); const reason = interruption(); - if (reason) throw new Error((revoked = reason)); + if (reason) { + if (context && state.context === context) state.context = undefined; + throw new Error((revoked = reason)); + } }; const verifyEvidence = async () => { if (!context || !verifiedFrames) throw new Error((revoked = "Synchronize before acting")); @@ -1066,10 +1099,9 @@ export class CooperationController implements ControlAuthority { ); } // Learned text is tolerated only around a resolved target present in both - // observations, whose region can be protected. A target-less key press or - // script acts on whatever is focused, and a selector may resolve outside - // the observation (for example inside aria-hidden content); both keep the - // exact whole-page check unless explicit dependencies narrow the decision. + // observations, whose region can be protected. Untargeted actions never + // learn text. Keys with explicit dependencies additionally protect their + // actual focus; other untargeted actions keep the exact whole-page check. const anchored = actionTarget !== undefined && observesTarget(context.evidence, actionTarget) && @@ -1080,9 +1112,34 @@ export class CooperationController implements ControlAuthority { tolerant && state.volatile?.documents === pages ? state.volatile.keys : undefined; // The target's own semantic region and every declared dependency stay // exact; sameObservationEvidence only relaxes broad layout around a target. + const keyboardRegionMatches = () => { + if (actionTarget) return true; + // Only keyboard input has an implicit recipient. An untargeted script + // still requires the exact whole page, as before scoped-key support. + if (req.method !== "tool.press") + return sameObservationEvidence(context.evidence!, evidence!); + const recipients = observedKeyboardTargets(context.evidence!); + const current = observedKeyboardTargets(evidence!); + const specificFrames = [...context.frames.values()].filter( + (frame) => frame.focusIsDocument !== true, + ); + if (!specificFrames.length && !recipients.length && !current.length) return true; + // A single, stable, observed recipient gets its own exact region. Missing + // AX focus, hidden targets and ambiguous frame/shadow focus stay strict. + if ( + specificFrames.length === 1 && + recipients.length === 1 && + current.length === 1 && + recipients[0].backendNodeId === current[0].backendNodeId && + recipients[0].frameId === current[0].frameId + ) + return sameObservationEvidence(context.evidence!, evidence!, recipients[0]); + return sameObservationEvidence(context.evidence!, evidence!); + }; const matches = (keys?: ReadonlySet) => { if (evidenceTargets) { return ( + keyboardRegionMatches() && (!actionTarget || sameObservationEvidence(context.evidence!, evidence!, actionTarget, keys)) && evidenceTargets.every((dependency) => diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index 0d74ccb3..593d6c03 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -416,3 +416,34 @@ it("never learns media values or relaxes an explicitly observed timeline", () => ).toBe(false); expect(sameObservationEvidence(before, after, target, learned)).toBe(true); }); + +it("allows incidental progress within the pause button's region, but keeps declared containers exact", () => { + const before = { nodes: mediaScene() }; + before.nodes[1].tag = "section"; + before.nodes[1].role = "region"; + before.nodes[5].parentId = 2; + before.nodes[6].parentId = 2; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, target)).toBe(true); + expect(sameObservationEvidence(before, after, { backendNodeId: 7, frameId: "frame" })).toBe( + false, + ); + expect(sameObservationEvidence(before, after, { backendNodeId: 2, frameId: "frame" })).toBe( + false, + ); + expect(sameObservationEvidence(before, after)).toBe(false); + after.nodes[2].value = "new order"; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); + +it("keeps explicitly declared DOM ancestors exact even when an AX parent differs", () => { + const before = { nodes: mediaScene() }; + before.nodes.push(node(8, 1, "section", "Player")); + before.nodes[6].domAncestorIds = [8, 1]; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, { backendNodeId: 8, frameId: "frame" })).toBe( + false, + ); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index a00ab9cf..b50bb4c3 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -64,6 +64,20 @@ export function observesTarget(evidence: ObservationEvidence, target: EvidenceTa return evidence.nodes.some((node) => isTarget(node, target)); } +/** Specific keyboard recipients visible to the browser's accessibility tree. + * Document/iframe focus is not a reliable substitute for the actual recipient. */ +export function observedKeyboardTargets(evidence: ObservationEvidence): EvidenceTarget[] { + return evidence.nodes + .filter( + (node) => + node.focused && + node.backendNodeId !== undefined && + !["body", "html", "iframe", "frame"].includes(node.tag) && + node.role !== "RootWebArea", + ) + .map((node) => ({ backendNodeId: node.backendNodeId!, frameId: node.frameId })); +} + const positioned = (node: VomNode) => ["absolute", "fixed"].includes(node.position); // Form state, value-bearing widgets and modal layers are never page activity. const protectedState = (node: VomNode) => @@ -239,14 +253,18 @@ function signature( learnedText.set(holder, set); } } - // Only progress values outside the protected region may be incidental. + // A semantic region may include both Pause and the timeline. Only a direct + // dependency on the timeline, its player, or a containing element makes its + // value exact; sharing the inferred region with a button does not. // Keep the original nodes intact: media controls cannot become learned text. const media = nodes.filter((node) => node.tag === "video" || node.tag === "audio"); const forms = new Set(nodes.filter((node) => node.tag === "form").map((node) => node.id)); const ignoredProgress = new Map(); + const explicitlyObserved = (node: VomNode) => + !!anchor && (tree.within(node, anchor) || (node.domAncestorIds ?? []).includes(anchor.id)); if (target) for (const node of nodes) { - if (inside(node) || node.value === undefined) continue; + if (explicitlyObserved(node) || node.value === undefined) continue; const owner = playbackProgressMedia( { ...node, @@ -258,7 +276,7 @@ function signature( .filter((item) => item.frameId === node.frameId) .map((item) => ({ id: item.attrs?.id, node: item })), ); - if (owner) ignoredProgress.set(node, owner.node); + if (owner && !explicitlyObserved(owner.node)) ignoredProgress.set(node, owner.node); } // Identity, structure, state and visibility of learned nodes remain checked. const facts = (node: VomNode, relax: boolean) => { @@ -307,7 +325,7 @@ function signature( * whole page. With a target it never applies inside the target's own region, * as described in signature(). Form values, checked states and modal layers * stay exact everywhere except positively identified, non-submitted media - * progress values outside the protected region of a scoped comparison. + * progress values outside the explicitly observed target subtree. */ export function sameObservationEvidence( before: ObservationEvidence, diff --git a/apps/extension/src/cooperation/types.ts b/apps/extension/src/cooperation/types.ts index 9cd0fc2c..9416a75c 100644 --- a/apps/extension/src/cooperation/types.ts +++ b/apps/extension/src/cooperation/types.ts @@ -32,10 +32,23 @@ export interface HumanActivity { context?: string; } +/** Media identity and source selection, excluding ordinary playback progress. */ +export interface MediaState { + id: string; + src: string | null; + currentSrc: string; + srcObject: string | null; + tracks?: string[]; + sources: { id: string; src: string; type: string; media: string }[]; +} + export interface FrameState { document: string; /** Collector-local identity of the deep active element in this document. */ focus?: string; + /** True only for the ordinary document focus, without a specific recipient. */ + focusIsDocument?: boolean; + media?: MediaState[]; revision: number; structure: number; human: number; diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index 85452385..92033e7c 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -954,3 +954,34 @@ describe("semantic VOM graph", () => { expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id); }); }); + +it("retains browser focus on semantic nodes without trusting a DOM focused attribute", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "buy", + backendDOMNodeId: 2, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Buy" }, + properties: [{ name: "focused", value: { value: true } }], + }, + { + nodeId: "pause", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Pause" }, + properties: [{ name: "focused", value: { value: false } }], + }, + ], + [dom(2, null, "button"), dom(3, null, "button", { focused: "true" })], + ), + ], + }); + expect(scene.nodes.find((node) => node.backendNodeId === 2)?.focused).toBe(true); + expect(scene.nodes.find((node) => node.backendNodeId === 3)?.focused).toBeUndefined(); +}); diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 4de56b8a..d4f241c5 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -498,6 +498,9 @@ export function resolveSemanticGraph( ...(name ? { name } : {}), ...(value !== undefined ? { value } : {}), ...(checked !== undefined ? { checked } : {}), + ...(node.ax?.ignored !== true && axProperty(node.ax, "focused") === "true" + ? { focused: true } + : {}), ...(clean(node.dom?.formPlaceholder ?? attrs.placeholder) ? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) } : {}), diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index 184c9dfc..0e4c0005 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -55,11 +55,14 @@ such as login or a captcha. Explicit user stop always ends the task. On continuously updating pages, declare the observed decision regions instead of repeatedly retrying a whole-page action. `press` honors dependencies even without a focus target; specify -a target when the key must be delivered there. Untargeted keys reject a changed focus. A -clearly labelled media timeline outside all declared regions may advance without revoking the -action when it is unambiguously associated with a video/audio element. Submitted inputs, form members, -ordinary sliders, and other business values remain protected; declaring the timeline itself -as a dependency protects its value too. Unknown media controls retain the ordinary checks. +a target when the key must be delivered there. Untargeted keys reject a changed focus and +protect the actual focused region even if the declared dependencies are elsewhere. Unknown +or ambiguous focus keeps whole-page checking; untargeted scripts also keep that strict check. +A clearly labelled media timeline with an unambiguous video/audio association may advance +beside a pause button, including in the same player region. Directly observing the timeline, +its media element or a containing element keeps its value exact. Submitted inputs, form members, +ordinary sliders and other business values remain protected. Media identity, child `` +declarations and the selected source stay exact. Unknown media controls retain ordinary checks. ## Scripts and file transfers diff --git a/evals/browser/cases/regression/cooperation-video/README.md b/evals/browser/cases/regression/cooperation-video/README.md index 80fc5306..61129c40 100644 --- a/evals/browser/cases/regression/cooperation-video/README.md +++ b/evals/browser/cases/regression/cooperation-video/README.md @@ -19,20 +19,30 @@ port 52800. `BSK_COOP_EXTENSION` overrides the built extension directory. Chrome uses `Extensions.loadUnpacked`; use `BSK_COOP_LOAD_CDP=0` with builds that support the load extension command-line flags instead. `BSK_COOP_ARTIFACTS` selects the result directory. -The 20 cases cover continuously advancing progress during sync (including incremental sync), +The 37 cases cover continuously advancing progress during sync (including incremental sync), dependencies without a focus target, ordinary business sliders, submitted timelines, form-owned timelines, simultaneous video and order changes, and a programmatic focus change. They also combine PR #497's learned-text behavior with the media fix: a learned price becomes exact again when it belongs to the current target or an explicit dependency, bare keys remain -strict, a refused context cannot be replayed, and an explicitly observed timeline stays exact. Positive cases require one received -Space event and a paused video. Negative cases require no Space event, no pause call and +strict, a refused context cannot be replayed, and an explicitly observed timeline stays exact. +Review regressions cover price changes around the actual keyboard focus despite unrelated +dependencies, child ``/effective source changes, and a pause button beside its labelled +timeline in one player region. Controls declare the timeline or whole player, focus the timeline, +or change a business field in the same player. Focus changes before a delta sync must refresh +semantic evidence, and a document becoming an explicit keyboard recipient revokes the key. +Closed shadow-root players also allow progress but reject a source change with or without +a simultaneous timeline update. +Positive cases require one received Space event +and a paused video. Negative cases require no key event, no simulated purchase, no pause call and `effect_state: none`. Every created session, daemon and temporary browser profile is cleaned up. A dependency must still cover the action's actual decision. A key without a target or dependencies keeps the exact whole-page check. Targeted actions retain PR #497's bounded -text learning after a refusal and a fresh sync. Only positively -identified, non-submitted media progress outside all declared regions may change incidentally; -ordinary fields/sliders and unknown media controls keep the conservative behavior. +text learning after a refusal and a fresh sync. Implicit keyboard recipients have their own +protected region; uncertain focus falls back to the whole page. Only positively identified, +non-submitted media progress may change incidentally; explicitly declaring the timeline, +its media element or a containing element protects that value. Ordinary fields/sliders and +unknown media controls keep the conservative behavior. The two-second VP8 test clip can be regenerated with FFmpeg: diff --git a/evals/browser/cases/regression/cooperation-video/fixture.html b/evals/browser/cases/regression/cooperation-video/fixture.html index d96a6350..6a08cf21 100644 --- a/evals/browser/cases/regression/cooperation-video/fixture.html +++ b/evals/browser/cases/regression/cooperation-video/fixture.html @@ -24,10 +24,15 @@

Looping preview

const variant = query.get('variant') || 'pixels'; const interval = Number(query.get('interval') || 80); const video = document.querySelector('#preview'); + if (query.get('shadow') === 'closed') { + const host = document.createElement('div'); + video.before(host); + host.attachShadow({mode: 'closed'}).append(video); + } const dynamic = document.querySelector('#dynamic'); const stats = { variant, keydowns: 0, clicks: 0, pauseCalls: 0, ticks: 0, presentedFrames: 0 }; if (variant === 'text') dynamic.innerHTML = 'Playback tick 0'; - if (['range', 'submitted-range', 'form-range', 'combined'].includes(variant)) dynamic.innerHTML = ''; + if (['range', 'submitted-range', 'form-range', 'combined', 'source-range', 'src-range', 'same-region', 'same-region-field'].includes(variant)) dynamic.innerHTML = ''; if (variant === 'business-range') dynamic.innerHTML = ''; if (variant === 'submitted-range') document.querySelector('#progress').name = 'order_amount'; if (variant === 'form-range') { @@ -36,10 +41,27 @@

Looping preview

if (variant === 'combined') dynamic.insertAdjacentHTML('beforeend', ''); if (variant === 'field' || variant === 'focus') dynamic.innerHTML = ''; if (variant === 'cosmetic') dynamic.innerHTML = 'Preview information'; - if (variant === 'learned-price') dynamic.innerHTML = '
Order total 100
'; + if (['learned-price', 'focused-price'].includes(variant)) dynamic.innerHTML = '
Order total 100
'; + stats.purchases = 0; + if (variant === 'focused-price') { + document.querySelector('#buy').addEventListener('click', () => stats.purchases++); + document.querySelector('#buy').focus(); + } + if (variant.startsWith('same-region')) { + const controls = document.querySelector('#controls'); + const region = document.createElement('section'); + region.id = 'player'; + region.setAttribute('aria-label', 'Video player'); + controls.before(region); + const timeline = document.querySelector('#progress'); + region.append(video, document.querySelector('#pause'), timeline.closest('label') || timeline); + controls.remove(); + if (variant === 'same-region-field') region.insertAdjacentHTML('beforeend', ''); + } const pause = () => { stats.pauseCalls++; video.pause(); }; document.querySelector('#pause').addEventListener('click', () => { stats.clicks++; pause(); }); document.addEventListener('keydown', event => { + if (variant === 'focused-price' && event.key === 'Enter') { stats.keydowns++; return; } if (event.code !== 'Space' && event.key !== ' ') return; event.preventDefault(); stats.keydowns++; @@ -50,10 +72,17 @@

Looping preview

stats.ticks++; if (variant === 'text') document.querySelector('#clock').textContent = `Playback tick ${stats.ticks}`; if (document.querySelector('#progress')) document.querySelector('#progress').value = String(stats.ticks % 10000); - if (variant === 'field' || variant === 'combined') document.querySelector('#order').value = String(100 + stats.ticks); + if (['field', 'combined', 'same-region-field'].includes(variant)) document.querySelector('#order').value = String(100 + stats.ticks); + if (variant === 'body-recipient') { document.body.tabIndex = 0; document.body.setAttribute('role', 'button'); } if (variant === 'focus') document.querySelector('#order').focus(); if (variant === 'cosmetic') document.querySelector('#cosmetic').className = stats.ticks % 2 ? 'phase-a' : 'phase-b'; - if (variant === 'learned-price') document.querySelector('#price').firstChild.nodeValue = `Order total ${100 + stats.ticks}`; + if (['learned-price', 'focused-price'].includes(variant)) document.querySelector('#price').firstChild.nodeValue = `Order total ${100 + stats.ticks}`; + if (variant === 'source-range' || variant === 'source-only') { + video.querySelector('source').src = '/preview.webm?version=2'; + video.load(); + void video.play(); + } + if (variant === 'src-range') { video.src = '/preview.webm?version=2'; void video.play(); } }; window.startNoise = () => { if (!window.noiseTimer) window.noiseTimer = setInterval(window.tickNoise, interval); @@ -70,6 +99,8 @@

Looping preview

loop: video.loop, currentTime: video.currentTime, duration: video.duration, + currentSrc: video.currentSrc, + sourceSrc: video.querySelector('source')?.src, focused: document.activeElement?.id || document.activeElement?.tagName, clock: document.querySelector('#clock')?.textContent, progress: document.querySelector('#progress')?.value, @@ -78,7 +109,8 @@

Looping preview

}); (async () => { // A bundled synthetic WebM avoids depending on live MediaRecorder scheduling. - video.src = '/preview.webm'; + if (variant === 'source-range' || variant === 'source-only') video.innerHTML = ''; + else video.src = '/preview.webm'; const frame = () => { stats.presentedFrames++; video.requestVideoFrameCallback(frame); }; video.requestVideoFrameCallback(frame); await video.play(); diff --git a/evals/browser/cases/regression/cooperation-video/run.mjs b/evals/browser/cases/regression/cooperation-video/run.mjs index e39973f6..6fac3c23 100644 --- a/evals/browser/cases/regression/cooperation-video/run.mjs +++ b/evals/browser/cases/regression/cooperation-video/run.mjs @@ -84,7 +84,7 @@ const poll = async (read, label, timeout = 15000) => { const html = await readFile(join(directory, "fixture.html")); const clip = await readFile(join(directory, "preview.webm")); const server = createServer((req, res) => { - if (req.url === "/preview.webm") { + if (req.url.startsWith("/preview.webm")) { res.setHeader("Content-Type", "video/webm"); res.setHeader("Content-Length", clip.length); res.end(clip); @@ -234,6 +234,150 @@ const cases = [ dependencies: ["#progress"], allow: false, }, + { + name: "focused_buy_price_change_unrelated_dependency", + variant: "focused-price", + key: "Enter", + target: false, + dependencies: ["#pause"], + allow: false, + manual: true, + }, + { + name: "focused_buy_price_change_no_dependency", + variant: "focused-price", + key: "Enter", + target: false, + dependencies: false, + allow: false, + manual: true, + }, + { + name: "focused_buy_price_change_explicit_target", + variant: "focused-price", + key: "Enter", + target: true, + targetSelector: "#buy", + dependencies: ["#pause"], + allow: false, + manual: true, + }, + { + name: "child_source_changes_with_progress", + variant: "source-range", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "child_source_changes_without_progress", + variant: "source-only", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "video_src_changes_with_progress", + variant: "src-range", + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "progress_and_pause_in_same_player_region", + variant: "same-region", + target: true, + dependencies: ["#pause"], + allow: true, + }, + { + name: "same_player_business_field_still_blocked", + variant: "same-region-field", + target: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "same_player_declared_timeline_still_blocked", + variant: "same-region", + target: true, + dependencies: ["#progress"], + allow: false, + }, + { + name: "same_player_declared_container_still_blocked", + variant: "same-region", + target: true, + dependencies: ["#player"], + allow: false, + }, + { + name: "focused_pause_in_same_player_allows_progress", + variant: "same-region", + target: false, + focusBefore: "#pause", + dependencies: ["#pause"], + allow: true, + }, + { + name: "focused_timeline_still_exact", + variant: "same-region", + target: false, + focusBefore: "#progress", + dependencies: ["#pause"], + allow: false, + }, + { + name: "focused_pause_after_delta_sync_allows_progress", + variant: "same-region", + target: false, + dependencies: ["#pause"], + allow: true, + focusAfterFirstSync: "#pause", + }, + { + name: "closed_shadow_media_progress_is_incidental", + variant: "range", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: true, + manual: true, + }, + { + name: "closed_shadow_child_source_changes_with_progress", + variant: "source-range", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "closed_shadow_child_source_changes_without_progress", + variant: "source-only", + closedShadow: true, + target: true, + dependencies: ["#pause"], + allow: false, + manual: true, + source: true, + }, + { + name: "body_becomes_specific_recipient_is_blocked", + variant: "body-recipient", + target: false, + dependencies: ["#pause"], + allow: false, + focus: true, + }, ]; try { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); @@ -279,7 +423,7 @@ try { const scope = ["--session", session]; let oracle; try { - const url = `${base}/?variant=${scenario.variant}&case=${scenario.name}&interval=${scenario.continuous ? 10 : 80}`; + const url = `${base}/?variant=${scenario.variant}&case=${scenario.name}&interval=${scenario.continuous ? 10 : 80}&shadow=${scenario.closedShadow ? "closed" : "none"}`; await ok(["navigate", url, ...scope]); const target = (await send("Target.getTargets")).targetInfos.find( (target) => target.url === url, @@ -316,6 +460,10 @@ try { ); throw error; } + if (scenario.focusBefore) + await read( + `document.querySelector(${JSON.stringify(scenario.focusBefore)}).focus(); true`, + ); let learning; if (scenario.learn) { const initial = await ok(["cooperate", "sync", ...scope]); @@ -359,27 +507,43 @@ try { sync = await ok(["cooperate", "sync", ...scope]); } } + if (scenario.focusAfterFirstSync) { + await read( + `document.querySelector(${JSON.stringify(scenario.focusAfterFirstSync)}).focus(); true`, + ); + sync = await ok(["cooperate", "sync", ...scope]); + assert(sync.observation, "A new recipient must refresh semantic focus evidence"); + } const before = await read("window.report()"); assert.equal(before.paused, false); // Start page-origin updates after the observation, before the proposed input. - await read(scenario.learn ? "window.tickNoise(); true" : "window.startNoise(); true"); + await read( + scenario.learn || scenario.manual + ? "window.tickNoise(); true" + : "window.startNoise(); true", + ); await delay(320); const atAction = await read("window.report()"); assert( - atAction.decodedFrames > before.decodedFrames, - "Video must actually decode frames", + scenario.source + ? atAction.presentedFrames > before.presentedFrames + : atAction.decodedFrames > before.decodedFrames, + "Video must actually decode frames, including after source reload", ); + if (scenario.source) assert.notEqual(atAction.currentSrc, before.currentSrc); assert(atAction.ticks > before.ticks, "Page updates must run before the input"); if (atAction.progress !== undefined) assert.notEqual(atAction.progress, before.progress); if (scenario.variant === "text") assert.notEqual(atAction.clock, before.clock); - if (["field", "combined"].includes(scenario.variant)) + if (["field", "combined", "same-region-field"].includes(scenario.variant)) assert.notEqual(atAction.order, before.order); - if (scenario.learn) assert.notEqual(atAction.price, before.price); - if (scenario.focus) assert.notEqual(atAction.focused, before.focused); + if (scenario.learn || scenario.variant === "focused-price") + assert.notEqual(atAction.price, before.price); + if (scenario.focus && scenario.variant !== "body-recipient") + assert.notEqual(atAction.focused, before.focused); const wait = await ok(["cooperate", "wait", ...scope, "--timeout-ms", "0"]); const action = await bsk([ "press", - "Space", + scenario.key ?? "Space", ...scope, "--context", sync.context_id, @@ -414,6 +578,7 @@ try { ); } assert.equal(wait.ready, true, "No person should be holding control"); + assert.equal(after.purchases, 0, "No simulated purchase may execute"); if (scenario.allow) { assert.equal(action.exitCode, 0, JSON.stringify(action)); assert.equal(after.keydowns, 1); @@ -424,7 +589,9 @@ try { action.result.message, scenario.focus ? "The keyboard focus changed; synchronize before pressing a key" - : "The action's observed content changed; synchronize and replan", + : scenario.source + ? "The media source changed; synchronize and replan" + : "The action's observed content changed; synchronize and replan", ); assert.equal(action.result.data?.effect_state, "none"); assert.equal(after.keydowns, 0); diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index 2f5f1609..45c495c2 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -49,11 +49,14 @@ such as login or a captcha. Explicit user stop always ends the task. On continuously updating pages, declare the observed decision regions instead of repeatedly retrying a whole-page action. `press` honors dependencies even without a focus target; specify -a target when the key must be delivered there. Untargeted keys reject a changed focus. A -clearly labelled media timeline outside all declared regions may advance without revoking the -action when it is unambiguously associated with a video/audio element. Submitted inputs, form members, -ordinary sliders, and other business values remain protected; declaring the timeline itself -as a dependency protects its value too. Unknown media controls retain the ordinary checks. +a target when the key must be delivered there. Untargeted keys reject a changed focus and +protect the actual focused region even if the declared dependencies are elsewhere. Unknown +or ambiguous focus keeps whole-page checking; untargeted scripts also keep that strict check. +A clearly labelled media timeline with an unambiguous video/audio association may advance +beside a pause button, including in the same player region. Directly observing the timeline, +its media element or a containing element keeps its value exact. Submitted inputs, form members, +ordinary sliders and other business values remain protected. Media identity, child `` +declarations and the selected source stay exact. Unknown media controls retain ordinary checks. ## Scripts and file transfers diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 2b6855ac..a78e3cf6 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -34,6 +34,8 @@ export interface VomNode { value?: string; /** Checked state is separate from the control's submitted value. */ checked?: boolean | "mixed"; + /** Browser accessibility focus, used to protect implicit keyboard recipients. */ + focused?: boolean; placeholder?: string; inputState?: "empty" | "filled" | "default" | "unknown"; href?: string; // hostname of external link target; omitted for same-origin links From 4cd2772742470f1f5f201c8ebc68a380e3647b3d Mon Sep 17 00:00:00 2001 From: Zhang GH <5351707+iuyo5678@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:41:17 +0800 Subject: [PATCH 3/3] fix(cooperation): retain form ownership through semantic projection --- .../src/cooperation/controller.test.ts | 6 +- .../cooperation/observation-evidence.test.ts | 20 ++ .../src/cooperation/observation-evidence.ts | 8 +- .../tools/vom/__tests__/form-state.test.ts | 124 +++++++++ .../normalize-form-membership.test.ts | 140 ++++++++++ .../vom/__tests__/semantic-graph.test.ts | 27 ++ apps/extension/src/tools/vom/capture-types.ts | 3 + apps/extension/src/tools/vom/form-capture.ts | 71 +++++ apps/extension/src/tools/vom/normalize.ts | 63 +++++ .../src/tools/vom/semantic-graph/resolve.ts | 2 + apps/extension/src/tools/vom/snapshot.ts | 3 +- .../bsk-cli/skill/references/cooperation.md | 4 +- .../regression/cooperation-video/README.md | 9 +- .../regression/cooperation-video/fixture.html | 48 +++- .../regression/cooperation-video/run.mjs | 258 +++++++++++++++++- .../skill/references/cooperation.md | 4 +- packages/vom/src/types.ts | 3 + 17 files changed, 763 insertions(+), 30 deletions(-) create mode 100644 apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 12768c98..a5b2090d 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -2245,7 +2245,10 @@ describe("default collaboration preserves the original unattended lifetime", () }); describe("looping preview action guards", () => { - type PreviewNode = Pick; + type PreviewNode = Pick< + VomNode, + "id" | "parentId" | "tag" | "role" | "name" | "attrs" | "value" | "inForm" + >; const previewScene = (seconds: number, range = false): VomScene => ({ viewport: { width: 800, height: 600 }, nodes: ( @@ -2268,6 +2271,7 @@ describe("looping preview action guards", () => { tag: "input", role: "slider", name: "Playback progress", + inForm: false, value: String(seconds), attrs: { type: "range" }, } diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index 593d6c03..1f91d89a 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -305,6 +305,7 @@ const mediaScene = () => [ { ...node(7, 1, "slider", "Playback progress"), tag: "input", + inForm: false, value: "1", attrs: { type: "range" }, }, @@ -345,6 +346,7 @@ it.each([ } if (kind === "named input") progress.attrs!.name = "amount"; if (kind === "form member") { + progress.inForm = true; before.push(node(8, 1, "form", "Order")); progress.domAncestorIds = [8, 1]; } @@ -447,3 +449,21 @@ it("keeps explicitly declared DOM ancestors exact even when an AX parent differs false, ); }); + +it.each([ + true, + undefined, +])("keeps media-labelled controls exact when native form membership is %s", (inForm) => { + const before = { nodes: mediaScene() }; + before.nodes[6].inForm = inForm; + const after = structuredClone(before); + after.nodes[6].value = "2"; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); + +it("rejects a change in form membership even when the timeline value is unchanged", () => { + const before = { nodes: mediaScene() }; + const after = structuredClone(before); + after.nodes[6].inForm = true; + expect(sameObservationEvidence(before, after, target)).toBe(false); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index b50bb4c3..3ea00052 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -258,7 +258,6 @@ function signature( // value exact; sharing the inferred region with a button does not. // Keep the original nodes intact: media controls cannot become learned text. const media = nodes.filter((node) => node.tag === "video" || node.tag === "audio"); - const forms = new Set(nodes.filter((node) => node.tag === "form").map((node) => node.id)); const ignoredProgress = new Map(); const explicitlyObserved = (node: VomNode) => !!anchor && (tree.within(node, anchor) || (node.domAncestorIds ?? []).includes(anchor.id)); @@ -268,9 +267,9 @@ function signature( const owner = playbackProgressMedia( { ...node, - inForm: - tree.ancestors(node).some((ancestor) => ancestor.tag === "form") || - (node.domAncestorIds ?? []).some((id) => forms.has(id)), + // Only native ownership or complete raw DOM evidence can establish + // that a control is form-free. Pruned ancestors are not evidence. + inForm: node.inForm !== false, }, media .filter((item) => item.frameId === node.frameId) @@ -293,6 +292,7 @@ function signature( : node.name, tolerate ? undefined : node.text, ignoredProgress.has(node) ? undefined : node.value, + node.inForm, node.checked, node.disabled, node.inert, diff --git a/apps/extension/src/tools/vom/__tests__/form-state.test.ts b/apps/extension/src/tools/vom/__tests__/form-state.test.ts index febf141b..fdf45dc9 100644 --- a/apps/extension/src/tools/vom/__tests__/form-state.test.ts +++ b/apps/extension/src/tools/vom/__tests__/form-state.test.ts @@ -329,3 +329,127 @@ describe("form state identity", () => { expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); }); }); + +it("captures actual native form membership independently of the form's accessibility role", async () => { + const doc = document.implementation.createHTMLDocument("ownership"); + doc.body.innerHTML = + '
'; + const { cdp, nodes } = await fakeCdp( + [[11, 12, 13].map((id) => ({ id, type: "range", value: "0" }))], + (params) => { + const entries = new Function("document", `return ${params.expression}`)(doc) as [ + HTMLInputElement, + string, + ][]; + return reply(entries.map(([element, json]) => [Number(element.id), JSON.parse(json)])); + }, + ); + await enrichFormControlStates(cdp, 7, nodes); + expect(nodes[0].filter((n) => n.tag === "input").map((n) => n.inForm)).toEqual([ + true, + false, + true, + ]); +}); + +it.each([ + true, + false, + undefined, +])("reads missing range ownership by backend identity, preserving unknown results (%s)", async (inForm) => { + const nodes = await decodeControls(snapshot([[{ id: 11, type: "range", value: "0" }]])); + const send = vi.fn(async (_tabId, method) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: "closed-shadow-range" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: inForm } }; + return {}; + }); + await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + expect(send).toHaveBeenCalledWith( + 7, + "DOM.resolveNode", + expect.objectContaining({ backendNodeId: 11 }), + ); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(inForm); + expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); +}); + +it("bounds missing range-owner reads and keeps the uninspected remainder unknown", async () => { + const nodes = await decodeControls( + snapshot([Array.from({ length: 65 }, (_, i) => ({ id: i + 1, type: "range", value: "0" }))]), + ); + const send = vi.fn(async (_tabId, method, params) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: String(params.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") return { result: { value: false } }; + return {}; + }); + await enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes); + expect(send.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(64); + expect(nodes[0].find((n) => n.backendNodeId === 64)?.inForm).toBe(false); + expect(nodes[0].find((n) => n.backendNodeId === 65)?.inForm).toBeUndefined(); +}); + +it("drains in-flight range reads before releasing objects after a terminal error", async () => { + const nodes = await decodeControls( + snapshot([ + [ + { id: 11, type: "range", value: "0" }, + { id: 12, type: "range", value: "0" }, + ], + ]), + ); + let rejectFirst!: (reason: Error) => void; + let resolveSecond!: (value: unknown) => void; + const send = vi.fn(async (_tabId, method, params) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") + return new Promise((resolve, reject) => { + if (params.backendNodeId === 11) rejectFirst = reject; + else resolveSecond = resolve; + }); + return {}; + }); + let settled = false; + const result = enrichFormControlStates({ send: send as CdpRunner["send"] }, 7, nodes).catch( + (error) => { + settled = true; + return error; + }, + ); + await vi.waitFor(() => expect(resolveSecond).toBeTypeOf("function")); + const error = Object.assign(new Error("renderer timed out"), { name: "CdpReadTimeoutError" }); + rejectFirst(error); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(settled).toBe(false); + expect(send).not.toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); + resolveSecond({ object: { objectId: "second" } }); + expect(await result).toBe(error); + expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); + expect(send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); +}); + +it("keeps fallback owner reads with identical backend ids scoped to their OOPIF target", async () => { + const sendToTarget = vi.fn(async (target, method) => { + if (method === "Runtime.evaluate") return reply([]); + if (method === "DOM.resolveNode") return { object: { objectId: "range" } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: target.sessionId === "owned-frame" } }; + return {}; + }); + const send = vi.fn(async () => { + throw new Error("unexpected root target"); + }); + const cdp = { send, sendToTarget } as unknown as CdpRunner; + for (const sessionId of ["owned-frame", "free-frame"]) { + const nodes = await decodeControls(snapshot([[{ id: 11, type: "range", value: "0" }]])); + await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.inForm).toBe(sessionId === "owned-frame"); + expect(sendToTarget).toHaveBeenCalledWith( + { tabId: 7, sessionId }, + "Runtime.callFunctionOn", + expect.objectContaining({ objectId: "range" }), + ); + } + expect(send).not.toHaveBeenCalled(); +}); diff --git a/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts new file mode 100644 index 00000000..77a08d49 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/normalize-form-membership.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { normalizeDocument } from "../normalize"; +import { buildSemanticVomScene } from "../semantic-graph"; +import { BASIC_SNAPSHOT, type SnapshotProfile, VISUAL_SNAPSHOT } from "../snapshot"; + +interface Spec { + id: number; + parent?: number; + tag: string; + attrs?: Record; +} + +async function normalize(input: Spec[], profile: SnapshotProfile = BASIC_SNAPSHOT) { + const strings: string[] = []; + const str = (value: string) => strings.push(value) - 1; + return normalizeDocument( + { + nodes: { + backendNodeId: input.map((node) => node.id), + nodeType: input.map((node) => + node.tag === "#document" ? 9 : node.tag === "#document-fragment" ? 11 : 1, + ), + nodeName: input.map((node) => str(node.tag)), + parentIndex: input.map((node) => { + if (node.parent === undefined) return -1; + const index = input.findIndex((parent) => parent.id === node.parent); + return index >= 0 ? index : input.length + 99; + }), + attributes: input.map((node) => + Object.entries(node.attrs ?? {}).flatMap(([key, value]) => [str(key), str(value)]), + ), + }, + layout: { + nodeIndex: input.map((_, i) => i), + bounds: input.map(() => [0, 0, 100, 30]), + }, + }, + strings, + { + target: { tabId: 1 }, + frameId: "main", + ownerFrameBackendNodeId: null, + projection: null, + coordinates: null, + }, + undefined, + profile, + ); +} + +const roots: Spec[] = [ + { id: 1, tag: "#document" }, + { id: 2, parent: 1, tag: "html" }, +]; +const slider = { role: "slider", "aria-label": "Playback progress", "aria-valuenow": "0" }; + +describe("custom progress form membership", () => { + it.each([ + BASIC_SNAPSHOT, + VISUAL_SNAPSHOT, + ])("preserves form containment before document and semantic pruning: %j", async (profile) => { + const normalized = await normalize( + [ + ...roots, + { id: 3, parent: 2, tag: "form", attrs: { role: "none" } }, + { id: 4, parent: 3, tag: "div", attrs: slider }, + { id: 5, parent: 2, tag: "div", attrs: slider }, + { id: 6, parent: 99, tag: "div", attrs: slider }, + ], + profile, + ); + expect(normalized.nodes.some((node) => node.tag === "#document")).toBe(false); + expect(normalized.index.nodes.get(4)?.inForm).toBe(true); + expect(normalized.index.nodes.get(5)?.inForm).toBe(false); + expect(normalized.index.nodes.get(6)?.inForm).toBeUndefined(); + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + { + frameId: "main", + contextScopeId: "main", + target: { tabId: 1 }, + axNodes: [], + domNodes: normalized.nodes, + }, + ], + }); + expect(scene.nodes.some((node) => node.tag === "form")).toBe(false); + for (const [id, membership] of [ + [4, true], + [5, false], + [6, undefined], + ] as const) + expect(scene.nodes.find((node) => node.backendNodeId === id)?.inForm).toBe(membership); + }); + + it.each(["slider", "progressbar"])("retains raw shadow ancestry for %s widgets", async (role) => { + const normalized = await normalize([ + ...roots, + { id: 3, parent: 2, tag: "form", attrs: { role: "presentation" } }, + { id: 4, parent: 3, tag: "#document-fragment" }, + { id: 5, parent: 4, tag: "div", attrs: { ...slider, role } }, + { id: 6, parent: 2, tag: "#document-fragment" }, + { id: 7, parent: 6, tag: "div", attrs: { ...slider, role } }, + { id: 8, parent: 3, tag: "progress" }, + { id: 9, parent: 2, tag: "progress" }, + ]); + expect([5, 7, 8, 9].map((id) => normalized.index.nodes.get(id)?.inForm)).toEqual([ + true, + false, + true, + false, + ]); + }); + + it("does not infer native ownership from physical containment", async () => { + const normalized = await normalize([ + ...roots, + { id: 3, parent: 2, tag: "form" }, + { id: 4, parent: 3, tag: "input", attrs: { type: "range" } }, + { id: 5, parent: 2, tag: "input", attrs: { type: "range", form: "external" } }, + ]); + for (const id of [4, 5]) expect(normalized.index.nodes.get(id)?.inForm).toBeUndefined(); + }); + + it("keeps disconnected roots and cyclic ancestry unknown", async () => { + const normalized = await normalize([ + { id: 1, tag: "html" }, + { id: 2, parent: 1, tag: "div", attrs: slider }, + { id: 3, parent: 4, tag: "div", attrs: slider }, + { id: 4, parent: 3, tag: "div", attrs: slider }, + { id: 5, parent: 3, tag: "div", attrs: slider }, + { id: 6, parent: 99, tag: "#document" }, + { id: 7, parent: 6, tag: "div", attrs: slider }, + ]); + for (const id of [2, 3, 4, 5, 7]) + expect(normalized.index.nodes.get(id)?.inForm).toBeUndefined(); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index 92033e7c..d1b76af9 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -985,3 +985,30 @@ it("retains browser focus on semantic nodes without trusting a DOM focused attri expect(scene.nodes.find((node) => node.backendNodeId === 2)?.focused).toBe(true); expect(scene.nodes.find((node) => node.backendNodeId === 3)?.focused).toBeUndefined(); }); + +it.each([ + "none", + "presentation", +])("retains native form membership when the %s form is omitted", (role) => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [], + [ + dom(1, null, "main"), + dom(2, 1, "form", { role }), + { + ...dom(3, 2, "input", { type: "range", "aria-label": "Playback progress" }), + formValue: "0", + inForm: true, + }, + ], + ), + ], + }); + expect(scene.nodes.some((n) => n.tag === "form")).toBe(false); + expect(scene.nodes.find((n) => n.backendNodeId === 3)?.inForm).toBe(true); +}); diff --git a/apps/extension/src/tools/vom/capture-types.ts b/apps/extension/src/tools/vom/capture-types.ts index 74910bf0..bfd7abea 100644 --- a/apps/extension/src/tools/vom/capture-types.ts +++ b/apps/extension/src/tools/vom/capture-types.ts @@ -32,6 +32,9 @@ export interface CapturedNode { */ rendered?: boolean; textContent?: string; + /** Native form ownership, or raw containment for custom progress widgets; + * absent when unavailable. Never infer this from the projected semantic tree. */ + inForm?: boolean; formState?: "empty" | "filled" | "default"; formValue?: string; /** Live native checked state, independent of the submitted form value. */ diff --git a/apps/extension/src/tools/vom/form-capture.ts b/apps/extension/src/tools/vom/form-capture.ts index ff1735fc..36e51099 100644 --- a/apps/extension/src/tools/vom/form-capture.ts +++ b/apps/extension/src/tools/vom/form-capture.ts @@ -6,8 +6,10 @@ interface RuntimeEvaluateReply { result?: { deepSerializedValue?: DeepSerializedValue }; } const MAX_FORM_ENRICH_CONTROLS = 250; +const MAX_RANGE_FORM_READS = 64; interface CapturedFormState { + inForm?: boolean; value?: string; defaultValue?: string; placeholder?: string; @@ -37,6 +39,7 @@ function formStateBatchExpression(maxControls: number): string { state, sensitive, placeholder, + inForm: el.form !== null, ...(sensitive ? {} : { value: rawValue, defaultValue }), }; }; @@ -80,6 +83,7 @@ function formStatesByBackendId(result: RuntimeEvaluateReply): Map { + const pending = groups + .flat() + .filter( + (node) => + node.tag === "input" && + node.attrs.type?.toLowerCase() === "range" && + node.inForm === undefined, + ) + .slice(0, MAX_RANGE_FORM_READS); + let cursor = 0; + let failed = false; + let failure: unknown; + await Promise.all( + Array.from({ length: Math.min(4, pending.length) }, async () => { + while (cursor < pending.length && !failed && !signal?.aborted) { + const node = pending[cursor++]; + try { + throwIfAborted(signal); + const resolved = await cdp.send<{ object?: { objectId?: string } }>( + tabId, + "DOM.resolveNode", + { + backendNodeId: node.backendNodeId, + objectGroup, + }, + ); + throwIfAborted(signal); + if (failed) return; + if (!resolved.object?.objectId) continue; + const result = await cdp.send<{ + result?: { value?: unknown }; + exceptionDetails?: unknown; + }>(tabId, "Runtime.callFunctionOn", { + objectId: resolved.object.objectId, + functionDeclaration: "function() { return this.form !== null; }", + returnByValue: true, + throwOnSideEffect: true, + }); + throwIfAborted(signal); + if (!result.exceptionDetails && typeof result.result?.value === "boolean") + node.inForm = result.result.value; + } catch (error) { + if (isCaptureTerminalError(error)) { + if (!failed) failure = error; + failed = true; + } + // Unknown ownership stays protected; never guess from missing ancestors. + } + } + }), + ); + throwIfAborted(signal); + if (failed) throw failure; +} + export async function enrichFormControlStates( cdp: CdpRunner, tabId: number, @@ -143,6 +213,7 @@ export async function enrichFormControlStates( for (const nodes of frameNodeGroups) { applyFormStates(nodes, states); } + await enrichRangeFormMembership(cdp, tabId, frameNodeGroups, objectGroup, signal); return true; } catch (error) { if (isCaptureTerminalError(error)) throw error; diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 3bc5a775..0e331e79 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -51,6 +51,68 @@ export interface NormalizedDocument { documentElementBackendNodeId?: number; } +/** Custom progress widgets have no native .form owner. Preserve their raw + * containment before document/shadow roots and transparent forms are omitted. + * Only a complete path to a document can establish that a widget is form-free. */ +async function captureCustomProgressFormMembership( + index: DocumentIndex, + signal?: AbortSignal, +): Promise { + const checkpoint = createCaptureCheckpoint(signal); + const membership = new Map(); + let work = 0; + for (const node of index.nodes.values()) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (["input", "textarea", "select"].includes(node.tag)) continue; + if ( + node.tag !== "progress" && + !node.attrs.role + ?.toLowerCase() + .split(/\s+/) + .some((role) => ["slider", "progressbar"].includes(role)) + ) + continue; + const path: number[] = []; + const seen = new Set(); + let current: NodeFacts | undefined = node; + let inForm: boolean | undefined; + while (current) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (membership.has(current.backendNodeId)) { + inForm = membership.get(current.backendNodeId); + break; + } + if (seen.has(current.backendNodeId)) break; + seen.add(current.backendNodeId); + path.push(current.backendNodeId); + if (current.tag === "form") { + inForm = true; + break; + } + if (current.parentMissing) break; + if (current.parentBackendNodeId === null) { + if (current.nodeType === 9) inForm = false; + break; + } + current = index.nodes.get(current.parentBackendNodeId); + } + for (const id of path) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + membership.set(id, inForm); + } + node.inForm = inForm; + } +} + /** Interpret one document using a supplied projection. No live reads or frame scheduling. */ export async function normalizeDocument( doc: SnapshotDocument, @@ -88,6 +150,7 @@ export async function normalizeDocument( }); } const index = await buildDocumentIndex(nodes, signal, profile.includeVisualFacts); + await captureCustomProgressFormMembership(index, signal); const document: NormalizedDocument = { nodes: nodes.filter( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index d4f241c5..81e7a7dd 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -486,6 +486,7 @@ export function resolveSemanticGraph( if (isSensitive) delete attrs.value; const frame = graph.frames.get(node.frameId); const tag = node.dom?.tag.toLowerCase() ?? ""; + const inForm = node.dom?.inForm; const vom: ResolvedSemanticNode["vom"] = { tag, rect: node.dom?.rect ?? null, @@ -497,6 +498,7 @@ export function resolveSemanticGraph( ...(role ? { role } : {}), ...(name ? { name } : {}), ...(value !== undefined ? { value } : {}), + ...(inForm !== undefined ? { inForm } : {}), ...(checked !== undefined ? { checked } : {}), ...(node.ax?.ignored !== true && axProperty(node.ax, "focused") === "true" ? { focused: true } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 79172692..c23b2357 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -231,8 +231,7 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, - ...(profile.includeVisualFacts && - (dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null)) + ...(dn.parentIndex?.[n] === undefined || (parentIdx >= 0 && parentBackendNodeId === null) ? { parentMissing: true } : {}), diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index 0e4c0005..1a11b6c1 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -62,7 +62,9 @@ A clearly labelled media timeline with an unambiguous video/audio association ma beside a pause button, including in the same player region. Directly observing the timeline, its media element or a containing element keeps its value exact. Submitted inputs, form members, ordinary sliders and other business values remain protected. Media identity, child `` -declarations and the selected source stay exact. Unknown media controls retain ordinary checks. +declarations and the selected source stay exact. Form ownership remains protected even when +a form is omitted from the observation. Unknown media controls or unavailable form-ownership +evidence retain ordinary checks. ## Scripts and file transfers diff --git a/evals/browser/cases/regression/cooperation-video/README.md b/evals/browser/cases/regression/cooperation-video/README.md index 61129c40..37dd0907 100644 --- a/evals/browser/cases/regression/cooperation-video/README.md +++ b/evals/browser/cases/regression/cooperation-video/README.md @@ -19,7 +19,7 @@ port 52800. `BSK_COOP_EXTENSION` overrides the built extension directory. Chrome uses `Extensions.loadUnpacked`; use `BSK_COOP_LOAD_CDP=0` with builds that support the load extension command-line flags instead. `BSK_COOP_ARTIFACTS` selects the result directory. -The 37 cases cover continuously advancing progress during sync (including incremental sync), +The 53 cases cover continuously advancing progress during sync (including incremental sync), dependencies without a focus target, ordinary business sliders, submitted timelines, form-owned timelines, simultaneous video and order changes, and a programmatic focus change. They also combine PR #497's learned-text behavior with the media fix: a learned price becomes @@ -31,7 +31,12 @@ timeline in one player region. Controls declare the timeline or whole player, fo or change a business field in the same player. Focus changes before a delta sync must refresh semantic evidence, and a document becoming an explicit keyboard recipient revokes the key. Closed shadow-root players also allow progress but reject a source change with or without -a simultaneous timeline update. +a simultaneous timeline update. Form-ownership cases keep the timeline's accessible name +while removing the form from the semantic tree (`role=none`, `role=presentation`, and +`display:contents`). They cover native form owners, external `form` associations, open and +closed shadow roots, custom progress widgets, and formless positive controls. The oracle +asserts the actual form owner and accessible name before changing the value, so losing a +label cannot accidentally make a negative case pass. Positive cases require one received Space event and a paused video. Negative cases require no key event, no simulated purchase, no pause call and `effect_state: none`. Every created session, daemon and temporary browser profile is cleaned up. diff --git a/evals/browser/cases/regression/cooperation-video/fixture.html b/evals/browser/cases/regression/cooperation-video/fixture.html index 6a08cf21..69a41eeb 100644 --- a/evals/browser/cases/regression/cooperation-video/fixture.html +++ b/evals/browser/cases/regression/cooperation-video/fixture.html @@ -34,9 +34,10 @@

Looping preview

if (variant === 'text') dynamic.innerHTML = 'Playback tick 0'; if (['range', 'submitted-range', 'form-range', 'combined', 'source-range', 'src-range', 'same-region', 'same-region-field'].includes(variant)) dynamic.innerHTML = ''; if (variant === 'business-range') dynamic.innerHTML = ''; - if (variant === 'submitted-range') document.querySelector('#progress').name = 'order_amount'; + let progress = document.querySelector('#progress'); + if (variant === 'submitted-range') progress.name = 'order_amount'; if (variant === 'form-range') { - const form = document.createElement('form'); form.id = 'order-form'; dynamic.append(form); form.append(document.querySelector('#progress')); + const form = document.createElement('form'); form.id = 'order-form'; dynamic.append(form); form.append(progress.closest('label') || progress); } if (variant === 'combined') dynamic.insertAdjacentHTML('beforeend', ''); if (variant === 'field' || variant === 'focus') dynamic.innerHTML = ''; @@ -53,11 +54,43 @@

Looping preview

region.id = 'player'; region.setAttribute('aria-label', 'Video player'); controls.before(region); - const timeline = document.querySelector('#progress'); + const timeline = progress; region.append(video, document.querySelector('#pause'), timeline.closest('label') || timeline); controls.remove(); if (variant === 'same-region-field') region.insertAdjacentHTML('beforeend', ''); } + // Preserve the accessible name while removing the form's own visible text. + // This makes role=none/presentation forms actually disappear from the VOM. + if (query.has('formRole')) { + const form = document.createElement(query.get('nonForm') === 'true' ? 'div' : 'form'); + form.id = 'order-form'; + form.setAttribute('role', query.get('formRole') === 'external' ? 'none' : query.get('formRole')); + if (query.get('formContents') === 'true') form.style.display = 'contents'; + const label = progress.closest('label'); + progress.setAttribute('aria-label', query.get('businessLabel') === 'true' ? 'Order amount' : 'Playback progress'); + dynamic.append(form); + if (query.get('formRole') === 'external') { progress.setAttribute('form', form.id); dynamic.append(progress); } + else form.append(progress); + label?.remove(); + if (query.get('named') === 'true') progress.name = 'start_time'; + } + if (query.get('customProgress') === 'true') { + const widget = document.createElement('div'); + widget.id = 'progress'; widget.setAttribute('role', 'slider'); widget.tabIndex = 0; + widget.setAttribute('aria-label', 'Playback progress'); + widget.setAttribute('aria-valuenow', '0'); widget.setAttribute('aria-valuemin', '0'); widget.setAttribute('aria-valuemax', '10000'); + widget.style.cssText = 'width:300px;height:20px;background:#ccd6e0'; + progress.replaceWith(widget); progress = widget; + } + if (query.has('controlShadow')) { + const container = progress.closest('form') || progress.closest('label') || progress; + const host = document.createElement('div'); dynamic.append(host); + host.attachShadow({mode: query.get('controlShadow')}).append(video, container); + if (query.get('shadowHostForm') === 'true') { + const outer = document.createElement('form'); outer.id = 'outer-form'; outer.setAttribute('role', 'none'); + host.before(outer); outer.append(host); + } + } const pause = () => { stats.pauseCalls++; video.pause(); }; document.querySelector('#pause').addEventListener('click', () => { stats.clicks++; pause(); }); document.addEventListener('keydown', event => { @@ -71,7 +104,8 @@

Looping preview

if (video.paused) return; stats.ticks++; if (variant === 'text') document.querySelector('#clock').textContent = `Playback tick ${stats.ticks}`; - if (document.querySelector('#progress')) document.querySelector('#progress').value = String(stats.ticks % 10000); + if (progress instanceof HTMLInputElement) progress.value = String(stats.ticks % 10000); + else if (progress) progress.setAttribute('aria-valuenow', String(stats.ticks % 10000)); if (['field', 'combined', 'same-region-field'].includes(variant)) document.querySelector('#order').value = String(100 + stats.ticks); if (variant === 'body-recipient') { document.body.tabIndex = 0; document.body.setAttribute('role', 'button'); } if (variant === 'focus') document.querySelector('#order').focus(); @@ -103,7 +137,11 @@

Looping preview

sourceSrc: video.querySelector('source')?.src, focused: document.activeElement?.id || document.activeElement?.tagName, clock: document.querySelector('#clock')?.textContent, - progress: document.querySelector('#progress')?.value, + progress: progress instanceof HTMLInputElement ? progress.value : progress?.getAttribute('aria-valuenow'), + formOwner: (progress instanceof HTMLInputElement ? progress.form : progress?.closest('form'))?.id || null, + progressLabel: progress?.getAttribute('aria-label') || progress?.labels?.[0]?.textContent?.trim(), + controlName: progress?.getAttribute('name') || '', + progressShadow: progress?.getRootNode() instanceof ShadowRoot, order: document.querySelector('#order')?.value, price: document.querySelector('#price')?.textContent, }); diff --git a/evals/browser/cases/regression/cooperation-video/run.mjs b/evals/browser/cases/regression/cooperation-video/run.mjs index 6fac3c23..ac8b8b9c 100644 --- a/evals/browser/cases/regression/cooperation-video/run.mjs +++ b/evals/browser/cases/regression/cooperation-video/run.mjs @@ -378,6 +378,177 @@ const cases = [ allow: false, focus: true, }, + { + name: "form_role_none_progress_still_blocked", + formRole: "none", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_presentation_progress_still_blocked", + formRole: "presentation", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_display_contents_progress_still_blocked", + formRole: "none", + formContents: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "external_form_owner_progress_still_blocked", + formRole: "external", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_none_named_progress_still_blocked", + formRole: "none", + named: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "form_role_none_explicit_timeline_still_blocked", + formRole: "none", + dependencies: ["#progress"], + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + allow: false, + }, + { + name: "form_role_none_whole_page_still_blocked", + formRole: "none", + dependencies: false, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + allow: false, + }, + { + name: "form_role_none_business_label_still_blocked", + formRole: "none", + businessLabel: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "non_form_role_none_progress_allowed", + formRole: "none", + nonForm: true, + allow: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "open_shadow_form_progress_still_blocked", + formRole: "none", + controlShadow: "open", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "closed_shadow_form_progress_still_blocked", + formRole: "none", + controlShadow: "closed", + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "closed_shadow_continuous_progress_allowed", + controlShadow: "closed", + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "open_shadow_continuous_progress_allowed", + controlShadow: "open", + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "shadow_host_form_does_not_own_timeline", + controlShadow: "closed", + shadowHostForm: true, + continuous: true, + allow: true, + variant: "range", + target: true, + manual: false, + ownershipCheck: true, + dependencies: ["#pause"], + }, + { + name: "custom_progress_in_pruned_form_still_blocked", + formRole: "none", + customProgress: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + allow: false, + }, + { + name: "custom_non_form_progress_allowed", + customProgress: true, + allow: true, + variant: "range", + target: true, + manual: true, + ownershipCheck: true, + dependencies: ["#pause"], + }, ]; try { await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); @@ -423,7 +594,24 @@ try { const scope = ["--session", session]; let oracle; try { - const url = `${base}/?variant=${scenario.variant}&case=${scenario.name}&interval=${scenario.continuous ? 10 : 80}&shadow=${scenario.closedShadow ? "closed" : "none"}`; + const query = new URLSearchParams({ + variant: scenario.variant, + case: scenario.name, + interval: String(scenario.continuous ? 10 : 80), + shadow: scenario.closedShadow ? "closed" : "none", + }); + for (const key of [ + "formRole", + "formContents", + "named", + "businessLabel", + "nonForm", + "controlShadow", + "customProgress", + "shadowHostForm", + ]) + if (scenario[key] !== undefined) query.set(key, String(scenario[key])); + const url = `${base}/?${query}`; await ok(["navigate", url, ...scope]); const target = (await send("Target.getTargets")).targetInfos.find( (target) => target.url === url, @@ -516,20 +704,56 @@ try { } const before = await read("window.report()"); assert.equal(before.paused, false); + if (scenario.ownershipCheck || scenario.variant === "form-range") { + assert.equal( + before.progressLabel, + scenario.businessLabel ? "Order amount" : "Playback progress", + "The regression must keep the media-like accessible name", + ); + assert.equal( + before.formOwner, + scenario.variant === "form-range" || (scenario.formRole && !scenario.nonForm) + ? "order-form" + : null, + ); + if (scenario.ownershipCheck) + assert.equal(before.controlName, scenario.named ? "start_time" : ""); + assert.equal(before.progressShadow, !!scenario.controlShadow); + } // Start page-origin updates after the observation, before the proposed input. await read( scenario.learn || scenario.manual ? "window.tickNoise(); true" : "window.startNoise(); true", ); - await delay(320); - const atAction = await read("window.report()"); - assert( - scenario.source - ? atAction.presentedFrames > before.presentedFrames - : atAction.decodedFrames > before.decodedFrames, - "Video must actually decode frames, including after source reload", - ); + let progressReport; + let sourceBaseline; + let atAction; + try { + atAction = await poll(async () => { + progressReport = await read("window.report()"); + // A reload resets the native frame counter. Establish a new + // baseline only after currentSrc changes, then require more + // decoded frames from that same source on a later read. + if (scenario.source) { + if (progressReport.currentSrc === before.currentSrc || !progressReport.currentSrc) + return; + if (sourceBaseline?.currentSrc !== progressReport.currentSrc) { + sourceBaseline = progressReport; + return; + } + } + const advanced = + progressReport.decodedFrames > (sourceBaseline ?? before).decodedFrames; + return advanced && progressReport.ticks > before.ticks && progressReport; + }, "actual video frames and page updates before input"); + } catch (error) { + await writeFile( + join(artifacts, scenario.name + ".progress-error.json"), + JSON.stringify({ before, current: progressReport }, null, 2), + ); + throw error; + } if (scenario.source) assert.notEqual(atAction.currentSrc, before.currentSrc); assert(atAction.ticks > before.ticks, "Page updates must run before the input"); if (atAction.progress !== undefined) assert.notEqual(atAction.progress, before.progress); @@ -561,6 +785,7 @@ try { learning, before, atAction, + sourceBaseline, coordinationReady: wait.ready, action, after, @@ -571,11 +796,16 @@ try { JSON.stringify(evidence, null, 2), ); if (scenario.name === "timer_text_untargeted_space") { - const screenshot = await send("Page.captureScreenshot", { format: "png" }, oracle); - await writeFile( - join(artifacts, "blocked-preview.png"), - Buffer.from(screenshot.data, "base64"), - ); + // Diagnostic only: all input and page-state assertions still run. + try { + const screenshot = await send("Page.captureScreenshot", { format: "png" }, oracle); + await writeFile( + join(artifacts, "blocked-preview.png"), + Buffer.from(screenshot.data, "base64"), + ); + } catch (error) { + await writeFile(join(artifacts, "blocked-preview-error.txt"), String(error)); + } } assert.equal(wait.ready, true, "No person should be holding control"); assert.equal(after.purchases, 0, "No simulated purchase may execute"); diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index 45c495c2..9831fb53 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -56,7 +56,9 @@ A clearly labelled media timeline with an unambiguous video/audio association ma beside a pause button, including in the same player region. Directly observing the timeline, its media element or a containing element keeps its value exact. Submitted inputs, form members, ordinary sliders and other business values remain protected. Media identity, child `` -declarations and the selected source stay exact. Unknown media controls retain ordinary checks. +declarations and the selected source stay exact. Form ownership remains protected even when +a form is omitted from the observation. Unknown media controls or unavailable form-ownership +evidence retain ordinary checks. ## Scripts and file transfers diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index a78e3cf6..50d7a58c 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -37,6 +37,9 @@ export interface VomNode { /** Browser accessibility focus, used to protect implicit keyboard recipients. */ focused?: boolean; placeholder?: string; + /** Native form ownership, or raw DOM form containment for custom widgets. + * Absence means unknown, never proof that a control is outside a form. */ + inForm?: boolean; inputState?: "empty" | "filled" | "default" | "unknown"; href?: string; // hostname of external link target; omitted for same-origin links text?: string;