From 075cf43aa3f35f53b3bb351736b60d642e92dc61 Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Sun, 25 Oct 2026 23:23:46 -0500 Subject: [PATCH 1/4] fix(cooperation): stop live page text from refusing every action Without --depends-on, the pre-action evidence check compares the whole observed page, including every node's text. A clock, countdown or live counter outside the action's region changes that text between sync and action, so sync -> act -> "observed content changed" could repeat forever. When a whole-page check fails only because persistent, unprotected nodes changed their text in place, with no human input since the sync and the action's own region unchanged, report the change once (with before/after text) and remember those nodes for the current documents. Later default checks ignore further text-only changes to them. Identity, structure, visibility, form values, modal state, target regions and explicit --depends-on dependencies stay exact. --- .../src/cooperation/controller.test.ts | 123 ++++++++++++++++++ apps/extension/src/cooperation/controller.ts | 63 +++++++-- .../cooperation/observation-evidence.test.ts | 46 ++++++- .../src/cooperation/observation-evidence.ts | 81 ++++++++++-- .../bsk-cli/skill/references/cooperation.md | 7 +- docs/shared-browser-control.zh-CN.md | 2 +- .../skill/references/cooperation.md | 7 +- 7 files changed, 305 insertions(+), 24 deletions(-) diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 3fe146ab..3a7c5244 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -499,6 +499,129 @@ describe("shared-tab dispatch fences", () => { } expect(resolveActionTarget).toHaveBeenCalled(); }); + describe("self-changing page content", () => { + // A clock (node 5) ticks on every observation; the action targets node 4. + const scene = (clock: string, button = "Continue", extra = false): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: [ + { id: 1, parentId: null, tag: "main", name: "", role: "main" }, + { id: 2, parentId: 1, tag: "form", name: "", role: "form" }, + { id: 4, parentId: 2, tag: "button", name: button, role: "button" }, + { id: 5, parentId: 1, tag: "span", name: "", text: clock }, + ...(extra ? [{ id: 6, parentId: 1, tag: "p", name: "", text: "Sold out" }] : []), + ].map((node) => ({ + ...node, + backendNodeId: node.id, + frameId: "frame", + rect: { x: 0, y: 0, w: 20, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + })), + }); + const live = (next: (tick: number) => VomScene) => { + let tick = 0; + vi.mocked(handleSnapshot).mockImplementation(async (_m, _p, deps) => { + deps!.onScene?.(next(tick++)); + return { text: "Continue", ref_count: 1, tab_id: 7, truncated: false }; + }); + vi.mocked(captureVomObservation).mockImplementation(async (_c, _t, _u, options) => { + options?.onScene?.(next(tick++)); + return {} as Awaited>; + }); + }; + const act = (f: Awaited>, context: string, dependsOn?: string[]) => + f.controller.run( + { + id: "live", + method: "tool.click", + params: { + session_id: "shared", + tab_id: 7, + context_id: context, + ref: "e4", + ...(dependsOn ? { depends_on: dependsOn } : {}), + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchMouseEvent", { type: "mousePressed" }), + ) as Promise>; + // Every mutation of the live region also bumps the structure counter. + const cycle = async ( + f: Awaited>, + context: string, + deps?: string[], + ) => { + f.change({ structure: f.controller.view(7)!.revision + 100 }); + return act(f, context, deps); + }; + afterEach(() => { + vi.mocked(handleSnapshot).mockReset(); + vi.mocked(resolveActionTarget).mockReset(); + }); + + it("reports a ticking clock once instead of refusing every sync and action", async () => { + live((tick) => scene(`10:00:0${tick}`)); + const f = await fixture(); + const refused = await cycle(f, f.first.context_id); + expect(refused).toMatchObject({ + message: expect.stringContaining('"10:00:00" → "10:00:01"'), + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(refused.message).toContain("--depends-on"); + expect(f.native).not.toHaveBeenCalled(); + for (let attempt = 0; attempt < 3; attempt++) { + const synced = await f.command("sync"); + expect(synced.context_id).toBeTruthy(); + expect(await cycle(f, synced.context_id)).toEqual({}); + } + expect(f.native).toHaveBeenCalledTimes(3); + }); + + it("keeps refusing when the action's own region changes", async () => { + live((tick) => scene("10:00:00", `Pay ${tick}`)); + const f = await fixture(); + for (let attempt = 0, context = f.first.context_id; attempt < 2; attempt++) { + expect(await cycle(f, context)).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + }); + context = (await f.command("sync")).context_id; + } + expect(f.native).not.toHaveBeenCalled(); + }); + + it("still refuses new content beside a tolerated clock", async () => { + live((tick) => scene(`10:00:0${tick}`, "Continue", tick >= 3)); + const f = await fixture(); + expect((await cycle(f, f.first.context_id)).message).toContain("changed by itself"); + const synced = await f.command("sync"); + expect(await cycle(f, synced.context_id)).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + }); + expect(f.native).not.toHaveBeenCalled(); + }); + + it("keeps explicit dependencies exact after learning page activity", async () => { + live((tick) => scene(`10:00:0${tick}`)); + vi.mocked(resolveActionTarget).mockImplementation( + async (_c, _s, _t, params) => + ({ + backendNodeId: (params as { ref?: string }).ref === "e5" ? 5 : 4, + frameId: "frame", + cdpTarget: { tabId: 7 }, + tab: { tabId: 7 }, + }) as Awaited>, + ); + const f = await fixture(); + expect((await cycle(f, f.first.context_id)).message).toContain("changed by itself"); + const synced = await f.command("sync"); + expect(await cycle(f, synced.context_id, ["e5"])).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + }); + expect(f.native).not.toHaveBeenCalled(); + }); + }); it("uses updated field evidence after a delta when a later cosmetic mutation occurs", async () => { const scene = (value: string): VomScene => ({ viewport: { width: 800, height: 600 }, diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 3fa82667..74e93cc2 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -11,6 +11,7 @@ import type { RequestFrame, RpcError } from "@/transport/types"; import { guardedRunner } from "./action-guard"; import { type ControlAuthority, ControlAuthorityError } from "./control-authority"; import { + contentChanges, type ObservationEvidence, observationEvidence, sameObservationEvidence, @@ -60,6 +61,11 @@ interface TabState { controlRevision: number; /** Revokes permission guards on human intervention, independently of page revisions. */ authorityRevision: number; + /** + * Nodes whose text changed by itself between a sync and an action (clocks, + * counters, tickers), keyed to the documents they were observed in. + */ + volatile?: { documents: string; keys: Set }; } export interface CooperationDeps { sessions: SessionManager; @@ -98,6 +104,8 @@ const PASSIVE_DEBUG = new Set( ), ); const QUIET_MS = 650; +/** Upper bound on nodes whose self-changing text the default check tolerates. */ +const MAX_VOLATILE = 256; function isInitialBlank(tab: { url?: string; pendingUrl?: string }): boolean { // Chrome can report the agent-created blank page before its initial load commits. @@ -116,6 +124,12 @@ function sameFrames(a: Frames, b: Frames, field: "revision" | "structure" | "hum }) ); } +function documents(frames: Frames): string { + return JSON.stringify([...frames].map(([id, frame]) => [id, frame.document])); +} +function quote(text: string): string { + return JSON.stringify(text.length > 40 ? `${text.slice(0, 39)}…` : text); +} function sameFields(a: Frames, b: Frames): boolean { return ( a.size === b.size && @@ -651,7 +665,7 @@ export class CooperationController implements ControlAuthority { }), state: { ...this.view(state.tabId), mode: "ready" }, guidance: - "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. To tolerate unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", + "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked; text seen changing by itself outside the target region (clocks, counters) is reported once and then tolerated. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", }; } finally { state.working = false; @@ -963,6 +977,7 @@ export class CooperationController implements ControlAuthority { const epoch = state.epoch; let verifiedFrames = context?.frames; let evidenceTargets: { backendNodeId: number; frameId?: string }[] | undefined; + let actionTarget: { backendNodeId: number; frameId?: string } | undefined; state.working = true; const intervention = new AbortController(); state.interruption = intervention; @@ -1001,17 +1016,46 @@ export class CooperationController implements ControlAuthority { }); await this.refresh(state); assert(); - if ( - !evidence || - !(evidenceTargets ?? [undefined]).every((dependency) => - sameObservationEvidence(context.evidence!, evidence!, dependency), - ) - ) + if (!evidence) throw new Error( (revoked = "The action's observed content changed; synchronize and replan"), ); - // Keep the pre-capture revision: later mutations must be checked again. - verifiedFrames = before; + // Explicit dependencies stay exact. The default whole-page check tolerates + // text that was already seen changing by itself in these documents. + const pages = documents(state.frames); + const learning = !evidenceTargets && !params.depends_on?.length; + const volatile = + learning && state.volatile?.documents === pages ? state.volatile.keys : undefined; + if ( + evidenceTargets + ? evidenceTargets.every((dependency) => + sameObservationEvidence(context.evidence!, evidence!, dependency), + ) + : sameObservationEvidence(context.evidence!, evidence!, undefined, volatile) + ) { + // Keep the pre-capture revision: later mutations must be checked again. + verifiedFrames = before; + return; + } + // Learn page activity only from in-place text changes outside the target + // region with no human input since the sync. The agent still replans once. + const changes = + learning && + sameFrames(context.frames, state.frames, "human") && + (!actionTarget || sameObservationEvidence(context.evidence, evidence, actionTarget)) + ? contentChanges(context.evidence, evidence, volatile) + : undefined; + if (changes && (volatile?.size ?? 0) + changes.length <= MAX_VOLATILE) { + state.volatile = { + documents: pages, + keys: new Set([...(volatile ?? []), ...changes.map((change) => change.key)]), + }; + const [first] = changes; + throw new Error( + (revoked = `Page content outside the action's region changed by itself (${quote(first.before)} → ${quote(first.after)}${changes.length > 1 ? ` and ${changes.length - 1} more` : ""}); synchronize and replan. Later text-only changes there will not block actions without --depends-on; declare them as dependencies if the decision relies on them`), + ); + } + throw new Error((revoked = "The action's observed content changed; synchronize and replan")); }; try { await this.refresh(state); @@ -1032,6 +1076,7 @@ export class CooperationController implements ControlAuthority { "shared action", ); if (isRpcError(resolved)) return resolved; + actionTarget = resolved; if (params.depends_on?.length) evidenceTargets = [resolved]; } if (params.depends_on !== undefined) { diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index af718e65..4049336d 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -1,6 +1,6 @@ import type { VomNode } from "@browser-skill/vom"; import { expect, it } from "vitest"; -import { sameObservationEvidence } from "./observation-evidence"; +import { contentChanges, sameObservationEvidence } from "./observation-evidence"; const node = (id: number, parentId: number | null, role: string, name = ""): VomNode => ({ id, @@ -70,3 +70,47 @@ it("checks fields outside the action region and falls back to the whole document expect(sameObservationEvidence({ nodes: before }, { nodes: after }, target)).toBe(false); expect(sameObservationEvidence({ nodes: scene() }, { nodes: after })).toBe(false); }); + +it("identifies in-place text changes and tolerates only those in the whole-page check", () => { + const before = scene(); + before.push({ ...node(6, 1, "timer"), text: "10:00:00" }); + const after = structuredClone(before); + after[5].text = "10:00:01"; + expect(contentChanges({ nodes: before }, { nodes: after })).toEqual([ + { key: "frame:6", before: "10:00:00", after: "10:00:01" }, + ]); + const volatile = new Set(["frame:6"]); + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, undefined, volatile)).toBe( + true, + ); + for (const change of ["removed", "disabled", "hidden", "added"]) { + const next = structuredClone(after); + if (change === "removed") next.pop(); + if (change === "disabled") next[5].disabled = true; + if (change === "hidden") next[5].rect = { x: 0, y: 0, w: 0, h: 0 }; + if (change === "added") next.push(node(7, 1, "status", "Sold out")); + expect(sameObservationEvidence({ nodes: before }, { nodes: next }, undefined, volatile)).toBe( + false, + ); + expect(contentChanges({ nodes: before }, { nodes: next }, volatile)).toBeUndefined(); + } + // Explicit targets never use the tolerated set. + expect( + sameObservationEvidence({ nodes: before }, { nodes: after }, { backendNodeId: 6 }, volatile), + ).toBe(false); +}); + +it("never treats form values or modal content as page activity", () => { + const before = scene(); + before.push({ ...node(6, 1, "textbox", "Amount"), value: "1" }); + before.push({ ...node(7, 1, "dialog", "Confirm"), modal: true }); + for (const index of [5, 6]) { + const after = structuredClone(before); + after[index].name = "changed"; + const volatile = new Set([`frame:${index + 1}`]); + expect(contentChanges({ nodes: before }, { nodes: after })).toBeUndefined(); + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, undefined, volatile)).toBe( + false, + ); + } +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index e8615071..4a63b154 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -30,11 +30,33 @@ const semanticAttributes = [ "required", ]; +/** A text change of one persistent node between two observations. */ +export interface ContentChange { + key: string; + before: string; + after: string; +} + export function observationEvidence(scene: VomScene): ObservationEvidence { return { nodes: scene.nodes }; } -function signature(evidence: ObservationEvidence, target?: EvidenceTarget): string | undefined { +/** Stable identity of an observed node within its document. */ +function evidenceKey(node: VomNode): string | undefined { + return node.backendNodeId === undefined + ? undefined + : `${node.frameId ?? ""}:${node.backendNodeId}`; +} + +// Form state and modal layers are never tolerated as page activity. +const protectedState = (node: VomNode) => + node.value !== undefined || node.checked !== undefined || !!node.modal; + +function signature( + evidence: ObservationEvidence, + target?: EvidenceTarget, + volatile?: ReadonlySet, +): string | undefined { const nodes = evidence.nodes; const byId = new Map(nodes.map((node) => [node.id, node])); const parent = (node: VomNode) => byId.get(node.domParentId ?? node.parentId ?? -1); @@ -69,13 +91,17 @@ function signature(evidence: ObservationEvidence, target?: EvidenceTarget): stri } return false; }; - const facts = (node: VomNode) => [ + // Only the whole-page default check tolerates text that changed by itself; + // identity, structure, state and visibility of those nodes remain checked. + const tolerated = (node: VomNode) => + !target && !protectedState(node) && volatile?.has(evidenceKey(node) ?? "") === true; + const facts = (node: VomNode, tolerate = false) => [ node.frameId, node.backendNodeId, node.role, node.tag, - node.name, - node.text, + tolerate ? undefined : node.name, + tolerate ? undefined : node.text, node.value, node.checked, node.disabled, @@ -87,18 +113,55 @@ function signature(evidence: ObservationEvidence, target?: EvidenceTarget): stri ]; return JSON.stringify([ scope && [scope.frameId, scope.backendNodeId], - nodes.filter(inside).map(facts), + nodes.filter(inside).map((node) => facts(node, tolerated(node))), // Form state outside the action region can still determine its meaning. - nodes.filter((node) => node.value !== undefined || node.checked !== undefined).map(facts), - nodes.filter((node) => node.modal).map(facts), + nodes + .filter((node) => node.value !== undefined || node.checked !== undefined) + .map((node) => facts(node)), + nodes.filter((node) => node.modal).map((node) => facts(node)), ]); } +/** + * `volatile` applies only to the whole-page check (no target): text of those + * nodes may differ. Explicit dependencies and target regions stay exact. + */ export function sameObservationEvidence( before: ObservationEvidence, after: ObservationEvidence, target?: EvidenceTarget, + volatile?: ReadonlySet, ): boolean { - const a = signature(before, target); - return a !== undefined && a === signature(after, target); + const a = signature(before, target, volatile); + return a !== undefined && a === signature(after, target, volatile); +} + +/** + * Returns the persistent, unprotected nodes whose text changed when that is the + * only difference between two whole-page observations; otherwise undefined. + * Added, removed, reordered or restyled nodes and any form or modal change + * are not page activity and keep requiring a new decision. + */ +export function contentChanges( + before: ObservationEvidence, + after: ObservationEvidence, + known: ReadonlySet = new Set(), +): ContentChange[] | undefined { + const previous = new Map(before.nodes.map((node) => [evidenceKey(node), node])); + const changes: ContentChange[] = []; + for (const node of after.nodes) { + const key = evidenceKey(node); + const old = key === undefined ? undefined : previous.get(key); + if (!key || !old || known.has(key) || protectedState(node) || protectedState(old)) continue; + if (old.name !== node.name || old.text !== node.text) + changes.push({ + key, + before: old.text ?? old.name ?? "", + after: node.text ?? node.name ?? "", + }); + } + const volatile = new Set([...known, ...changes.map((change) => change.key)]); + return changes.length && sameObservationEvidence(before, after, undefined, volatile) + ? changes + : undefined; } diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index 97d22b17..bc2357aa 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -36,8 +36,11 @@ The default guard checks the whole observed page. When the decision depends on k CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every region used in the decision: for example both the options list and the current form. The target's form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Dependencies must come from the current -observation; they do not authorize replaying an old plan. Cosmetic changes still require a +uncertain, omit it and retain whole-page checking. Text outside the target's region that changes +by itself between sync and action (a clock, a counter) is reported once with the before/after +text; after you sync and replan, the default check tolerates further text-only changes there. +If the decision relies on such text, declare it with `--depends-on`, which stays exact. +Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and discrete controls return control automatically; text editing, composition, held keys and dragging keep the person's lease until they finish. diff --git a/docs/shared-browser-control.zh-CN.md b/docs/shared-browser-control.zh-CN.md index 8e1cd5a5..b339a67f 100644 --- a/docs/shared-browser-control.zh-CN.md +++ b/docs/shared-browser-control.zh-CN.md @@ -50,7 +50,7 @@ DSH 使用 `bsk_session` 的 `sync`、`wait-user`、`cooperation-status`。旧 ` 会话生命周期复用 `session.activity` 事件。Daemon 只接受所属浏览器的状态更新，将协作保持与最后工具调用时间一起保存在既有活动记录中；结束保持后有完整的空闲间隔。释放、断线和 session id 复用均清理这一状态。 -默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。持续刷新且与决策无关的区域可以不阻断动作，但有意义的变化仍要求重新同步和规划。 +默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。目标区域以外、在同步与动作之间自行变化的文字（如时钟、计数器）会先被报告一次并附带前后文本；Agent 重新同步和规划后，默认整页校验不再因这些节点的纯文本变化而拒绝。结构、可见性、表单值和模态变化仍会拒绝；若决策依赖这些文字，用 `--depends-on` 声明，显式依赖始终精确校验。 `cooperation-capture.ts` 在隔离世界采集输入和 DOM 变化。输入保持即时通知，DOM 变更合并后通知；主文档及 iframe 都必须响应。普通字段变化走增量路径，字段以文档内稳定标识区分，同名字段和开放 Shadow DOM 输入框也分别记录。结构变化、文档替换、字段过多或内容截断时，复用现有 VOM 快照刷新基线。增量字段变更同步后会更新内部语义证据，避免后续动作仍拿旧值比较；不重复输出整页文本，不替换引用表。采集不触发 hover。密码、文件选择不进入字段变化流。 diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index abf5fab0..c694d587 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -30,8 +30,11 @@ The default guard checks the whole observed page. When the decision depends on k CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every region used in the decision: for example both the options list and the current form. The target's form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Dependencies must come from the current -observation; they do not authorize replaying an old plan. Cosmetic changes still require a +uncertain, omit it and retain whole-page checking. Text outside the target's region that changes +by itself between sync and action (a clock, a counter) is reported once with the before/after +text; after you sync and replan, the default check tolerates further text-only changes there. +If the decision relies on such text, declare it with `--depends-on`, which stays exact. +Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and discrete controls return control automatically; text editing, composition, held keys and dragging keep the person's lease until they finish. From a98cc560f79aa97b68dd8b230426538f0ad403c4 Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Mon, 26 Oct 2026 04:53:11 -0500 Subject: [PATCH 2/4] fix(cooperation): keep target regions exact and require a sync after refusal Address review on #497: - The tolerant whole-page check now also requires the current action's own region to match exactly, so a node learned while acting elsewhere stays protected when a later action targets its region. - Any evidence refusal consumes the refused context, and newly learned text changes stay pending until the next successful sync. Retrying with the refused context_id can no longer dispatch input. Add regression tests for switching action regions after learning and for retrying the refused context without a sync; both fail on 075cf43. --- .../src/cooperation/controller.test.ts | 73 +++++++++++++++++++ apps/extension/src/cooperation/controller.ts | 53 +++++++++++--- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 3a7c5244..558f5e23 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -621,6 +621,79 @@ describe("shared-tab dispatch fences", () => { }); expect(f.native).not.toHaveBeenCalled(); }); + + it("does not let a refused context retry with newly learned tolerance", async () => { + live((tick) => scene(`10:00:0${tick}`)); + const f = await fixture(); + expect((await cycle(f, f.first.context_id)).message).toContain("changed by itself"); + expect(await cycle(f, f.first.context_id)).toMatchObject({ + message: "This action requires the context returned by the latest cooperate sync", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + // The fresh observation activates the learned clock, preserving the loop fix. + const synced = await f.command("sync"); + expect(await cycle(f, synced.context_id)).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); + }); + + it("keeps a learned node exact when it belongs to a later action's region", async () => { + // Form A holds Continue (4); form B holds a changing price (7) and Pay (8). + const shop = (price: string): VomScene => ({ + viewport: { width: 800, height: 600 }, + nodes: [ + { id: 1, parentId: null, tag: "main", name: "", role: "main" }, + { id: 2, parentId: 1, tag: "form", name: "A", role: "form" }, + { id: 4, parentId: 2, tag: "button", name: "Continue", role: "button" }, + { id: 6, parentId: 1, tag: "form", name: "B", role: "form" }, + { id: 7, parentId: 6, tag: "span", name: "", text: price }, + { id: 8, parentId: 6, tag: "button", name: "Pay", role: "button" }, + ].map((node) => ({ + ...node, + backendNodeId: node.id, + frameId: "frame", + rect: { x: 0, y: 0, w: 20, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + })), + }); + live((tick) => shop(`$${100 + tick}`)); + vi.mocked(resolveActionTarget).mockImplementation( + async (_c, _s, _t, params) => + ({ + backendNodeId: (params as { ref?: string }).ref === "e8" ? 8 : 4, + frameId: "frame", + cdpTarget: { tabId: 7 }, + tab: { tabId: 7 }, + }) as Awaited>, + ); + const click = (f: Awaited>, context: string, ref: string) => { + f.change({ structure: f.controller.view(7)!.revision + 100 }); + return f.controller.run( + { + id: "shop", + method: "tool.click", + params: { session_id: "shared", tab_id: 7, context_id: context, ref }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchMouseEvent", { type: "mousePressed" }), + ) as Promise>; + }; + const f = await fixture(); + // Acting in form A learns the price in form B as page activity. + expect((await click(f, f.first.context_id, "e4")).message).toContain("changed by itself"); + // Form A actions now tolerate the price... + expect(await click(f, (await f.command("sync")).context_id, "e4")).toEqual({}); + expect(f.native).toHaveBeenCalledOnce(); + // ...but paying in form B still requires the price observed at sync. + expect(await click(f, (await f.command("sync")).context_id, "e8")).toMatchObject({ + message: "The action's observed content changed; synchronize and replan", + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).toHaveBeenCalledOnce(); + }); }); it("uses updated field evidence after a delta when a later cosmetic mutation occurs", async () => { const scene = (value: string): VomScene => ({ diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index 74e93cc2..dabc7efd 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -65,7 +65,13 @@ interface TabState { * Nodes whose text changed by itself between a sync and an action (clocks, * counters, tickers), keyed to the documents they were observed in. */ - volatile?: { documents: string; keys: Set }; + volatile?: Volatile; + /** Changes learned from a refused action; active only after a fresh sync. */ + learned?: Volatile; +} +interface Volatile { + documents: string; + keys: Set; } export interface CooperationDeps { sessions: SessionManager; @@ -616,6 +622,19 @@ export class CooperationController implements ControlAuthority { }; state.context = context; state.baseline = context; + // The agent has now observed the refused change; tolerate it from here on. + if (state.learned) { + const pages = documents(before); + if (state.learned.documents === pages) + state.volatile = { + documents: pages, + keys: new Set([ + ...(state.volatile?.documents === pages ? state.volatile.keys : []), + ...state.learned.keys, + ]), + }; + state.learned = undefined; + } const activity = state.activity.filter( (event) => event.cursor > (baseline?.activityCursor ?? 0), ); @@ -1016,39 +1035,55 @@ export class CooperationController implements ControlAuthority { }); await this.refresh(state); assert(); - if (!evidence) + // A refusal always requires a fresh sync; the refused context cannot retry. + const consumeRefused = () => { + if (state.context !== context) return; + state.context = undefined; + this.changed(state); + }; + if (!evidence) { + consumeRefused(); throw new Error( (revoked = "The action's observed content changed; synchronize and replan"), ); + } // Explicit dependencies stay exact. The default whole-page check tolerates // text that was already seen changing by itself in these documents. const pages = documents(state.frames); const learning = !evidenceTargets && !params.depends_on?.length; const volatile = learning && state.volatile?.documents === pages ? state.volatile.keys : undefined; + // The action's own region stays exact even when it holds learned nodes. + const targetIntact = + !actionTarget || sameObservationEvidence(context.evidence, evidence, actionTarget); if ( evidenceTargets ? evidenceTargets.every((dependency) => sameObservationEvidence(context.evidence!, evidence!, dependency), ) - : sameObservationEvidence(context.evidence!, evidence!, undefined, volatile) + : targetIntact && + sameObservationEvidence(context.evidence!, evidence!, undefined, volatile) ) { // Keep the pre-capture revision: later mutations must be checked again. verifiedFrames = before; return; } + consumeRefused(); // Learn page activity only from in-place text changes outside the target // region with no human input since the sync. The agent still replans once. const changes = - learning && - sameFrames(context.frames, state.frames, "human") && - (!actionTarget || sameObservationEvidence(context.evidence, evidence, actionTarget)) + learning && targetIntact && sameFrames(context.frames, state.frames, "human") ? contentChanges(context.evidence, evidence, volatile) : undefined; - if (changes && (volatile?.size ?? 0) + changes.length <= MAX_VOLATILE) { - state.volatile = { + const pending = state.learned?.documents === pages ? state.learned.keys : undefined; + if ( + changes && + (volatile?.size ?? 0) + (pending?.size ?? 0) + changes.length <= MAX_VOLATILE + ) { + // Learned keys stay inactive until the agent observes them in a new sync. + state.learned = { documents: pages, - keys: new Set([...(volatile ?? []), ...changes.map((change) => change.key)]), + keys: new Set([...(pending ?? []), ...changes.map((change) => change.key)]), }; const [first] = changes; throw new Error( From cfee6486799e8cc1294ee8831b716f28959819f6 Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Mon, 26 Oct 2026 07:39:24 -0500 Subject: [PATCH 3/4] fix(cooperation): protect target-less actions and cover common live layouts Follow-up review fixes for #497: - Learned self-updating text is tolerated only for actions with a resolved ref or selector. A bare key press, script or scroll acts on whatever is focused, so it keeps the exact whole-page check; previously Enter on a focused form could submit after a learned price there changed. - Learned text is also tolerated around a target whose region is a
landmark or the whole page, never in the target or its contents. Forms, dialogs, lists, tables, sections, articles and positioned panels stay exact. This fixes the refusal loop for buttons placed directly in
or on pages without landmarks, including with --depends-on, where the target region is checked with the same rule and dependencies stay exact. - Clocks that mirror their time into aria-label or title are learned too. - Reword the refusal message, sync guidance and skill docs to match, add a CHANGELOG entry, reset the captureVomObservation mock between tests, and truncate quoted text by code point instead of UTF-16 unit. --- CHANGELOG.md | 8 + .../src/cooperation/controller.test.ts | 223 ++++++++++++++++++ apps/extension/src/cooperation/controller.ts | 51 ++-- .../cooperation/observation-evidence.test.ts | 42 ++++ .../src/cooperation/observation-evidence.ts | 54 ++++- .../bsk-cli/skill/references/cooperation.md | 10 +- docs/shared-browser-control.zh-CN.md | 2 +- .../skill/references/cooperation.md | 10 +- 8 files changed, 360 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc7d19e9..9c6fdd62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,14 @@ Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. DSH 0.1.5 and 0.2.x alike), and conversation switches on 0.2.x hosts keep re-triggering it ([#484](https://github.com/Tencent/BrowserSkill/issues/484)). +- Extension: shared-control actions no longer refuse forever on pages with + self-updating text such as clocks or counters. The first change is reported + with its before/after text and requires a sync; after that, text-only + changes to those nodes stop blocking actions on a ref or selector unless + they sit inside the target or its form, list, dialog or similar region. + Actions without a target, form values, modal layers and `--depends-on` + regions stay exact, and a refused context can no longer be retried + ([#495](https://github.com/Tencent/BrowserSkill/issues/495)). ## [0.3.3] - 2026-10-14 diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index 558f5e23..e3831744 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -558,8 +558,231 @@ describe("shared-tab dispatch fences", () => { }; afterEach(() => { vi.mocked(handleSnapshot).mockReset(); + vi.mocked(captureVomObservation).mockReset(); vi.mocked(resolveActionTarget).mockReset(); }); + type Spec = [id: number, parent: number | null, role: string, extra?: Record]; + // Builds a scene from compact node specs; `tick` drives the live text. + const page = (specs: (tick: number) => Spec[]) => (tick: number) => ({ + viewport: { width: 800, height: 600 }, + nodes: specs(tick).map(([id, parentId, role, extra]) => ({ + id, + parentId, + backendNodeId: id, + frameId: "frame", + tag: role, + role, + name: "", + rect: { x: 0, y: 0, w: 20, h: 20 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + ...extra, + })), + }); + // Resolves "eN" refs to backend node N. + const refs = () => + vi.mocked(resolveActionTarget).mockImplementation( + async (_c, _s, _t, params) => + ({ + backendNodeId: Number((params as { ref?: string }).ref?.replace(/^@?e/, "")), + frameId: "frame", + cdpTarget: { tabId: 7 }, + tab: { tabId: 7 }, + }) as Awaited>, + ); + const run = ( + f: Awaited>, + context: string, + action: { ref?: string; dependsOn?: string[]; key?: string }, + ) => { + f.change({ structure: f.controller.view(7)!.revision + 100 }); + return f.controller.run( + { + id: "live", + method: action.key ? "tool.press" : "tool.click", + params: { + session_id: "shared", + tab_id: 7, + context_id: context, + ...(action.ref ? { ref: action.ref } : {}), + ...(action.key ? { key: action.key } : {}), + ...(action.dependsOn ? { depends_on: action.dependsOn } : {}), + }, + }, + f.cdp, + f.abort.signal, + (runner) => runner.send(7, "Input.dispatchMouseEvent", { type: "mousePressed" }), + ) as Promise>; + }; + const changed = "The action's observed content changed; synchronize and replan"; + /** One learning refusal, then `attempts` fresh-sync actions all succeed. */ + const learnsThenActs = async ( + build: (tick: number) => VomScene, + action: { ref?: string; dependsOn?: string[] }, + attempts = 2, + ) => { + live(build); + refs(); + const f = await fixture(); + expect((await run(f, f.first.context_id, action)).message).toContain("changed by itself"); + for (let attempt = 0; attempt < attempts; attempt++) + expect(await run(f, (await f.command("sync")).context_id, action)).toEqual({}); + expect(f.native).toHaveBeenCalledTimes(attempts); + }; + /** Every fresh-sync action keeps being refused without dispatching input. */ + const keepsRefusing = async ( + build: (tick: number) => VomScene, + action: { ref?: string; dependsOn?: string[]; key?: string }, + ) => { + live(build); + refs(); + const f = await fixture(); + for (let attempt = 0, context = f.first.context_id; attempt < 3; attempt++) { + expect(await run(f, context, action)).toMatchObject({ + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + context = (await f.command("sync")).context_id; + } + expect(f.native).not.toHaveBeenCalled(); + }; + const clock = (tick: number) => `10:00:0${tick}`; + + it("tolerates a clock beside a button directly inside
", () => + learnsThenActs( + page((tick) => [ + [1, null, "main"], + [3, 1, "button", { name: "Buy" }], + [4, 1, "span", { text: clock(tick) }], + ]), + { ref: "e3" }, + )); + + it("tolerates a clock on a page without any landmark", () => + learnsThenActs( + page((tick) => [ + [1, null, "body"], + [3, 1, "button", { name: "Buy" }], + [4, 1, "span", { text: clock(tick) }], + ]), + { ref: "e3" }, + )); + + it("tolerates a clock that mirrors its time into aria-label and title", () => + learnsThenActs( + page((tick) => [ + [1, null, "body"], + [2, 1, "form"], + [3, 2, "button", { name: "Continue" }], + [ + 4, + 1, + "timer", + { text: clock(tick), attrs: { "aria-label": clock(tick), title: clock(tick) } }, + ], + ]), + { ref: "e3" }, + )); + + it("lets declared dependencies coexist with a clock in the target's
", () => + learnsThenActs( + page((tick) => [ + [1, null, "main"], + [3, 1, "button", { name: "Buy" }], + [4, 1, "span", { text: clock(tick) }], + [5, null, "form"], + [6, 5, "textbox", { name: "Quantity", value: "1" }], + ]), + { ref: "e3", dependsOn: ["e6"] }, + )); + + it("never tolerates key presses or other actions without a resolved target", () => + keepsRefusing( + page((tick) => [ + [1, null, "main"], + [3, 1, "button", { name: "Buy" }], + [4, 1, "span", { text: clock(tick) }], + ]), + { key: "Enter" }, + )); + + it("keeps a learned node exact for a later key press on its form", async () => { + // Clicking Continue in form A learns the price in form B; Enter on the + // focused form B must still require the price observed at sync. + live( + page((tick) => [ + [1, null, "main"], + [2, 1, "form", { name: "A" }], + [4, 2, "button", { name: "Continue" }], + [6, 1, "form", { name: "B" }], + [7, 6, "span", { text: `$${100 + tick}` }], + [8, 6, "button", { name: "Pay" }], + ]), + ); + refs(); + const f = await fixture(); + expect((await run(f, f.first.context_id, { ref: "e4" })).message).toContain( + "changed by itself", + ); + expect(await run(f, (await f.command("sync")).context_id, { key: "Enter" })).toMatchObject({ + message: changed, + data: { effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + }); + + it("never tolerates changes to the target or its contents", async () => { + await keepsRefusing( + page((tick) => [ + [1, null, "main"], + [3, 1, "button", { name: `Buy for $${100 + tick}` }], + [4, 1, "span", { text: clock(tick) }], + ]), + { ref: "e3" }, + ); + await keepsRefusing( + page((tick) => [ + [1, null, "main"], + [3, 1, "button", { name: "Buy" }], + [4, 3, "span", { text: `$${100 + tick}` }], + ]), + { ref: "e3" }, + ); + }); + + it("keeps a countdown inside the target's own form exact", () => + keepsRefusing( + page((tick) => [ + [1, null, "main"], + [2, 1, "form"], + [3, 2, "button", { name: "Pay" }], + [4, 2, "span", { text: `Expires in 0:0${9 - tick}` }], + ]), + { ref: "e3" }, + )); + + it("keeps a learned node exact when it shares a card with a later target", async () => { + // The price sits in an
card next to its Buy button. + live( + page((tick) => [ + [1, null, "main"], + [2, 1, "button", { name: "Refresh" }], + [5, 1, "article"], + [6, 5, "span", { text: `$${100 + tick}` }], + [7, 5, "button", { name: "Buy" }], + ]), + ); + refs(); + const f = await fixture(); + expect((await run(f, f.first.context_id, { ref: "e2" })).message).toContain( + "changed by itself", + ); + expect(await run(f, (await f.command("sync")).context_id, { ref: "e2" })).toEqual({}); + expect(await run(f, (await f.command("sync")).context_id, { ref: "e7" })).toMatchObject({ + message: changed, + }); + expect(f.native).toHaveBeenCalledOnce(); + }); it("reports a ticking clock once instead of refusing every sync and action", async () => { live((tick) => scene(`10:00:0${tick}`)); diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index dabc7efd..d2fe66ae 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -134,7 +134,8 @@ function documents(frames: Frames): string { return JSON.stringify([...frames].map(([id, frame]) => [id, frame.document])); } function quote(text: string): string { - return JSON.stringify(text.length > 40 ? `${text.slice(0, 39)}…` : text); + const points = Array.from(text); + return JSON.stringify(points.length > 40 ? `${points.slice(0, 39).join("")}…` : text); } function sameFields(a: Frames, b: Frames): boolean { return ( @@ -684,7 +685,7 @@ export class CooperationController implements ControlAuthority { }), state: { ...this.view(state.tabId), mode: "ready" }, guidance: - "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked; text seen changing by itself outside the target region (clocks, counters) is reported once and then tolerated. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", + "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector, unless it is inside the target or shares its form, list, dialog or similar region. Actions without a ref or selector always check the whole page exactly. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", }; } finally { state.working = false; @@ -1047,37 +1048,49 @@ export class CooperationController implements ControlAuthority { (revoked = "The action's observed content changed; synchronize and replan"), ); } - // Explicit dependencies stay exact. The default whole-page check tolerates - // text that was already seen changing by itself in these documents. + // Learned text is tolerated only for actions with a resolved target. A + // target-less key press or script acts on whatever is focused, so its + // region cannot be protected and the whole page stays exact. + const tolerant = actionTarget !== undefined; const pages = documents(state.frames); - const learning = !evidenceTargets && !params.depends_on?.length; const volatile = - learning && state.volatile?.documents === pages ? state.volatile.keys : undefined; - // The action's own region stays exact even when it holds learned nodes. - const targetIntact = - !actionTarget || sameObservationEvidence(context.evidence, evidence, actionTarget); - if ( - evidenceTargets - ? evidenceTargets.every((dependency) => + tolerant && state.volatile?.documents === pages ? state.volatile.keys : undefined; + // The target's own semantic region and every declared dependency stay + // exact; sameObservationEvidence only relaxes broad layout around a target. + const matches = (keys?: ReadonlySet) => { + if (evidenceTargets) { + const [own, ...dependencies] = evidenceTargets; + return ( + sameObservationEvidence(context.evidence!, evidence!, own, keys) && + dependencies.every((dependency) => sameObservationEvidence(context.evidence!, evidence!, dependency), ) - : targetIntact && - sameObservationEvidence(context.evidence!, evidence!, undefined, volatile) - ) { + ); + } + return ( + (!actionTarget || + sameObservationEvidence(context.evidence!, evidence!, actionTarget, keys)) && + sameObservationEvidence(context.evidence!, evidence!, undefined, keys) + ); + }; + if (matches(volatile)) { // Keep the pre-capture revision: later mutations must be checked again. verifiedFrames = before; return; } consumeRefused(); - // Learn page activity only from in-place text changes outside the target - // region with no human input since the sync. The agent still replans once. + // Learn page activity only from in-place text changes that every protected + // region would tolerate, with no human input since the sync. const changes = - learning && targetIntact && sameFrames(context.frames, state.frames, "human") + tolerant && sameFrames(context.frames, state.frames, "human") ? contentChanges(context.evidence, evidence, volatile) : undefined; + const candidate = + changes && new Set([...(volatile ?? []), ...changes.map((change) => change.key)]); const pending = state.learned?.documents === pages ? state.learned.keys : undefined; if ( changes && + matches(candidate) && (volatile?.size ?? 0) + (pending?.size ?? 0) + changes.length <= MAX_VOLATILE ) { // Learned keys stay inactive until the agent observes them in a new sync. @@ -1087,7 +1100,7 @@ export class CooperationController implements ControlAuthority { }; const [first] = changes; throw new Error( - (revoked = `Page content outside the action's region changed by itself (${quote(first.before)} → ${quote(first.after)}${changes.length > 1 ? ` and ${changes.length - 1} more` : ""}); synchronize and replan. Later text-only changes there will not block actions without --depends-on; declare them as dependencies if the decision relies on them`), + (revoked = `Page text outside the action's own region changed by itself (${quote(first.before)} → ${quote(first.after)}${changes.length > 1 ? ` and ${changes.length - 1} more` : ""}); synchronize and replan. After that sync, further text-only changes there stop blocking actions on other elements unless they share a form, list, dialog or similar region; if the decision relies on that text, declare it with --depends-on to keep it exact`), ); } throw new Error((revoked = "The action's observed content changed; synchronize and replan")); diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index 4049336d..d367cf50 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -114,3 +114,45 @@ it("never treats form values or modal content as page activity", () => { ); } }); + +it("reports aria-label and title rewrites as content changes", () => { + const before = [ + node(1, null, "main"), + { ...node(2, 1, "timer"), attrs: { "aria-label": "10:00" } }, + ]; + const after = structuredClone(before); + after[1].attrs = { "aria-label": "10:01" }; + expect(contentChanges({ nodes: before }, { nodes: after })).toEqual([ + { key: "frame:2", before: "10:00", after: "10:01" }, + ]); + const volatile = new Set(["frame:2"]); + expect(sameObservationEvidence({ nodes: before }, { nodes: after }, undefined, volatile)).toBe( + true, + ); +}); + +it("relaxes learned text only around targets in
or without a region", () => { + const volatile = new Set(["frame:4"]); + const check = (parent: string, sibling = true) => { + const before = [ + node(1, null, "body"), + node(2, 1, parent), + node(3, 2, "button", "Buy"), + { ...node(4, sibling ? 2 : 3, "span"), text: "1" }, + ]; + const after = structuredClone(before); + after[3].text = "2"; + return sameObservationEvidence( + { nodes: before }, + { nodes: after }, + { backendNodeId: 3 }, + volatile, + ); + }; + expect(check("main")).toBe(true); + expect(check("div")).toBe(true); + for (const region of ["form", "dialog", "list", "table", "section", "article", "region"]) + expect(check(region)).toBe(false); + // Text inside the target itself is never relaxed. + expect(check("main", false)).toBe(false); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index 4a63b154..c041ee0b 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -51,6 +51,16 @@ function evidenceKey(node: VomNode): string | undefined { // Form state and modal layers are never tolerated as page activity. const protectedState = (node: VomNode) => node.value !== undefined || node.checked !== undefined || !!node.modal; +// Text-bearing facts that self-updating widgets rewrite (clocks often mirror +// their text into aria-label/title for assistive technology). +const contentAttributes = new Set(["aria-label", "title"]); +const content = (node: VomNode) => [ + node.text, + node.name, + node.attrs?.["aria-label"], + node.attrs?.title, +]; +const describe = (node: VomNode) => content(node).find((value) => value) ?? ""; function signature( evidence: ObservationEvidence, @@ -91,10 +101,28 @@ function signature( } return false; }; - // Only the whole-page default check tolerates text that changed by itself; - // identity, structure, state and visibility of those nodes remain checked. + // Semantic decision units (forms, dialogs, lists, cards, popovers) stay exact. + // Only page layout around the target, a
landmark or a page without + // any enclosing region, may contain learned self-updating text. + const broad = !scope || scope.role === "main" || scope.tag === "main"; + const withinAnchor = (node: VomNode) => { + const seen = new Set(); + for ( + let current: VomNode | undefined = node; + current && !seen.has(current.id); + current = parent(current) + ) { + if (current === anchor) return true; + seen.add(current.id); + } + return false; + }; + // Identity, structure, state and visibility of tolerated nodes remain checked. + // The target itself and its contents are never tolerated. const tolerated = (node: VomNode) => - !target && !protectedState(node) && volatile?.has(evidenceKey(node) ?? "") === true; + volatile?.has(evidenceKey(node) ?? "") === true && + !protectedState(node) && + (!target || (broad && !withinAnchor(node))); const facts = (node: VomNode, tolerate = false) => [ node.frameId, node.backendNodeId, @@ -109,7 +137,9 @@ function signature( node.modal, !!node.rect && node.rect.w > 0 && node.rect.h > 0, node.pointerEvents === "none", - semanticAttributes.map((key) => node.attrs?.[key]), + semanticAttributes.map((key) => + tolerate && contentAttributes.has(key) ? undefined : node.attrs?.[key], + ), ]; return JSON.stringify([ scope && [scope.frameId, scope.backendNodeId], @@ -123,8 +153,12 @@ function signature( } /** - * `volatile` applies only to the whole-page check (no target): text of those - * nodes may differ. Explicit dependencies and target regions stay exact. + * `volatile` lists nodes whose text (name, text, aria-label, title) may differ. + * Without a target it applies to the whole page. With a target it applies only + * when the target's region is a
landmark or the whole page, and never to + * the target or its contents. Forms, dialogs, lists, tables, menus, sections, + * articles and positioned panels around a target stay exact, as do form values, + * checked states and modal layers everywhere. */ export function sameObservationEvidence( before: ObservationEvidence, @@ -153,12 +187,8 @@ export function contentChanges( const key = evidenceKey(node); const old = key === undefined ? undefined : previous.get(key); if (!key || !old || known.has(key) || protectedState(node) || protectedState(old)) continue; - if (old.name !== node.name || old.text !== node.text) - changes.push({ - key, - before: old.text ?? old.name ?? "", - after: node.text ?? node.name ?? "", - }); + if (JSON.stringify(content(old)) !== JSON.stringify(content(node))) + changes.push({ key, before: describe(old), after: describe(node) }); } const volatile = new Set([...known, ...changes.map((change) => change.key)]); return changes.length && sameObservationEvidence(before, after, undefined, volatile) diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index bc2357aa..83694b81 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -36,10 +36,12 @@ The default guard checks the whole observed page. When the decision depends on k CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every region used in the decision: for example both the options list and the current form. The target's form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Text outside the target's region that changes -by itself between sync and action (a clock, a counter) is reported once with the before/after -text; after you sync and replan, the default check tolerates further text-only changes there. -If the decision relies on such text, declare it with `--depends-on`, which stays exact. +uncertain, omit it and retain whole-page checking. Text that changes by itself between sync and action +(a clock, a counter) is reported once with the before/after text. After you sync and replan, +further text-only changes there no longer block actions on a ref or selector, unless the text +is inside the target or shares its form, list, dialog, card or similar region. Actions without +a ref or selector, such as a bare key press, always check the whole page exactly. If the +decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and discrete controls return control automatically; text editing, composition, held keys and dragging keep the person's lease until they finish. diff --git a/docs/shared-browser-control.zh-CN.md b/docs/shared-browser-control.zh-CN.md index b339a67f..701ebb66 100644 --- a/docs/shared-browser-control.zh-CN.md +++ b/docs/shared-browser-control.zh-CN.md @@ -50,7 +50,7 @@ DSH 使用 `bsk_session` 的 `sync`、`wait-user`、`cooperation-status`。旧 ` 会话生命周期复用 `session.activity` 事件。Daemon 只接受所属浏览器的状态更新，将协作保持与最后工具调用时间一起保存在既有活动记录中；结束保持后有完整的空闲间隔。释放、断线和 session id 复用均清理这一状态。 -默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。目标区域以外、在同步与动作之间自行变化的文字（如时钟、计数器）会先被报告一次并附带前后文本；Agent 重新同步和规划后，默认整页校验不再因这些节点的纯文本变化而拒绝。结构、可见性、表单值和模态变化仍会拒绝；若决策依赖这些文字，用 `--depends-on` 声明，显式依赖始终精确校验。 +默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。在同步与动作之间自行变化的文字（如时钟、计数器）会先被报告一次并附带前后文本；Agent 重新同步和规划后，这些节点的纯文本变化不再阻断带 ref 或 selector 的动作，除非它位于目标内部，或与目标同属一个表单、列表、对话框、卡片等区域。没有 ref 或 selector 的动作（如直接按键）始终精确校验整页。结构、可见性、表单值和模态变化仍会拒绝；若决策依赖这些文字，用 `--depends-on` 声明，显式依赖始终精确校验。 `cooperation-capture.ts` 在隔离世界采集输入和 DOM 变化。输入保持即时通知，DOM 变更合并后通知；主文档及 iframe 都必须响应。普通字段变化走增量路径，字段以文档内稳定标识区分，同名字段和开放 Shadow DOM 输入框也分别记录。结构变化、文档替换、字段过多或内容截断时，复用现有 VOM 快照刷新基线。增量字段变更同步后会更新内部语义证据，避免后续动作仍拿旧值比较；不重复输出整页文本，不替换引用表。采集不触发 hover。密码、文件选择不进入字段变化流。 diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index c694d587..e120ba0a 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -30,10 +30,12 @@ The default guard checks the whole observed page. When the decision depends on k CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every region used in the decision: for example both the options list and the current form. The target's form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Text outside the target's region that changes -by itself between sync and action (a clock, a counter) is reported once with the before/after -text; after you sync and replan, the default check tolerates further text-only changes there. -If the decision relies on such text, declare it with `--depends-on`, which stays exact. +uncertain, omit it and retain whole-page checking. Text that changes by itself between sync and action +(a clock, a counter) is reported once with the before/after text. After you sync and replan, +further text-only changes there no longer block actions on a ref or selector, unless the text +is inside the target or shares its form, list, dialog, card or similar region. Actions without +a ref or selector, such as a bare key press, always check the whole page exactly. If the +decision relies on such text, declare it with `--depends-on`, which stays exact. Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and discrete controls return control automatically; text editing, composition, held keys and dragging keep the person's lease until they finish. From 6b277f0a734f9a2c30687e541c7b7e1220e6427b Mon Sep 17 00:00:00 2001 From: NianJiuZst <3235467914@qq.com> Date: Mon, 26 Oct 2026 08:58:10 -0500 Subject: [PATCH 4/4] fix(cooperation): learn real Chrome text nodes and keep target panels exact Address review on #497, verified with real Chrome observations: - Plain text clocks are learned again. Chrome adds an InlineTextBox with no backend node under each StaticText; such virtual text now belongs to its nearest backed ancestor instead of blocking learning. - Container names are derived from descendant text, so they are never learned themselves. They are compared with learned text removed and stay exact otherwise, which keeps aria-hidden content such as a price that exists only in a container's name protected. - Inside
or a page without regions, the target's own branch stays exact: the child of the first layout node that is not a single-child wrapper, never descending into a positioned panel. A price in the target's fixed panel or card can no longer change unnoticed. - A selector target missing from the observation (such as inside aria-hidden content) keeps the exact whole-page check, as before this PR, instead of being refused after any cosmetic change. - The refusal reports one clock once, not once per node. Add a real-Chrome regression suite that drives the controller with production VOM capture, target resolution and native input, and run it in the browser input CI job. Its plain clock, positioned panel and outside-observation cases fail on cfee648. --- .github/workflows/ci.yml | 6 + CHANGELOG.md | 10 +- .../src/cooperation/controller.test.ts | 15 + apps/extension/src/cooperation/controller.ts | 29 +- .../src/cooperation/live-text.browser.test.ts | 388 ++++++++++++++++++ .../cooperation/observation-evidence.test.ts | 144 ++++++- .../src/cooperation/observation-evidence.ts | 361 +++++++++++----- .../bsk-cli/skill/references/cooperation.md | 24 +- docs/shared-browser-control.zh-CN.md | 2 +- .../skill/references/cooperation.md | 24 +- 10 files changed, 864 insertions(+), 139 deletions(-) create mode 100644 apps/extension/src/cooperation/live-text.browser.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53bb828b..f7391dfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,12 @@ jobs: BSK_CLICK_CHROME: google-chrome run: pnpm exec vitest run src/tools/__tests__/waits.browser.test.ts + - name: Run shared-control live text browser regression + working-directory: apps/extension + env: + BSK_CLICK_CHROME: google-chrome + run: pnpm exec vitest run src/cooperation/live-text.browser.test.ts + - name: Run emulation-lifecycle browser regression working-directory: apps/extension env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c6fdd62..1633424b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,11 +23,11 @@ Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. ([#484](https://github.com/Tencent/BrowserSkill/issues/484)). - Extension: shared-control actions no longer refuse forever on pages with self-updating text such as clocks or counters. The first change is reported - with its before/after text and requires a sync; after that, text-only - changes to those nodes stop blocking actions on a ref or selector unless - they sit inside the target or its form, list, dialog or similar region. - Actions without a target, form values, modal layers and `--depends-on` - regions stay exact, and a refused context can no longer be retried + with its before/after text and requires a sync; after that, changes to that + text stop blocking actions on a ref or selector unless it shares a form, + list, dialog, panel or other container with the target. Actions without a + target, form values, modal layers and `--depends-on` regions stay exact, + and a refused context can no longer be retried ([#495](https://github.com/Tencent/BrowserSkill/issues/495)). ## [0.3.3] - 2026-10-14 diff --git a/apps/extension/src/cooperation/controller.test.ts b/apps/extension/src/cooperation/controller.test.ts index e3831744..335d748a 100644 --- a/apps/extension/src/cooperation/controller.test.ts +++ b/apps/extension/src/cooperation/controller.test.ts @@ -696,6 +696,21 @@ describe("shared-tab dispatch fences", () => { { ref: "e3", dependsOn: ["e6"] }, )); + it("never learns text that changed alongside human input", async () => { + live((tick) => scene(`10:00:0${tick}`)); + const f = await fixture(); + // The person acts on the page while the clock ticks. + f.change({ human: 1, structure: 50 }); + expect(await act(f, f.first.context_id)).toMatchObject({ + data: { reason: "cooperation_sync_required", effect_state: "none" }, + }); + expect(f.native).not.toHaveBeenCalled(); + // Nothing was learned: the next tick is reported as new page activity. + const synced = await f.command("sync"); + expect((await cycle(f, synced.context_id)).message).toContain("changed by itself"); + expect(f.native).not.toHaveBeenCalled(); + }); + it("never tolerates key presses or other actions without a resolved target", () => keepsRefusing( page((tick) => [ diff --git a/apps/extension/src/cooperation/controller.ts b/apps/extension/src/cooperation/controller.ts index d2fe66ae..4366b7a8 100644 --- a/apps/extension/src/cooperation/controller.ts +++ b/apps/extension/src/cooperation/controller.ts @@ -14,6 +14,7 @@ import { contentChanges, type ObservationEvidence, observationEvidence, + observesTarget, sameObservationEvidence, } from "./observation-evidence"; import { pointerTargetValidator } from "./pointer-target"; @@ -685,7 +686,7 @@ export class CooperationController implements ControlAuthority { }), state: { ...this.view(state.tabId), mode: "ready" }, guidance: - "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector, unless it is inside the target or shares its form, list, dialog or similar region. Actions without a ref or selector always check the whole page exactly. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", + "Continue toward the assigned goal using this current state. Incorporate helpful progress and correct recoverable mistakes. Page activity is not a change of goal. Each context authorizes one action; sync after it. By default the whole page is checked. Text that changes by itself (clocks, counters) is reported once; after the next sync it no longer blocks actions on a ref or selector unless it shares a form, list, dialog, panel or other container with the target. Actions without a ref or selector always check the whole page exactly. To tolerate other unrelated updates, declare all observed decision regions with --depends-on refs/selectors (plugin: dependsOn); the target region and form values remain protected. Verify completion and prior submissions from application evidence.", }; } finally { state.working = false; @@ -1048,10 +1049,16 @@ export class CooperationController implements ControlAuthority { (revoked = "The action's observed content changed; synchronize and replan"), ); } - // Learned text is tolerated only for actions with a resolved target. A - // target-less key press or script acts on whatever is focused, so its - // region cannot be protected and the whole page stays exact. - const tolerant = actionTarget !== undefined; + // Learned text is tolerated only around a resolved target present in both + // observations, whose region can be protected. A target-less key press or + // script acts on whatever is focused, and a selector may resolve outside + // the observation (for example inside aria-hidden content); both keep the + // exact whole-page check. + const anchored = + actionTarget !== undefined && + observesTarget(context.evidence, actionTarget) && + observesTarget(evidence, actionTarget); + const tolerant = anchored; const pages = documents(state.frames); const volatile = tolerant && state.volatile?.documents === pages ? state.volatile.keys : undefined; @@ -1068,7 +1075,7 @@ export class CooperationController implements ControlAuthority { ); } return ( - (!actionTarget || + (!anchored || sameObservationEvidence(context.evidence!, evidence!, actionTarget, keys)) && sameObservationEvidence(context.evidence!, evidence!, undefined, keys) ); @@ -1098,9 +1105,15 @@ export class CooperationController implements ControlAuthority { documents: pages, keys: new Set([...(pending ?? []), ...changes.map((change) => change.key)]), }; - const [first] = changes; + // One clock is several nodes (element, text, line box); report it once. + const reported = [ + ...new Map( + changes.map((change) => [`${change.before}\u0000${change.after}`, change]), + ).values(), + ]; + const [first] = reported; throw new Error( - (revoked = `Page text outside the action's own region changed by itself (${quote(first.before)} → ${quote(first.after)}${changes.length > 1 ? ` and ${changes.length - 1} more` : ""}); synchronize and replan. After that sync, further text-only changes there stop blocking actions on other elements unless they share a form, list, dialog or similar region; if the decision relies on that text, declare it with --depends-on to keep it exact`), + (revoked = `Page text changed by itself (${quote(first.before)} → ${quote(first.after)}${reported.length > 1 ? ` and ${reported.length - 1} more` : ""}); synchronize and replan. After that sync, further changes to that text no longer block actions on a ref or selector elsewhere. They still block actions whose target shares a form, list, dialog, panel or other container with it, and actions without a ref or selector. If the decision relies on that text, declare it with --depends-on to keep it exact`), ); } throw new Error((revoked = "The action's observed content changed; synchronize and replan")); diff --git a/apps/extension/src/cooperation/live-text.browser.test.ts b/apps/extension/src/cooperation/live-text.browser.test.ts new file mode 100644 index 00000000..3538a90e --- /dev/null +++ b/apps/extension/src/cooperation/live-text.browser.test.ts @@ -0,0 +1,388 @@ +// @vitest-environment node +// Opt in with BSK_CLICK_CHROME; each test owns its browser, profile and HTTP server. +// Real Chrome observations, target resolution and native input run through the +// production controller. The page reports DOM revisions like the content +// script's collector, so this is integration rather than installed-extension +// end-to-end coverage. +import { createServer } from "node:http"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; +import { SessionManager } from "@/session-manager/manager"; +import { handleClick, handleFill, handlePress } from "@/tools/interaction"; +import type { CdpRunner } from "@/tools/shared"; +import { CooperationController } from "./controller"; +import type { FrameState } from "./types"; + +type Send = >( + method: string, + params?: object, + sessionId?: string, +) => Promise; +type Reply = Record; + +// Counts DOM mutations like cooperation-capture.ts and records trusted input. +const collector = ``; + +const pages: Record = { + // Continue in form A; a plain clock in form B. + clock: `
+
10:00:00
`, + // Pay shares a fixed panel with its price inside
. + panel: `
+
+ +
`, + "visible-panel": `
+
+ $100 +
`, + // A selector can target an element the observation leaves out. + // It is not focusable: Chrome drops aria-hidden around a focused element. + hidden: `
+ + 10:00:00
`, + // A search form beside a clock in another form. + search: `
+ +
+
10:00:00
`, + // A landmark-free app: clock and menu in the header, a priced card below. + app: `
+
10:00:00
+
$100 +
`, +}; + +async function withCooperation( + run: (page: { + open(name: string): Promise; + set(selector: string, text: string): Promise; + mutate(expression: string): Promise; + sync(): Promise; + act(method: string, params: object): Promise; + click(context: string, selector: string): Promise; + press(context: string, key: string): Promise; + inputs(): Promise<{ clicks: string[]; keys: string[] }>; + value(selector: string): Promise; + }) => Promise, +) { + const server = createServer((request, response) => { + const name = decodeURIComponent((request.url ?? "/").slice(1)); + response.setHeader("Content-Type", "text/html"); + response.end(`${name}${pages[name] ?? ""}${collector}`); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const origin = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const { withChrome } = await import( + new URL( + "../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs", + import.meta.url, + ).href + ); + try { + await withChrome( + { executable: process.env.BSK_CLICK_CHROME, deviceScale: 1, zoom: 1, startupTimeout: 30_000 }, + async (send: Send) => { + const { targetId } = await send<{ targetId: string }>("Target.createTarget", { + url: "about:blank", + }); + const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", { + targetId, + flatten: true, + }); + const local: Send = (method, params) => send(method, params, sessionId); + const evaluate = async (expression: string) => { + const reply = await local<{ result: { value: T }; exceptionDetails?: unknown }>( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + ); + expect(reply.exceptionDetails).toBeUndefined(); + return reply.result.value; + }; + await local("Page.enable"); + await local("Page.bringToFront"); + const graph = async (): Promise => { + const { frameTree } = await local<{ frameTree: { frame: { id: string } } }>( + "Page.getFrameTree", + ); + const root: CdpFrame = { frameId: frameTree.frame.id, target: { tabId: 4 } }; + return { rootFrameId: frameTree.frame.id, frames: [root] }; + }; + const cdp: CdpRunner = { + send: (_tabId, method, params) => local(method, params), + sendToTarget: (_target, method, params) => local(method, params), + getAttachmentId: () => sessionId, + getFrameGraph: graph, + }; + const sessions = new SessionManager({ + agentWindow: { + create: async () => ({ windowId: 100, initialTabIds: [4] }), + remove: async () => {}, + ensureActiveTab: async () => 4, + }, + }); + await sessions.start("shared"); + const tab = { id: 4, windowId: 100, active: true, url: origin } as chrome.tabs.Tab; + const tabsApi = { get: async () => tab, query: async () => [tab] }; + vi.stubGlobal("chrome", { tabs: tabsApi }); + const frame = async (): Promise => { + const state = await evaluate<{ document: string; revision: number }>("window.coop"); + return { + document: state.document, + revision: state.revision, + structure: state.revision, + human: 0, + busy: false, + editing: false, + lastInput: 0, + fields: [], + fieldsComplete: true, + events: [], + }; + }; + const controller = new CooperationController({ + sessions, + readFrames: async () => new Map([[0, await frame()]]), + resumeFrames: async () => {}, + changed: () => {}, + }); + const signal = new AbortController().signal; + await controller.run( + { id: "start", method: "tool.session_start", params: { session_id: "shared" } }, + cdp, + signal, + async () => ({ agent_window_id: 100 }), + ); + const handlers: Record = { + "tool.click": handleClick, + "tool.press": handlePress as never, + "tool.fill": handleFill as never, + }; + const act = (method: string, params: object) => + controller.run( + { id: method, method, params: { session_id: "shared", tab_id: 4, ...params } }, + cdp, + signal, + (runner, operationSignal) => + handlers[method](sessions, { session_id: "shared", tab_id: 4, ...params } as never, { + cdp: runner as CdpRunner, + tabsApi, + signal: operationSignal as AbortSignal, + }), + ) as Promise; + await run({ + async open(name) { + tab.url = `${origin}/${name}`; + await local("Page.navigate", { url: tab.url }); + await expect + .poll(() => + evaluate("document.title === " + JSON.stringify(name) + " && !!window.coop"), + ) + .toBe(true); + }, + async set(selector, text) { + await this.mutate( + `document.querySelector(${JSON.stringify(selector)}).firstChild.nodeValue = ${JSON.stringify(text)}`, + ); + }, + async mutate(expression) { + const before = (await frame()).revision; + await evaluate(expression); + // The action check only re-observes after the collector reports a mutation. + await expect.poll(async () => (await frame()).revision > before).toBe(true); + }, + sync: () => + controller.command( + { session_id: "shared", tab_id: 4, action: "sync", timeout_ms: 0 }, + cdp, + signal, + ) as Promise, + act, + click: (context, selector) => act("tool.click", { context_id: context, selector }), + press: (context, key) => act("tool.press", { context_id: context, key }), + inputs: () => evaluate("({ clicks, keys })"), + value: (selector) => + evaluate(`document.querySelector(${JSON.stringify(selector)}).value`), + }); + }, + ); + } finally { + server.close(); + } +} + +const learned = expect.stringContaining("changed by itself"); +const changed = "The action's observed content changed; synchronize and replan"; +const refused = { data: { reason: "cooperation_sync_required", effect_state: "none" } }; + +describe.skipIf(!process.env.BSK_CLICK_CHROME)("shared control with self-updating text", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("reports a plain text clock once, then stops refusing every action", async () => { + await withCooperation(async (page) => { + await page.open("clock"); + const first = await page.sync(); + await page.set("#clock", "10:00:01"); + const replies = [await page.click(first.context_id, "#continue")]; + for (let tick = 2; tick <= 4; tick++) { + const synced = await page.sync(); + await page.set("#clock", `10:00:0${tick}`); + replies.push(await page.click(synced.context_id, "#continue")); + } + // Previously every cycle was refused and no click ever arrived. + expect((await page.inputs()).clicks).toEqual([ + "continue:true", + "continue:true", + "continue:true", + ]); + expect(replies[0]).toMatchObject({ + ...refused, + message: expect.stringContaining('"10:00:00" → "10:00:01"'), + }); + for (const reply of replies.slice(1)) expect(reply).not.toHaveProperty("code"); + }); + }, 120_000); + + it("does not let a refused context retry with newly learned text", async () => { + await withCooperation(async (page) => { + await page.open("clock"); + const first = await page.sync(); + await page.set("#clock", "10:00:01"); + expect(await page.click(first.context_id, "#continue")).toMatchObject({ message: learned }); + expect(await page.click(first.context_id, "#continue")).toMatchObject({ + ...refused, + message: "This action requires the context returned by the latest cooperate sync", + }); + expect((await page.inputs()).clicks).toEqual([]); + }); + }, 120_000); + + it("never relaxes key presses without a target", async () => { + await withCooperation(async (page) => { + await page.open("clock"); + await page.set("#clock", "10:00:01"); + const first = await page.sync(); + await page.set("#clock", "10:00:02"); + expect(await page.click(first.context_id, "#continue")).toMatchObject({ message: learned }); + const synced = await page.sync(); + await page.set("#clock", "10:00:03"); + expect(await page.press(synced.context_id, "Enter")).toMatchObject({ + ...refused, + message: changed, + }); + expect(await page.inputs()).toEqual({ clicks: [], keys: [] }); + }); + }, 120_000); + + it.each([ + "panel", + "visible-panel", + ])("keeps a price in the target's positioned panel exact (%s)", async (name) => { + await withCooperation(async (page) => { + await page.open(name); + const first = await page.sync(); + await page.set("#price", "$200"); + const elsewhere = await page.click(first.context_id, "#continue"); + const synced = await page.sync(); + await page.set("#price", "$300"); + const paid = await page.click(synced.context_id, "#pay"); + expect((await page.inputs()).clicks).toEqual([]); + expect(paid).toMatchObject({ ...refused, message: changed }); + // Acting elsewhere may learn the visible price, never the aria-hidden one. + expect(elsewhere).toMatchObject({ + ...refused, + message: name === "panel" ? changed : learned, + }); + }); + }, 120_000); + + it("keeps declared dependencies working while unrelated text ticks", async () => { + await withCooperation(async (page) => { + await page.open("clock"); + for (let tick = 1; tick <= 2; tick++) { + const context = (await page.sync()).context_id; + await page.set("#clock", `10:00:0${tick}`); + expect( + await page.act("tool.click", { + context_id: context, + selector: "#continue", + depends_on: ["#continue"], + }), + ).not.toHaveProperty("code"); + } + expect((await page.inputs()).clicks).toEqual(["continue:true", "continue:true"]); + }); + }, 120_000); + + it("tolerates learned text for fills and key presses on a target", async () => { + await withCooperation(async (page) => { + await page.open("search"); + let context = (await page.sync()).context_id; + await page.set("#clock", "10:00:01"); + const fill = (text: string) => + page.act("tool.fill", { context_id: context, selector: "#query", value: text }); + expect(await fill("ignored")).toMatchObject({ ...refused, message: learned }); + context = (await page.sync()).context_id; + await page.set("#clock", "10:00:02"); + expect(await fill("weather")).not.toHaveProperty("code"); + expect(await page.value("#query")).toBe("weather"); + context = (await page.sync()).context_id; + await page.set("#clock", "10:00:03"); + // A press result reports the pressed key's code, so match its shape. + expect( + await page.act("tool.press", { context_id: context, selector: "#query", key: "Enter" }), + ).toMatchObject({ key: "Enter", code: "Enter" }); + expect((await page.inputs()).keys).toContain("Enter:true"); + }); + }, 120_000); + + it("keeps the exact whole-page check for a target outside the observation", async () => { + await withCooperation(async (page) => { + await page.open("hidden"); + // A cosmetic change passes the whole-page check, as before this change. + let context = (await page.sync()).context_id; + await page.mutate("document.querySelector('#styled').className = 'hovered'"); + expect(await page.click(context, "#hidden")).not.toHaveProperty("code"); + // Without an observed region to protect, self-updating text is never learned. + for (let tick = 1; tick <= 2; tick++) { + context = (await page.sync()).context_id; + await page.set("#clock", `10:00:0${tick}`); + expect(await page.click(context, "#hidden")).toMatchObject({ + ...refused, + message: changed, + }); + } + expect((await page.inputs()).clicks).toEqual(["hidden:true"]); + }); + }, 120_000); + + it("tolerates a header clock in a landmark-free app but keeps a card's price exact", async () => { + await withCooperation(async (page) => { + await page.open("app"); + let context = (await page.sync()).context_id; + await page.set("#clock", "10:00:01"); + expect(await page.click(context, "#buy")).toMatchObject({ message: learned }); + context = (await page.sync()).context_id; + await page.set("#clock", "10:00:02"); + expect(await page.click(context, "#buy")).not.toHaveProperty("code"); + // A price learned while acting in the header still protects its own card. + context = (await page.sync()).context_id; + await page.set("#price", "$200"); + expect(await page.click(context, "#menu")).toMatchObject({ message: learned }); + context = (await page.sync()).context_id; + await page.set("#price", "$300"); + expect(await page.click(context, "#buy")).toMatchObject({ ...refused, message: changed }); + expect((await page.inputs()).clicks).toEqual(["buy:true"]); + }); + }, 120_000); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.test.ts b/apps/extension/src/cooperation/observation-evidence.test.ts index d367cf50..00b496ec 100644 --- a/apps/extension/src/cooperation/observation-evidence.test.ts +++ b/apps/extension/src/cooperation/observation-evidence.test.ts @@ -1,5 +1,5 @@ import type { VomNode } from "@browser-skill/vom"; -import { expect, it } from "vitest"; +import { describe, expect, it } from "vitest"; import { contentChanges, sameObservationEvidence } from "./observation-evidence"; const node = (id: number, parentId: number | null, role: string, name = ""): VomNode => ({ @@ -156,3 +156,145 @@ it("relaxes learned text only around targets in
or without a region", () // Text inside the target itself is never relaxed. expect(check("main", false)).toBe(false); }); + +// Real Chrome observations add text nodes and derive container names from +// descendant text: a tick changes every ancestor's name, a StaticText node and +// an InlineTextBox without a backend node. +const page = (clock: string, price = "$100", hidden = false): VomNode[] => [ + { ...node(1, null, "main"), name: hidden ? clock : `${clock} ${price}` }, + { ...node(2, 1, "span"), name: clock, text: clock }, + { ...node(3, 2, "StaticText"), name: clock }, + { ...node(4, 3, "InlineTextBox"), name: clock, backendNodeId: undefined }, + { ...node(5, 1, "generic"), name: price }, + ...(hidden + ? [] + : [ + { ...node(6, 5, "span"), name: price, text: price }, + { ...node(7, 6, "StaticText"), name: price }, + ]), + node(8, 1, "button", "Buy"), +]; + +it("learns virtual text through the node that owns it, never derived names", () => { + const before = { nodes: page("10:00:00") }; + const after = { nodes: page("10:00:01") }; + expect(contentChanges(before, after)?.map((change) => change.key)).toEqual([ + "frame:2", + "frame:3", + ]); + expect(sameObservationEvidence(before, after, undefined, new Set(["frame:2", "frame:3"]))).toBe( + true, + ); + // The InlineTextBox and derived names stay exact without their owner. + expect(sameObservationEvidence(before, after, undefined, new Set(["frame:2"]))).toBe(false); +}); + +it("keeps content without its own node exact inside derived names", () => { + // An aria-hidden price exists only in its ancestors' derived names. + expect( + contentChanges({ nodes: page("1", "$100", true) }, { nodes: page("1", "$200", true) }), + ).toBe(undefined); + const learned = new Set(["frame:2", "frame:3"]); + const before = { nodes: page("10:00:00", "$100", true) }; + // The learned clock ticks while the hidden price changes in the same container. + const after = { nodes: page("10:00:01", "$100", true) }; + after.nodes[0].name = "10:00:01 $200"; + before.nodes[0].name = "10:00:00 $100"; + expect(sameObservationEvidence(before, after, undefined, learned)).toBe(false); +}); + +it("learns the same value shown twice", () => { + const twins = (clock: string): VomNode[] => [ + { ...node(1, null, "main"), name: `${clock} ${clock}` }, + { ...node(2, 1, "span"), name: clock, text: clock }, + { ...node(3, 1, "span"), name: clock, text: clock }, + node(4, 1, "button", "Go"), + ]; + const changes = contentChanges({ nodes: twins("1") }, { nodes: twins("2") }); + expect(changes?.map((change) => change.key)).toEqual(["frame:2", "frame:3"]); + // A different value elsewhere in the derived name is not learned text. + const after = twins("2"); + after[0].name = "2 3"; + expect( + sameObservationEvidence( + { nodes: twins("1") }, + { nodes: after }, + undefined, + new Set(["frame:2"]), + ), + ).toBe(false); +}); + +describe("learned text around a target inside
or without a region", () => { + const shape = ( + specs: [id: number, parent: number | null, role: string, extra?: Partial][], + ): VomNode[] => + // Like Chrome, an element's name includes its own text. + specs.map(([id, parent, role, extra]) => ({ + ...node(id, parent, role, extra?.text), + ...extra, + })); + const tick = (nodes: VomNode[], id: number, text: string) => { + const next = structuredClone(nodes); + const changed = next.find((candidate) => candidate.id === id)!; + changed.text = text; + changed.name = text; + return next; + }; + const allowed = (nodes: VomNode[], target: number, learned: number) => + sameObservationEvidence( + { nodes }, + { nodes: tick(nodes, learned, "changed") }, + { backendNodeId: target, frameId: "frame" }, + new Set([`frame:${learned}`]), + ); + + it("keeps a positioned panel exact", () => { + const nodes = shape([ + [1, null, "main"], + [2, 1, "form"], + [3, 2, "button", { name: "Continue" }], + [4, 1, "generic", { position: "fixed" }], + [5, 4, "span", { text: "$100" }], + [6, 4, "button", { name: "Pay" }], + ]); + expect(allowed(nodes, 6, 5)).toBe(false); + // The same panel nested under a single-child wrapper stays exact as well. + const wrapped = shape([ + [1, null, "main"], + [7, 1, "generic"], + [4, 7, "generic", { position: "absolute" }], + [5, 4, "span", { text: "$100" }], + [6, 4, "button", { name: "Pay" }], + ]); + expect(allowed(wrapped, 6, 5)).toBe(false); + }); + + it("keeps the target's own branch exact and tolerates other branches", () => { + const nodes = shape([ + [1, null, "RootWebArea"], + [2, 1, "html"], + [3, 2, "body"], + [4, 3, "generic"], + [5, 4, "banner"], + [6, 5, "span", { text: "10:00:00" }], + [7, 4, "group"], + [8, 7, "generic"], + [9, 8, "span", { text: "$100" }], + [10, 8, "button", { name: "Buy" }], + [11, 7, "button", { name: "Refresh" }], + ]); + // Single-child wrappers collapse, so the header is a separate branch. + expect(allowed(nodes, 10, 6)).toBe(true); + // The card's price shares the content branch with Buy and Refresh. + expect(allowed(nodes, 10, 9)).toBe(false); + expect(allowed(nodes, 11, 9)).toBe(false); + // A target beside its learned text directly inside
is its own branch. + const flat = shape([ + [1, null, "main"], + [2, 1, "button", { name: "Go" }], + [3, 1, "span", { text: "10:00:00" }], + ]); + expect(allowed(flat, 2, 3)).toBe(true); + }); +}); diff --git a/apps/extension/src/cooperation/observation-evidence.ts b/apps/extension/src/cooperation/observation-evidence.ts index c041ee0b..0e3a7a15 100644 --- a/apps/extension/src/cooperation/observation-evidence.ts +++ b/apps/extension/src/cooperation/observation-evidence.ts @@ -11,6 +11,13 @@ export interface EvidenceTarget { frameId?: string; } +/** A text change of one persistent node between two observations. */ +export interface ContentChange { + key: string; + before: string; + after: string; +} + const scopes = new Set(["form", "dialog", "table", "list", "grid", "listbox", "menu"]); const regions = new Set(["main", "region", "section", "article"]); const semanticAttributes = [ @@ -29,38 +36,140 @@ const semanticAttributes = [ "readonly", "required", ]; - -/** A text change of one persistent node between two observations. */ -export interface ContentChange { - key: string; - before: string; - after: string; -} +// Accessibility text nodes state their own text as their name. Other names may +// be derived from descendant text, so they are never learned directly. +const textRoles = new Set(["StaticText", "InlineTextBox", "ListMarker"]); +// Attributes that self-updating widgets rewrite alongside their text. +const contentAttributes = new Set(["aria-label", "title"]); +// Stands in for learned text inside a derived name; HTML text never contains it. +const LEARNED = "\u0000"; export function observationEvidence(scene: VomScene): ObservationEvidence { return { nodes: scene.nodes }; } -/** Stable identity of an observed node within its document. */ -function evidenceKey(node: VomNode): string | undefined { - return node.backendNodeId === undefined - ? undefined - : `${node.frameId ?? ""}:${node.backendNodeId}`; +const isTarget = (node: VomNode, target: EvidenceTarget) => + node.backendNodeId === target.backendNodeId && + (target.frameId === undefined || node.frameId === target.frameId); + +/** Whether the observation contains the target, so its region can be checked. */ +export function observesTarget(evidence: ObservationEvidence, target: EvidenceTarget): boolean { + return evidence.nodes.some((node) => isTarget(node, target)); } -// Form state and modal layers are never tolerated as page activity. +const positioned = (node: VomNode) => ["absolute", "fixed"].includes(node.position); +// Form state, value-bearing widgets and modal layers are never page activity. const protectedState = (node: VomNode) => node.value !== undefined || node.checked !== undefined || !!node.modal; -// Text-bearing facts that self-updating widgets rewrite (clocks often mirror -// their text into aria-label/title for assistive technology). -const contentAttributes = new Set(["aria-label", "title"]); -const content = (node: VomNode) => [ +/** Text a node states itself, excluding names derived from its descendants. */ +const ownContent = (node: VomNode) => [ + textRoles.has(node.role ?? "") ? node.name : undefined, node.text, - node.name, node.attrs?.["aria-label"], node.attrs?.title, ]; -const describe = (node: VomNode) => content(node).find((value) => value) ?? ""; +const clean = (value: string) => value.replace(/\s+/g, " ").trim(); +const wordCharacter = /[\p{L}\p{N}]/u; + +/** Parent/child view of one observation that tolerates malformed cycles. */ +class Tree { + private readonly byId: Map; + private readonly kids = new Map(); + private readonly keys = new Map(); + + constructor(nodes: readonly VomNode[]) { + this.byId = new Map(nodes.map((node) => [node.id, node])); + for (const node of nodes) { + const parent = this.parent(node); + if (!parent) continue; + const siblings = this.kids.get(parent); + if (siblings) siblings.push(node); + else this.kids.set(parent, [node]); + } + } + + parent(node: VomNode): VomNode | undefined { + const parent = this.byId.get(node.domParentId ?? node.parentId ?? -1); + return parent === node ? undefined : parent; + } + + children(node: VomNode): readonly VomNode[] { + return this.kids.get(node) ?? []; + } + + /** The node followed by its ancestors, nearest first. */ + ancestors(node: VomNode): VomNode[] { + const chain: VomNode[] = []; + const seen = new Set(); + for ( + let current: VomNode | undefined = node; + current && !seen.has(current); + current = this.parent(current) + ) { + seen.add(current); + chain.push(current); + } + return chain; + } + + within(node: VomNode, container: VomNode): boolean { + const seen = new Set(); + for ( + let current: VomNode | undefined = node; + current && !seen.has(current); + current = this.parent(current) + ) { + if (current === container) return true; + seen.add(current); + } + return false; + } + + /** + * Stable identity for learning: the node's own backend node, or for virtual + * accessibility text without one (such as an InlineTextBox) the nearest + * ancestor that has one, which owns that text. + */ + key(node: VomNode): string | undefined { + if (this.keys.has(node)) return this.keys.get(node); + const owner = this.ancestors(node).find((current) => current.backendNodeId !== undefined); + const key = owner && `${owner.frameId ?? ""}:${owner.backendNodeId}`; + this.keys.set(node, key); + return key; + } +} + +/** Replaces each occurrence of `value` that is not part of a longer word. */ +function replaceWords(text: string, value: string): string { + let result = ""; + let from = 0; + for (let at = text.indexOf(value); at !== -1; at = text.indexOf(value, at + 1)) { + const end = at + value.length; + if ( + at < from || + (at > 0 && wordCharacter.test(text[at - 1])) || + (end < text.length && wordCharacter.test(text[end])) + ) + continue; + result += text.slice(from, at) + LEARNED; + from = end; + } + return result + text.slice(from); +} + +/** + * Removes learned text from a name derived from descendant text, such as a + * container's joined content. Everything else in the name stays exact, so a + * change hides only if other text changes in lockstep with learned text, as a + * duplicate of the same value does. + */ +function withoutLearned(name: string | undefined, learned?: ReadonlySet) { + if (!name || !learned?.size) return name; + let result = clean(name); + for (const value of [...learned].sort((a, b) => b.length - a.length || (a < b ? -1 : 1))) + result = replaceWords(result, value); + return result; +} function signature( evidence: ObservationEvidence, @@ -68,97 +177,106 @@ function signature( volatile?: ReadonlySet, ): string | undefined { const nodes = evidence.nodes; - const byId = new Map(nodes.map((node) => [node.id, node])); - const parent = (node: VomNode) => byId.get(node.domParentId ?? node.parentId ?? -1); - const anchor = - target && - nodes.find( - (node) => - node.backendNodeId === target.backendNodeId && - (target.frameId === undefined || node.frameId === target.frameId), - ); + const tree = new Tree(nodes); + const anchor = target && nodes.find((node) => isTarget(node, target)); if (target && !anchor) return; - const ancestors: VomNode[] = []; - for (let node = anchor; node && !ancestors.includes(node); node = parent(node)) - ancestors.push(node); + const ancestors = anchor ? tree.ancestors(anchor) : []; // A whole list/table participates, so a changed competing option invalidates // a choice even when the selected row itself did not change. const scope = ancestors.find((node) => scopes.has(node.role ?? "") || scopes.has(node.tag)) ?? ancestors.find((node) => node.modal) ?? ancestors.find((node) => regions.has(node.role ?? "") || regions.has(node.tag)) ?? - ancestors.find((node) => ["absolute", "fixed"].includes(node.position) && node !== anchor); - const inside = (node: VomNode) => { - if (!scope) return true; - const seen = new Set(); - for ( - let current: VomNode | undefined = node; - current && !seen.has(current.id); - current = parent(current) - ) { - if (current.id === scope.id) return true; - seen.add(current.id); + ancestors.find((node) => positioned(node) && node !== anchor); + const inside = (node: VomNode) => !scope || tree.within(node, scope); + // Where learned text may differ. The whole-page check (no target) allows it + // everywhere. A target inside a form, dialog, list, table, section, article, + // positioned panel or other region keeps that region exact. Inside
or + // a page without regions, the target's own branch stays exact: the child of + // the first layout node that is not a single-child wrapper, never descending + // into a positioned panel. This keeps cards, panels and other content + // sharing a container with the target exact. + let relaxable = (_node: VomNode) => true; + if (anchor) { + const broad = !scope || scope.role === "main" || scope.tag === "main"; + if (!broad) relaxable = () => false; + else { + let top = scope ?? ancestors[ancestors.length - 1]; + while (top !== anchor) { + const [only, ...others] = tree.children(top); + if (!only || others.length || positioned(only)) break; + top = only; + } + const index = ancestors.indexOf(top); + const branch = top === anchor ? anchor : index > 0 ? ancestors[index - 1] : undefined; + // An unexpected tree keeps the whole region exact. + relaxable = branch ? (node) => !tree.within(node, branch) : () => false; } - return false; - }; - // Semantic decision units (forms, dialogs, lists, cards, popovers) stay exact. - // Only page layout around the target, a
landmark or a page without - // any enclosing region, may contain learned self-updating text. - const broad = !scope || scope.role === "main" || scope.tag === "main"; - const withinAnchor = (node: VomNode) => { - const seen = new Set(); - for ( - let current: VomNode | undefined = node; - current && !seen.has(current.id); - current = parent(current) - ) { - if (current === anchor) return true; - seen.add(current.id); - } - return false; - }; - // Identity, structure, state and visibility of tolerated nodes remain checked. - // The target itself and its contents are never tolerated. + } const tolerated = (node: VomNode) => - volatile?.has(evidenceKey(node) ?? "") === true && + !!volatile?.size && + volatile.has(tree.key(node) ?? "") && !protectedState(node) && - (!target || (broad && !withinAnchor(node))); - const facts = (node: VomNode, tolerate = false) => [ - node.frameId, - node.backendNodeId, - node.role, - node.tag, - tolerate ? undefined : node.name, - tolerate ? undefined : node.text, - node.value, - node.checked, - node.disabled, - node.inert, - node.modal, - !!node.rect && node.rect.w > 0 && node.rect.h > 0, - node.pointerEvents === "none", - semanticAttributes.map((key) => - tolerate && contentAttributes.has(key) ? undefined : node.attrs?.[key], - ), - ]; + relaxable(node); + // Names derived from descendants hide only the learned text they contain. + const learnedText = new Map>(); + for (const node of volatile?.size ? nodes : []) { + if (!tolerated(node)) continue; + const values = ownContent(node) + .filter((value): value is string => !!value) + .map(clean) + .filter(Boolean); + for (const holder of values.length ? tree.ancestors(node) : []) { + const set = learnedText.get(holder) ?? new Set(); + for (const value of values) set.add(value); + learnedText.set(holder, set); + } + } + // Identity, structure, state and visibility of learned nodes remain checked. + const facts = (node: VomNode, relax: boolean) => { + const tolerate = relax && tolerated(node); + return [ + node.frameId, + node.backendNodeId, + node.role, + node.tag, + tolerate && textRoles.has(node.role ?? "") + ? undefined + : relax + ? withoutLearned(node.name, learnedText.get(node)) + : node.name, + tolerate ? undefined : node.text, + node.value, + node.checked, + node.disabled, + node.inert, + node.modal, + !!node.rect && node.rect.w > 0 && node.rect.h > 0, + node.pointerEvents === "none", + semanticAttributes.map((key) => + tolerate && contentAttributes.has(key) ? undefined : node.attrs?.[key], + ), + ]; + }; return JSON.stringify([ scope && [scope.frameId, scope.backendNodeId], - nodes.filter(inside).map((node) => facts(node, tolerated(node))), + nodes.filter(inside).map((node) => facts(node, true)), // Form state outside the action region can still determine its meaning. nodes .filter((node) => node.value !== undefined || node.checked !== undefined) - .map((node) => facts(node)), - nodes.filter((node) => node.modal).map((node) => facts(node)), + .map((node) => facts(node, false)), + nodes.filter((node) => node.modal).map((node) => facts(node, false)), ]); } /** - * `volatile` lists nodes whose text (name, text, aria-label, title) may differ. - * Without a target it applies to the whole page. With a target it applies only - * when the target's region is a
landmark or the whole page, and never to - * the target or its contents. Forms, dialogs, lists, tables, menus, sections, - * articles and positioned panels around a target stay exact, as do form values, - * checked states and modal layers everywhere. + * `volatile` lists the stable identities (backend nodes, owning any virtual + * text beneath them) whose own text, aria-label and title may differ. Names + * derived from descendant text are compared with that learned text removed + * wherever it appears as whole words. Without a target this applies to the + * whole page. With a target it never applies inside the target's own region, + * as described in signature(). Form values, checked states and modal layers + * stay exact everywhere. */ export function sameObservationEvidence( before: ObservationEvidence, @@ -170,25 +288,64 @@ export function sameObservationEvidence( return a !== undefined && a === signature(after, target, volatile); } +interface ContentGroup { + content: (string | undefined)[][]; + protected: boolean; +} + +function contentGroups(evidence: ObservationEvidence): Map { + const tree = new Tree(evidence.nodes); + const groups = new Map(); + for (const node of evidence.nodes) { + const key = tree.key(node); + if (!key) continue; + const group = groups.get(key) ?? { content: [], protected: false }; + group.content.push(ownContent(node)); + group.protected ||= protectedState(node); + groups.set(key, group); + } + return groups; +} + +type Content = (string | undefined)[]; +/** The first own-content field that differs, for reporting the change. */ +function described(before: Content | undefined, after: Content | undefined): [string, string] { + const fields = Math.max(before?.length ?? 0, after?.length ?? 0); + for (let field = 0; field < fields; field++) + if ((before?.[field] ?? "") !== (after?.[field] ?? "")) + return [clean(before?.[field] ?? ""), clean(after?.[field] ?? "")]; + const first = (content?: Content) => + content?.map((value) => value && clean(value)).find(Boolean) ?? ""; + return [first(before), first(after)]; +} + /** - * Returns the persistent, unprotected nodes whose text changed when that is the - * only difference between two whole-page observations; otherwise undefined. - * Added, removed, reordered or restyled nodes and any form or modal change - * are not page activity and keep requiring a new decision. + * Returns the persistent, unprotected nodes whose own text changed when that is + * the only difference between two whole-page observations; otherwise undefined. + * Added, removed, reordered or restyled nodes, any form or modal change, and + * any derived name that the changed text does not explain keep requiring a new + * decision. */ export function contentChanges( before: ObservationEvidence, after: ObservationEvidence, known: ReadonlySet = new Set(), ): ContentChange[] | undefined { - const previous = new Map(before.nodes.map((node) => [evidenceKey(node), node])); + const previous = contentGroups(before); const changes: ContentChange[] = []; - for (const node of after.nodes) { - const key = evidenceKey(node); - const old = key === undefined ? undefined : previous.get(key); - if (!key || !old || known.has(key) || protectedState(node) || protectedState(old)) continue; - if (JSON.stringify(content(old)) !== JSON.stringify(content(node))) - changes.push({ key, before: describe(old), after: describe(node) }); + for (const [key, group] of contentGroups(after)) { + const old = previous.get(key); + if (!old || known.has(key) || old.protected || group.protected) continue; + if (JSON.stringify(old.content) === JSON.stringify(group.content)) continue; + const length = Math.max(old.content.length, group.content.length); + let member = 0; + while ( + member < length - 1 && + JSON.stringify(old.content[member]) === JSON.stringify(group.content[member]) + ) + member++; + const [was, now] = described(old.content[member], group.content[member]); + changes.push({ key, before: was, after: now }); } const volatile = new Set([...known, ...changes.map((change) => change.key)]); return changes.length && sameObservationEvidence(before, after, undefined, volatile) diff --git a/crates/bsk-cli/skill/references/cooperation.md b/crates/bsk-cli/skill/references/cooperation.md index 83694b81..bf01819f 100644 --- a/crates/bsk-cli/skill/references/cooperation.md +++ b/crates/bsk-cli/skill/references/cooperation.md @@ -34,17 +34,19 @@ History is bounded and document transitions can have gaps; events are evidence, The default guard checks the whole observed page. When the decision depends on known regions, CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every -region used in the decision: for example both the options list and the current form. The target's -form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Text that changes by itself between sync and action -(a clock, a counter) is reported once with the before/after text. After you sync and replan, -further text-only changes there no longer block actions on a ref or selector, unless the text -is inside the target or shares its form, list, dialog, card or similar region. Actions without -a ref or selector, such as a bare key press, always check the whole page exactly. If the -decision relies on such text, declare it with `--depends-on`, which stays exact. -Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a -live target and hit check. Readonly pickers and discrete controls return control automatically; -text editing, composition, held keys and dragging keep the person's lease until they finish. +region used in the decision: for example both the options list and the current form. The +target's form/list/dialog, all form values, and modal state remain protected. If the dependency +set is uncertain, omit it and retain whole-page checking. Text that changes by itself between +sync and action (a clock, a counter) is reported once with the before/after text. After you sync +and replan, further changes to that text no longer block actions on a ref or selector unless it +shares a form, list, dialog, panel or other container with the target. Actions without a ref or +selector, such as a bare key press, always check the whole page exactly. Added, removed or +re-rendered elements and text without its own accessible node (such as `aria-hidden` content) +keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays +exact. Dependencies must come from the current observation; they do not authorize replaying an +old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and +discrete controls return control automatically; text editing, composition, held keys and +dragging keep the person's lease until they finish. A regular coordination result is not a failed browser attempt. Use wait then sync, and choose the next action from the actual result. Use request-help only for an actual human requirement diff --git a/docs/shared-browser-control.zh-CN.md b/docs/shared-browser-control.zh-CN.md index 701ebb66..8edff5f9 100644 --- a/docs/shared-browser-control.zh-CN.md +++ b/docs/shared-browser-control.zh-CN.md @@ -50,7 +50,7 @@ DSH 使用 `bsk_session` 的 `sync`、`wait-user`、`cooperation-status`。旧 ` 会话生命周期复用 `session.activity` 事件。Daemon 只接受所属浏览器的状态更新，将协作保持与最后工具调用时间一起保存在既有活动记录中；结束保持后有完整的空闲间隔。释放、断线和 session id 复用均清理这一状态。 -默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。在同步与动作之间自行变化的文字（如时钟、计数器）会先被报告一次并附带前后文本；Agent 重新同步和规划后，这些节点的纯文本变化不再阻断带 ref 或 selector 的动作，除非它位于目标内部，或与目标同属一个表单、列表、对话框、卡片等区域。没有 ref 或 selector 的动作（如直接按键）始终精确校验整页。结构、可见性、表单值和模态变化仍会拒绝；若决策依赖这些文字，用 `--depends-on` 声明，显式依赖始终精确校验。 +默认校验整个已观察页面。Agent 已明确知道决策范围时，可重复传入 `--depends-on @eN`，DSH 对应 `dependsOn`。范围必须包含决策依赖的所有区域；目标所在表单、列表或对话框、全部字段值和模态状态仍会检查。比如选择最低价，不能只声明一个按钮而漏掉其他候选。依赖不明确时保留整页校验。在同步与动作之间自行变化的文字（如时钟、计数器）会先被报告一次并附带前后文本；Agent 重新同步和规划后，这些文字的变化不再阻断带 ref 或 selector 的动作，除非它与目标同属一个表单、列表、对话框、浮层或其他容器。没有 ref 或 selector 的动作（如直接按键）始终精确校验整页。新增、删除或重新渲染的元素，以及没有独立无障碍节点的文字（如 `aria-hidden` 内容）仍会阻断；表单值和模态变化同样会拒绝。若决策依赖这些文字，用 `--depends-on` 声明，显式依赖始终精确校验。 `cooperation-capture.ts` 在隔离世界采集输入和 DOM 变化。输入保持即时通知，DOM 变更合并后通知；主文档及 iframe 都必须响应。普通字段变化走增量路径，字段以文档内稳定标识区分，同名字段和开放 Shadow DOM 输入框也分别记录。结构变化、文档替换、字段过多或内容截断时，复用现有 VOM 快照刷新基线。增量字段变更同步后会更新内部语义证据，避免后续动作仍拿旧值比较；不重复输出整页文本，不替换引用表。采集不触发 hover。密码、文件选择不进入字段变化流。 diff --git a/packages/dsh-plugin-browserskill/skill/references/cooperation.md b/packages/dsh-plugin-browserskill/skill/references/cooperation.md index e120ba0a..ad368b99 100644 --- a/packages/dsh-plugin-browserskill/skill/references/cooperation.md +++ b/packages/dsh-plugin-browserskill/skill/references/cooperation.md @@ -28,17 +28,19 @@ History is bounded and document transitions can have gaps; events are evidence, The default guard checks the whole observed page. When the decision depends on known regions, CLI actions can repeat `--depends-on @eN` (the plugin uses `dependsOn: ["@eN"]`). Include every -region used in the decision: for example both the options list and the current form. The target's -form/list/dialog, all form values, and modal state remain protected. If the dependency set is -uncertain, omit it and retain whole-page checking. Text that changes by itself between sync and action -(a clock, a counter) is reported once with the before/after text. After you sync and replan, -further text-only changes there no longer block actions on a ref or selector, unless the text -is inside the target or shares its form, list, dialog, card or similar region. Actions without -a ref or selector, such as a bare key press, always check the whole page exactly. If the -decision relies on such text, declare it with `--depends-on`, which stays exact. -Dependencies must come from the current observation; they do not authorize replaying an old plan. Cosmetic changes still require a -live target and hit check. Readonly pickers and discrete controls return control automatically; -text editing, composition, held keys and dragging keep the person's lease until they finish. +region used in the decision: for example both the options list and the current form. The +target's form/list/dialog, all form values, and modal state remain protected. If the dependency +set is uncertain, omit it and retain whole-page checking. Text that changes by itself between +sync and action (a clock, a counter) is reported once with the before/after text. After you sync +and replan, further changes to that text no longer block actions on a ref or selector unless it +shares a form, list, dialog, panel or other container with the target. Actions without a ref or +selector, such as a bare key press, always check the whole page exactly. Added, removed or +re-rendered elements and text without its own accessible node (such as `aria-hidden` content) +keep blocking. If the decision relies on such text, declare it with `--depends-on`, which stays +exact. Dependencies must come from the current observation; they do not authorize replaying an +old plan. Cosmetic changes still require a live target and hit check. Readonly pickers and +discrete controls return control automatically; text editing, composition, held keys and +dragging keep the person's lease until they finish. A regular coordination result is not a failed browser attempt. Use wait then sync, and choose the next action from the actual result. Use request-help only for an actual human requirement