fix(cooperation): tolerate media progress while preserving action guards

apps/extension/src/content/__tests__/cooperation-capture.test.ts

@@ -20,9 +20,10 @@ afterEach(() => {
   vi.restoreAllMocks();
 });
 
-function collector() {
+function collector(shadowRoot?: (element: HTMLElement) => ShadowRoot | null) {
   let receive!: (message: unknown, sender: unknown, reply: (result: unknown) => void) => void;
   vi.stubGlobal("chrome", {
+    dom: shadowRoot ? { openOrClosedShadowRoot: shadowRoot } : undefined,
     runtime: {
       sendMessage: vi.fn(async () => {}),
       onMessage: {
@@ -404,3 +405,109 @@ it("announces renderer readiness once after the collection listener exists", ()
   expect(chrome.runtime.sendMessage).toHaveBeenCalledWith({ kind: COOPERATION_READY });
   expect(read()).toHaveProperty("document");
 });
+
+it("classifies a non-submitted media timeline without masking business fields", () => {
+  document.body.innerHTML =
+    '<video id="preview"></video><label>Playback progress<input id="timeline" type="range" value="1"></label><input aria-label="Order total" value="100"><input type="range" aria-label="Order amount" value="10">';
+  const read = collector();
+  const before = read()!;
+  expect(before.fields[0]).toMatchObject({ value: "1", kind: "media_progress" });
+  expect(before.fields.slice(1).every((field) => field.kind === undefined)).toBe(true);
+  const progress = document.querySelector<HTMLInputElement>("#timeline")!;
+  progress.value = "2";
+  expect(read()!.fields[0]).toMatchObject({ value: "2", kind: "media_progress" });
+  progress.name = "order_amount";
+  expect(read()!.fields[0].kind).toBeUndefined();
+});
+
+it.each([
+  "form",
+  "external",
+  "ambiguous",
+  "missing",
+])("does not classify a %s timeline as incidental", (kind) => {
+  document.body.innerHTML =
+    '<video id="preview"></video><input id="timeline" type="range" aria-label="Playback progress">';
+  const timeline = document.querySelector<HTMLInputElement>("#timeline")!;
+  if (kind === "form") {
+    const form = document.createElement("form");
+    document.body.append(form);
+    form.append(timeline);
+  }
+  if (kind === "external") {
+    const form = document.createElement("form");
+    form.id = "order";
+    document.body.append(form);
+    timeline.setAttribute("form", "order");
+  }
+  if (kind === "ambiguous") document.body.append(document.createElement("video"));
+  if (kind === "missing") document.querySelector("video")!.remove();
+  expect(collector()()!.fields[0].kind).toBeUndefined();
+});
+
+it("reports programmatic focus changes even without a human event", () => {
+  document.body.innerHTML = '<button id="pause">Pause</button><input id="order">';
+  const read = collector();
+  const first = read()!.focus;
+  document.querySelector<HTMLInputElement>("#order")!.focus();
+  const current = read()!;
+  expect(current.focus).not.toBe(first);
+  expect(current.human).toBe(0);
+});
+
+it("records media sources independently of playback time, including runtime-only source changes", () => {
+  document.body.innerHTML = '<video><source src="one.webm" type="video/webm"></video>';
+  const video = document.querySelector("video")!;
+  const source = video.querySelector("source")!;
+  const read = collector();
+  const first = read()!;
+  video.currentTime = 1;
+  expect(read()!.media).toEqual(first.media);
+  source.src = "two.webm";
+  const changed = read()!;
+  expect(changed.media).not.toEqual(first.media);
+  expect(changed.revision).toBeGreaterThan(first.revision);
+  Object.defineProperty(video, "currentSrc", { configurable: true, value: "selected.webm" });
+  const selected = read()!;
+  expect(selected.media![0].currentSrc).toBe("selected.webm");
+  expect(selected.revision).toBeGreaterThan(changed.revision);
+  Object.defineProperty(video, "srcObject", { configurable: true, value: {} });
+  const stream = read()!;
+  expect(stream.media![0].srcObject).toEqual(expect.any(String));
+  Object.defineProperty(video, "srcObject", { configurable: true, value: {} });
+  expect(read()!.media![0].srcObject).not.toBe(stream.media![0].srcObject);
+});
+
+it("collects media and deep focus in open shadow roots", () => {
+  document.body.innerHTML = '<div id="host"></div>';
+  const root = document.querySelector("#host")!.attachShadow({ mode: "open" });
+  root.innerHTML = '<audio><source src="one.ogg"></audio><button>Pause</button>';
+  const read = collector();
+  expect(read()!.focusIsDocument).toBe(true);
+  root.querySelector<HTMLButtonElement>("button")!.focus();
+  const focused = read()!;
+  expect(focused.focusIsDocument).toBe(false);
+  expect(focused.media).toHaveLength(1);
+  root.querySelector("source")!.src = "two.ogg";
+  expect(read()!.media).not.toEqual(focused.media);
+});
+
+it("collects source and recipient changes inside closed shadow roots", () => {
+  document.body.innerHTML = '<div id="host"></div>';
+  const host = document.querySelector<HTMLElement>("#host")!;
+  const root = host.attachShadow({ mode: "closed" });
+  root.innerHTML =
+    '<video><source src="one.webm"></video><button>Pause</button><button>Buy</button>';
+  const read = collector((element) => (element === host ? root : null));
+  const buttons = root.querySelectorAll<HTMLButtonElement>("button");
+  buttons[0].focus();
+  const first = read()!;
+  expect(host.shadowRoot).toBeNull();
+  expect(first.media).toHaveLength(1);
+  buttons[1].focus();
+  root.querySelector("source")!.src = "two.webm";
+  const changed = read()!;
+  expect(changed.focus).not.toBe(first.focus);
+  expect(changed.media).not.toEqual(first.media);
+  expect(changed.revision).toBeGreaterThan(first.revision);
+});

apps/extension/src/content/cooperation-capture.ts

@@ -1,3 +1,4 @@
+import { playbackProgressMedia } from "@/cooperation/media-progress";
 import {
   COOPERATION_ARM,
   COOPERATION_DISABLE,
@@ -42,9 +43,9 @@ export function attachCooperationCapture() {
   const keys = new Set<string>();
   const fields = new Map<Element, FieldChange>();
   const sampledValues = new WeakMap<Element, string>();
-  const ids = new WeakMap<Element, string>();
+  const ids = new WeakMap<object, string>();
   let nextId = 0;
-  const id = (element: Element) => {
+  const id = (element: object) => {
     let value = ids.get(element);
     if (!value) ids.set(element, (value = String(++nextId)));
     return value;
@@ -101,6 +102,20 @@ export function attachCooperationCapture() {
   };
   const observer = new MutationObserver(mutations);
   const roots = new Set<Document | ShadowRoot>();
+  // Page JS cannot see closed roots, but the extension can. Observation also
+  // traverses these roots, so source and focus guards must cover the same DOM.
+  const shadowRoot = (element: Element): ShadowRoot | null =>
+    element.shadowRoot ??
+    (element instanceof HTMLElement ? (chrome.dom?.openOrClosedShadowRoot(element) ?? null) : null);
+  const activeElement = () => {
+    let active = document.activeElement;
+    while (active) {
+      const nested = shadowRoot(active)?.activeElement;
+      if (!nested) break;
+      active = nested;
+    }
+    return active;
+  };
   const controls = function* (root: Document | ShadowRoot): Generator<Element> {
     if (!roots.has(root)) {
       observer.observe(root, {
@@ -114,17 +129,24 @@ export function attachCooperationCapture() {
     for (const element of root.querySelectorAll("*")) {
       if (overlay(element)) continue;
       if (element.matches("input,textarea,select,[contenteditable=true]")) yield element;
-      if (element.shadowRoot) yield* controls(element.shadowRoot);
+      const nested = shadowRoot(element);
+      if (nested) yield* controls(nested);
     }
   };
   const read = () => {
     mutations(observer.takeRecords());
+    const active = activeElement();
+    state.focus = active ? id(active) : undefined;
+    state.focusIsDocument =
+      (active === document.body || active === document.documentElement) &&
+      !active?.matches('[tabindex],[role],[contenteditable]:not([contenteditable="false"])');
     if ([...roots].some((root) => root instanceof ShadowRoot && !root.host.isConnected)) {
       observer.disconnect();
       roots.clear();
     }
     for (const element of fields.keys()) if (!element.isConnected) fields.delete(element);
     let fieldCount = 0;
+    const mediaByRoot = new Map<Document | ShadowRoot, { id: string }[]>();
     state.fieldsComplete = true;
     for (const target of controls(document)) {
       if (
@@ -140,16 +162,79 @@ export function attachCooperationCapture() {
             : (target.textContent ?? "");
       const value = fullValue.slice(0, 500);
       const name = label(target);
+      let mediaProgress = false;
+      if (target instanceof HTMLInputElement && target.type === "range") {
+        const root = target.getRootNode() as Document | ShadowRoot;
+        let media = mediaByRoot.get(root);
+        if (!media) {
+          media = [...root.querySelectorAll("video,audio")].map((element) => ({ id: element.id }));
+          mediaByRoot.set(root, media);
+        }
+        const labelledBy = (target.getAttribute("aria-labelledby") ?? "")
+          .split(/\s+/)
+          .filter(Boolean)
+          .map((id) => root.getElementById(id)?.textContent ?? "")
+          .join(" ");
+        const labels = [...new Set(target.labels ?? [])]
+          .map((label) => label.textContent)
+          .join(" ");
+        mediaProgress = !!playbackProgressMedia(
+          {
+            tag: "input",
+            name: target.getAttribute("aria-label") || labelledBy || labels,
+            attrs: Object.fromEntries(
+              [...target.attributes].map((attr) => [attr.name, attr.value]),
+            ),
+            inForm: target.form !== null,
+          },
+          media,
+        );
+      }
       fieldCount++;
       if (fieldCount > MAX_CHANGES || fullValue.length > 500) state.fieldsComplete = false;
       const sampled = sampledValues.get(target);
       if (sampled !== undefined && sampled !== fullValue) state.revision++;
       sampledValues.set(target, fullValue);
       const old = fields.get(target);
       if (fields.size < MAX_CHANGES || old)
-        fields.set(target, { id: id(target), label: name, value });
+        fields.set(target, {
+          id: id(target),
+          label: name,
+          value,
+          ...(mediaProgress ? { kind: "media_progress" as const } : {}),
+        });
     }
     for (const element of fields.keys()) if (!element.isConnected) fields.delete(element);
+    const media = [...roots].flatMap((root) =>
+      [...root.querySelectorAll<HTMLMediaElement>("video,audio")].map((element) => {
+        const stream = element.srcObject;
+        return {
+          id: id(element),
+          src: element.getAttribute("src"),
+          currentSrc: element.currentSrc,
+          srcObject: stream ? id(stream) : null,
+          tracks:
+            typeof MediaStream !== "undefined" && stream instanceof MediaStream
+              ? stream
+                  .getTracks()
+                  .map((track) => track.id)
+                  .sort()
+              : [],
+          sources: [...element.children]
+            .filter((child): child is HTMLSourceElement => child instanceof HTMLSourceElement)
+            .map((source) => ({
+              id: id(source),
+              src: source.src,
+              type: source.type,
+              media: source.media,
+            })),
+        };
+      }),
+    );
+    // currentSrc/srcObject may change without a DOM mutation. Keep the source
+    // evidence fresh even when the page's mutation counter did not advance.
+    if (state.media && JSON.stringify(state.media) !== JSON.stringify(media)) state.revision++;
+    state.media = media;
     state.busy = composing || pointer || keys.size > 0;
     state.fields = [...fields.values()];
     return { ...state };
@@ -219,8 +304,7 @@ export function attachCooperationCapture() {
       clearTimeout(focusTimer);
       focusTimer = setTimeout(() => {
         if (!enabled) return;
-        let active = document.activeElement;
-        while (active?.shadowRoot?.activeElement) active = active.shadowRoot.activeElement;
+        const active = activeElement();
         if (state.editing) state.revision++;
         state.editing = state.editing && editable(active);
         composing = false;