diff --git a/docs/design/sdk-design.md b/docs/design/sdk-design.md index ffde5c8d..f6bd4d07 100644 --- a/docs/design/sdk-design.md +++ b/docs/design/sdk-design.md @@ -703,7 +703,7 @@ Single source of truth. Owns: types, serializers, sanitizer lists, option transl | `traces.user` | `traces.user.json` | same | | `events.system` | `events.system.json` | JSON array of `{timestamp, name, params?}` | | `events.user` | `events.user.json` | same — **application-supplied only** (`client.event()`); SDK code never writes here | -| `input` | `input.json` | JSON array of device-input events — see §8.4.1 | +| `input` | `input.json` | `{version, events: [...]}` (version 3 — mouse capture, §8.4.1) | | `viewtree` | `viewtree.json` | DOM snapshot | | `log` | `logs.json` | `{timestamp, level (1-5), source, tag?, message}` | | `log.internal` | `internal.logs.json` | SDK self-diagnostics (only if `debug: true`) | @@ -728,26 +728,30 @@ Entry shape — the viewer's `RecordingTouchEvent`, so web and mobile render thr | field | meaning | | --- | --- | | `timestamp` | wall-clock ms | -| `id` | groups the stages of one gesture | -| `type` | `begin` \| `move` \| `end` for device input; the semantic name for `tool: Other`. Web: `move` is recorded for PEN gestures only (in contact, on change; none for a gesture that began on a masked target) | +| `id` | groups the stages of one gesture. `input.md` lists this field as always present; this SDK's own convention is to OMIT an optional field rather than emit a placeholder for it when it does not apply — so a mouse wheel `scroll` entry, which opens/closes no gesture, carries no `id` key at all (not `null`, not `""`) | +| `type` | `begin` \| `move` \| `end` \| `scroll` for device input; the semantic name for `tool: Other`. Web: pointer `move` is recorded for PEN gestures (in contact, on change; none for a gesture that began on a masked target) and, **new in version 3**, for a MOUSE while a button is held (drag, same terms as a pen stroke) plus sampled hover (~10/s, `HOVER_SAMPLE_INTERVAL_MS`) | | `x`, `y` | viewport CSS pixels (rounded) | | `force` | normalised pressure 0..1 | | `majorRadius`, `minorRadius` | contact geometry (touch/pen only) | | `altitudeAngle`, `azimuthAngle` | stylus orientation in radians — pen only, and only when the device reports it (iOS names/conventions: altitude 0 flat … π/2 perpendicular; azimuth 0 along +x, clockwise). Web: Pointer Events L3 angles, else converted from `tiltX`/`tiltY`; the spec's no-data defaults (π/2 + 0, tilt 0/0) are omitted (#6) | | `tool` | `InputTool`: 0 Unknown, 1 Touch, 2 Mouse, 3 Pen, 4 Remote, 5 Other, 6 Eraser, **7 Key** | | `view`, `view_id`, `view_tag` | target class / id / tag (the viewer maps these to `target.class/.id/.tag`) | +| `button` | **Version 3** (`input.md`): the button that CHANGED, in the shared cross-platform numbering (0 primary, 1 secondary, 2 middle, 3 back, 4 forward). Mouse `begin`/`end` only; omitted for an unrecognised button and always for pen/touch. The web tier MAPS the DOM's `MouseEvent.button` (0 primary, **1 middle**, **2 secondary** — the other way round) onto it, never passes it through | +| `buttonMask` | **Version 3:** buttons HELD, mouse only — the DOM's `MouseEvent.buttons` bitmask already matches the wire's `1< { vi.unstubAllGlobals(); @@ -231,7 +236,14 @@ function fakeTarget() { }; } -const ALL_INTERACTIONS = ['pointerdown', 'pointermove', 'pointerup', 'pointercancel', 'keydown']; +const ALL_INTERACTIONS = [ + 'pointerdown', + 'pointermove', + 'pointerup', + 'pointercancel', + 'keydown', + 'wheel', +]; /** The state-change DOM signals this source deliberately does NOT observe (they are breadcrumbs). */ const NOT_INPUT = ['change', 'submit', 'focusin']; @@ -242,13 +254,15 @@ function activate(env: BrowserInputEnv) { return { events, off }; } -/** A pointerdown/up-shaped fake event. */ +/** A pointerdown/up-shaped fake event. `buttons: 1` (primary held) matches what a real PointerEvent + * reports on the down that just pressed it; override to `0` to model the mask AFTER a release. */ const ptr = (over: Record = {}) => ({ pointerId: 1, pointerType: 'mouse', clientX: 12, clientY: 34, button: 0, + buttons: 1, pressure: 0.5, width: 1, height: 1, @@ -289,7 +303,7 @@ describe('createBrowserInputSource', () => { const target = fakeTarget(); const { events } = activate({ target }); target.emit('pointerdown', ptr()); - target.emit('pointerup', ptr({ pressure: 0 })); + target.emit('pointerup', ptr({ pressure: 0, buttons: 0 })); // released: nothing held any more expect(events).toStrictEqual([ { id: '1', @@ -299,6 +313,8 @@ describe('createBrowserInputSource', () => { force: 0.5, tool: InputTool.Mouse, button: 0, + buttonMask: 1, + metaState: 0, view_tag: 'button', target: { text: 'OK', selector: 'button' }, }, @@ -310,6 +326,8 @@ describe('createBrowserInputSource', () => { force: 0, tool: InputTool.Mouse, button: 0, + buttonMask: 0, + metaState: 0, view_tag: 'button', target: { text: 'OK', selector: 'button' }, }, @@ -426,12 +444,14 @@ describe('createBrowserInputSource', () => { expect(events[2]).toMatchObject({ x: 30, y: 50, altitudeAngle: 0.5, azimuthAngle: 2 }); }); - it.each(['mouse', 'touch'])('records no moves for a %s gesture', (pointerType) => { + // Touch still records no moves at all (unchanged by version 3 — see the dedicated "mouse moves" + // describe block below for the mouse drag/hover cases version 3 adds). + it('records no moves for a touch gesture', () => { const target = fakeTarget(); const { events } = activate({ target }); - target.emit('pointerdown', ptr({ pointerType })); - target.emit('pointermove', ptr({ pointerType, clientX: 99 })); - target.emit('pointerup', ptr({ pointerType, clientX: 99 })); + target.emit('pointerdown', ptr({ pointerType: 'touch' })); + target.emit('pointermove', ptr({ pointerType: 'touch', clientX: 99 })); + target.emit('pointerup', ptr({ pointerType: 'touch', clientX: 99 })); expect(events.map((e) => e.type)).toStrictEqual(['begin', 'end']); }); @@ -448,6 +468,17 @@ describe('createBrowserInputSource', () => { expect(events.map((e) => e.type)).toStrictEqual(['begin', 'end', 'begin', 'end']); }); + // Distinct from "records no moves for a HOVERING pen" above: here penPaths is NOT empty (pointer 7's + // gesture is open), so the fast size-check bail does not fire — the lookup for pointer 9 itself must + // still come back empty and record nothing. + it('records no move for a pen pointerId with no open gesture, even while a different pen gesture is open', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', pen({ pointerId: 7 })); + target.emit('pointermove', pen({ pointerId: 9, clientX: 1 })); + expect(events.map((e) => e.type)).toStrictEqual(['begin']); + }); + it('skips a move that changes nothing recorded, measured against the last entry of that gesture', () => { const target = fakeTarget(); const { events } = activate({ target }); @@ -567,6 +598,216 @@ describe('createBrowserInputSource', () => { }); }); + // ---- mouse moves (version 3): drag while a button is held, plus sampled hover ---- + describe('mouse moves', () => { + /** A clock the test fully controls, for the hover sample throttle. */ + function fakeClock(start = 0) { + let now = start; + return { now: () => now, advance: (ms: number) => (now += ms) }; + } + + // buttonMask is written on a move too (every mouse stage the platform can read `buttons` on) — but + // never `button`, since a move never changes which one is down. + it('records a drag move of an open mouse gesture, with buttonMask but no target or button', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); // buttons: 1 (default) — a button is held + target.emit('pointermove', ptr({ clientX: 20, clientY: 40, pressure: 0.7 })); + expect(events.map((e) => e.type)).toStrictEqual(['begin', 'move']); + expect(events[1]).toStrictEqual({ + id: '1', + type: 'move', + x: 20, + y: 40, + force: 0.7, + tool: InputTool.Mouse, + buttonMask: 1, + }); + expect(events[1]).not.toHaveProperty('button'); + }); + + it('reflects a chorded buttonMask on a drag move (a second button held mid-drag)', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); + target.emit('pointermove', ptr({ clientX: 20, buttons: 1 | 2 })); // primary + secondary now held + expect(events[1]?.buttonMask).toBe(3); + }); + + it('keeps a drag move under the gesture id opened by its begin', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); + target.emit('pointermove', ptr({ clientX: 50 })); + target.emit('pointerup', ptr({ buttons: 0 })); + expect(events.map((e) => e.id)).toStrictEqual(['1', '1', '1']); + }); + + it('skips a drag move that changes nothing recorded (measured against the last entry)', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); + target.emit('pointermove', ptr()); // identical to the begin + target.emit('pointermove', ptr({ clientX: 12.2 })); // rounds to the same x + target.emit('pointermove', ptr({ clientX: 13 })); // x actually changed + expect(events.map((e) => e.type)).toStrictEqual(['begin', 'move']); + expect(events[1]?.x).toBe(13); + }); + + it('records no drag moves for a gesture that began on a masked target', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + const secure = el({ tag: 'input', type: 'password' }); + target.emit('pointerdown', ptr({ target: secure })); + target.emit('pointermove', ptr({ clientX: 99, target: secure })); + target.emit('pointerup', ptr({ buttons: 0, target: secure })); + expect(events.map((e) => e.type)).toStrictEqual(['begin', 'end']); + }); + + it('records no drag move for a pointerId with no open mouse gesture', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointermove', ptr({ clientX: 99 })); // no preceding pointerdown at all + expect(events).toStrictEqual([]); + }); + + // Distinct from the above: here mouseDragPaths is NOT empty (pointer 7's drag is open), so the fast + // size-check bail does not fire — the lookup for pointer 9 itself must still come back empty. + it('records no drag move for a pointerId with no open gesture, even while a different drag is open', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ pointerId: 7 })); + target.emit('pointermove', ptr({ pointerId: 9, clientX: 1 })); + expect(events.map((e) => e.type)).toStrictEqual(['begin']); + }); + + it('keeps concurrent mouse drags apart: each move joins, and is compared against, its own gesture', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ pointerId: 7, clientX: 100 })); + target.emit('pointerdown', ptr({ pointerId: 8, clientX: 200 })); + target.emit('pointermove', ptr({ pointerId: 7, clientX: 200 })); // equals pointer 8's state + target.emit('pointermove', ptr({ pointerId: 8, clientX: 200 })); // unchanged for pointer 8 + expect(events.map((e) => [e.type, e.id, e.x])).toStrictEqual([ + ['begin', '1', 100], + ['begin', '2', 200], + ['move', '1', 200], + ]); + }); + + it('forgets drag state on deactivate, so a re-activation records no move for a stale gesture', () => { + const target = fakeTarget(); + const source = createBrowserInputSource({ target }); + const events: InputEventDetail[] = []; + const off = source.onAny((_s, e) => events.push(e)); + target.emit('pointerdown', ptr()); + off(); + const off2 = source.onAny((_s, e) => events.push(e)); + target.emit('pointermove', ptr({ clientX: 70 })); + off2(); + expect(events.map((e) => e.type)).toStrictEqual(['begin']); + }); + + it('samples a hovering mouse (no button held), with buttonMask 0 but no target and no button', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + target.emit('pointermove', ptr({ buttons: 0, clientX: 15, clientY: 25 })); + expect(events).toStrictEqual([ + { id: '1', type: 'move', x: 15, y: 25, force: 0.5, tool: InputTool.Mouse, buttonMask: 0 }, + ]); + }); + + // Deliberately NOT masked-target-gated, unlike a drag move (see the reviewer-flagged design + // decision in input-source.ts's PII discipline section): hover never opens a gesture, so it has no + // "began on" target to check, and it carries only a bare position — the same PII profile as every + // other pointer coordinate this source already records over a masked field. Pins that decision + // against a well-meaning "fix" that makes hover symmetric with drag and starts dropping it. + it('keeps sampling a hovering mouse over a MASKED target — hover is deliberately not mask-gated', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + const secure = el({ tag: 'input', type: 'password' }); + target.emit('pointermove', ptr({ buttons: 0, target: secure, clientX: 15, clientY: 25 })); + expect(events).toStrictEqual([ + { id: '1', type: 'move', x: 15, y: 25, force: 0.5, tool: InputTool.Mouse, buttonMask: 0 }, + ]); + }); + + it('throttles hover samples to the named interval, not faster', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + target.emit('pointermove', ptr({ buttons: 0, clientX: 1 })); // sampled (first ever) + clock.advance(50); + target.emit('pointermove', ptr({ buttons: 0, clientX: 2 })); // too soon — dropped + clock.advance(49); + target.emit('pointermove', ptr({ buttons: 0, clientX: 3 })); // still too soon (99ms total) + clock.advance(1); + target.emit('pointermove', ptr({ buttons: 0, clientX: 4 })); // exactly the interval — sampled + expect(events.map((e) => e.x)).toStrictEqual([1, 4]); + }); + + it('gives one continuous hover run a stable id across its samples', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + target.emit('pointermove', ptr({ buttons: 0, clientX: 1 })); + clock.advance(HOVER_SAMPLE_INTERVAL_MS); + target.emit('pointermove', ptr({ buttons: 0, clientX: 2 })); + expect(events.map((e) => e.id)).toStrictEqual(['1', '1']); + }); + + it('opens a fresh hover id after a press interrupts the run (never reuses the press id)', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + target.emit('pointermove', ptr({ buttons: 0, clientX: 1 })); // hover id 'X' + target.emit('pointerdown', ptr()); // a real press — its own id + target.emit('pointerup', ptr({ buttons: 0 })); + clock.advance(HOVER_SAMPLE_INTERVAL_MS); + target.emit('pointermove', ptr({ buttons: 0, clientX: 2 })); // fresh hover id + const ids = events.map((e) => e.id); + const [hover1, press, release, hover2] = ids; + expect(new Set([hover1, press, release, hover2]).size).toBe(3); // press === release; both != hovers + expect(press).toBe(release); + expect(hover1).not.toBe(hover2); + }); + + it('tracks separate hover runs per pointerId', () => { + const target = fakeTarget(); + const clock = fakeClock(); + const { events } = activate({ target, now: clock.now }); + target.emit('pointermove', ptr({ pointerId: 7, buttons: 0, clientX: 1 })); + target.emit('pointermove', ptr({ pointerId: 8, buttons: 0, clientX: 2 })); + expect(events[0]?.id).not.toBe(events[1]?.id); + }); + + it('never lets a hostile mouse move event reach the app', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); + const hostile = { + pointerType: 'mouse', + pointerId: 1, + buttons: 1, + get clientX(): number { + throw new Error('instrumented event'); + }, + }; + expect(() => target.emit('pointermove', hostile)).not.toThrow(); + expect(events.map((e) => e.type)).toStrictEqual(['begin']); + }); + + it('defaults the hover sample clock to Date.now when none is injected', () => { + const target = fakeTarget(); + const { events } = activate({ target }); // no `now` → real Date.now + target.emit('pointermove', ptr({ buttons: 0 })); + expect(events).toHaveLength(1); + expect(events[0]?.type).toBe('move'); + }); + }); + it('records stylus altitude/azimuth on every pen stage (begin, end, cancel)', () => { const target = fakeTarget(); const { events } = activate({ target }); @@ -586,7 +827,7 @@ describe('createBrowserInputSource', () => { altitudeAngle: 0.8, azimuthAngle: 1.9, tool: InputTool.Pen, - button: 0, + // No button/buttonMask/metaState: version 3 writes those on a MOUSE begin/end only. view_tag: 'button', target: { text: 'OK', selector: 'button' }, }); @@ -634,12 +875,72 @@ describe('createBrowserInputSource', () => { expect(events[0]).not.toHaveProperty('azimuthAngle'); }); - it('passes the actual device button through (secondary and middle, not a hardcoded 0)', () => { + // input.md v3: `button` is the button that changed, in the SHARED cross-platform numbering (0 primary, + // 1 secondary, 2 middle, 3 back, 4 forward). The DOM numbers middle and secondary the other way round + // (1 middle, 2 secondary), so this is a MAP, not a passthrough. + it.each([ + [0, 0], // primary → primary + [1, 2], // DOM middle → contract middle (2) + [2, 1], // DOM secondary → contract secondary (1) + [3, 3], // back → back + [4, 4], // forward → forward + ])('maps DOM button %i to the contract button %i', (domButton, contractButton) => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ button: domButton })); + expect(events[0]?.button).toBe(contractButton); + }); + + it('omits button for a DOM value outside the known 0-4 range, never inventing 0', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ button: 5 })); + expect(events[0]).not.toHaveProperty('button'); + }); + + // buttonMask (v3): the buttons HELD, from the DOM's `MouseEvent.buttons` — already the wire's bit + // layout (1 primary, 2 secondary, 4 middle, 8 back, 16 forward), so it passes through unchanged. + it('passes buttonMask through unchanged (the DOM bit layout already matches the wire)', () => { const target = fakeTarget(); const { events } = activate({ target }); - target.emit('pointerdown', ptr({ button: 2 })); // secondary / right - target.emit('pointerdown', ptr({ pointerId: 2, button: 1 })); // middle - expect(events.map((e) => e.button)).toStrictEqual([2, 1]); + target.emit('pointerdown', ptr({ button: 0, buttons: 1 | 4 })); // primary + middle chorded + expect(events[0]?.buttonMask).toBe(5); + }); + + it('omits buttonMask when the event does not report a usable buttons mask', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ buttons: undefined })); + expect(events[0]).not.toHaveProperty('buttonMask'); + }); + + // metaState (v3): written on a mouse begin/end too (previously key entries only), through the same + // mapping the keydown path uses. + it('writes metaState on a mouse begin/end from the held modifier keys', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ shiftKey: true, ctrlKey: true })); + // META_CTRL_ON 4096 | META_SHIFT_ON 1 + expect(events[0]?.metaState).toBe(4096 | 1); + }); + + it('reports mouse metaState as 0 when no modifier is held, never as an absent field', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr()); + expect(events[0]?.metaState).toBe(0); + }); + + it('never writes button/buttonMask/metaState for a pen or touch pointer', () => { + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('pointerdown', ptr({ pointerType: 'pen', button: 0, buttons: 1 })); + target.emit('pointerdown', ptr({ pointerId: 2, pointerType: 'touch', button: 0, buttons: 1 })); + for (const event of events) { + expect(event).not.toHaveProperty('button'); + expect(event).not.toHaveProperty('buttonMask'); + expect(event).not.toHaveProperty('metaState'); + } }); it('rounds coordinates and omits non-finite ones', () => { @@ -683,6 +984,8 @@ describe('createBrowserInputSource', () => { force: 0.5, tool: InputTool.Mouse, button: 0, + buttonMask: 1, + metaState: 0, view_tag: 'div', target: { selector: 'div' }, }); @@ -703,16 +1006,20 @@ describe('createBrowserInputSource', () => { force: 0.5, tool: InputTool.Mouse, button: 0, + buttonMask: 1, + metaState: 0, }); }); // Not every pointer event carries `button` (a synthetic or partially-implemented event). Emitting the // key anyway would put `"button": null` on the wire, which reads as "button 0 was not pressed". - it('omits button when the event does not report one', () => { + it('omits button when the event does not report one, but still writes buttonMask/metaState', () => { const target = fakeTarget(); const { events } = activate({ target }); target.emit('pointerdown', ptr({ button: undefined })); expect(events[0]).not.toHaveProperty('button'); + expect(events[0]?.buttonMask).toBe(1); + expect(events[0]?.metaState).toBe(0); }); it('masks via the DEFAULT mask selector ([data-bugsee-hidden]) when none is configured', () => { @@ -915,6 +1222,8 @@ describe('createBrowserInputSource', () => { force: 0.5, tool: InputTool.Mouse, button: 0, + buttonMask: 1, + metaState: 0, view_tag: 'input', target: { masked: true }, }); @@ -998,4 +1307,264 @@ describe('createBrowserInputSource', () => { expect(target.count('pointerdown')).toBe(1); off(); }); + + // ---- wheel / scroll (version 3) ---- + describe('wheel', () => { + /** A `scheduleFrame` the test drives by hand: captures the callback instead of using a real timer. */ + function fakeFrame() { + const pending: Array<() => void> = []; + return { + scheduleFrame: (cb: () => void) => pending.push(cb), + flush: () => { + const cbs = pending.splice(0); + for (const cb of cbs) cb(); + }, + pendingCount: () => pending.length, + }; + } + + // `buttons: 0` (no button held) is the common case for a wheel turn; override to model a chorded one. + const wheel = (over: Record = {}) => ({ + deltaX: 0, + deltaY: 10, + deltaMode: 0, + clientX: 5, + clientY: 6, + buttons: 0, + ...over, + }); + + it('does not emit a wheel event synchronously — only once the frame is flushed', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel()); + expect(events).toStrictEqual([]); + frame.flush(); + expect(events).toHaveLength(1); + }); + + it('emits one scroll entry: x/y, signed scrollX/scrollY (no flip), scrollUnit, tool, metaState, buttonMask — no id or button', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaX: -3, deltaY: 12, ctrlKey: true })); + frame.flush(); + expect(events).toStrictEqual([ + { + type: 'scroll', + x: 5, + y: 6, + scrollX: -3, + scrollY: 12, + scrollUnit: 'pixel', + tool: InputTool.Mouse, + metaState: 4096, // META_CTRL_ON + buttonMask: 0, // no button held — the common case + }, + ]); + expect(events[0]).not.toHaveProperty('id'); + expect(events[0]).not.toHaveProperty('button'); + }); + + // The scenario the contract names explicitly: a non-zero mask on a `scroll` entry means only "a + // button happened to be down while the wheel turned" — never a press, which only begin/end carries. + it('carries buttonMask (not button) on a wheel turn with a button held', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ buttons: 1 })); // primary held while the wheel turns + frame.flush(); + expect(events[0]?.buttonMask).toBe(1); + expect(events[0]).not.toHaveProperty('button'); + }); + + it('omits buttonMask on a scroll entry when no event in the window reports a usable buttons mask', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ buttons: undefined })); + frame.flush(); + expect(events[0]).not.toHaveProperty('buttonMask'); + }); + + it('uses the buttonMask of the LATEST event in the coalesced frame, like position and metaState', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ buttons: 1 })); + target.emit('wheel', wheel({ buttons: 0 })); // released mid-frame + frame.flush(); + expect(events[0]?.buttonMask).toBe(0); + }); + + it('sums same-frame wheel deltas into one coalesced entry', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaX: 1, deltaY: 2 })); + target.emit('wheel', wheel({ deltaX: 3, deltaY: 4 })); + target.emit('wheel', wheel({ deltaX: 5, deltaY: 6 })); + frame.flush(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ scrollX: 9, scrollY: 12 }); + }); + + it('uses the position and modifiers of the LATEST event in the coalesced frame', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ clientX: 1, clientY: 1, shiftKey: true })); + target.emit('wheel', wheel({ clientX: 9, clientY: 9 })); // no modifier on the last event + frame.flush(); + expect(events[0]).toMatchObject({ x: 9, y: 9, metaState: 0 }); + }); + + it('schedules exactly one frame callback per coalescing window', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel()); + target.emit('wheel', wheel()); + target.emit('wheel', wheel()); + expect(frame.pendingCount()).toBe(1); + }); + + it('schedules a fresh frame for the deltas that follow a flush', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaY: 1 })); + frame.flush(); + target.emit('wheel', wheel({ deltaY: 2 })); + frame.flush(); + expect(events.map((e) => e.scrollY)).toStrictEqual([1, 2]); + }); + + it.each([ + [0, 'pixel'], + [1, 'line'], + [2, 'page'], + [99, 'pixel'], // unrecognised deltaMode falls back to the DOM's own default + ])('maps deltaMode %i to scrollUnit %s', (deltaMode, unit) => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaMode })); + frame.flush(); + expect(events[0]?.scrollUnit).toBe(unit); + }); + + it('ignores a wheel event with no usable delta on either axis (nothing to coalesce)', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaX: 0, deltaY: 0 })); + expect(frame.pendingCount()).toBe(0); + frame.flush(); + expect(events).toStrictEqual([]); + }); + + it('treats a non-numeric deltaX as 0 and still coalesces deltaY', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaX: undefined, deltaY: 7 })); + frame.flush(); + expect(events[0]).toMatchObject({ scrollX: 0, scrollY: 7 }); + }); + + it('treats a non-numeric deltaY as 0 and still coalesces deltaX', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaX: 3, deltaY: Number.NaN })); + frame.flush(); + expect(events[0]).toMatchObject({ scrollX: 3, scrollY: 0 }); + }); + + it('falls back to pixel scrollUnit when deltaMode is not a number at all', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ deltaMode: undefined })); + frame.flush(); + expect(events[0]?.scrollUnit).toBe('pixel'); + }); + + it('omits x/y on the scroll entry when the wheel event reports no usable position', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + target.emit('wheel', wheel({ clientX: undefined, clientY: undefined })); + frame.flush(); + expect(events[0]).not.toHaveProperty('x'); + expect(events[0]).not.toHaveProperty('y'); + expect(events[0]).toMatchObject({ scrollX: 0, scrollY: 10 }); + }); + + it('never lets a hostile wheel event reach the app', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const { events } = activate({ target, scheduleFrame: frame.scheduleFrame }); + const hostile = { + get deltaX(): number { + throw new Error('instrumented event'); + }, + }; + expect(() => target.emit('wheel', hostile)).not.toThrow(); + frame.flush(); + expect(events).toStrictEqual([]); + }); + + it('drops a pending accumulation on deactivate — a stale flush emits nothing', () => { + const target = fakeTarget(); + const frame = fakeFrame(); + const source = createBrowserInputSource({ target, scheduleFrame: frame.scheduleFrame }); + const events: InputEventDetail[] = []; + const off = source.onAny((_s, e) => events.push(e)); + target.emit('wheel', wheel()); + off(); // deactivate with an un-flushed accumulator + frame.flush(); // the stale rAF callback still fires — must be a no-op + expect(events).toStrictEqual([]); + }); + + it('registers a capture-phase, passive wheel listener, removed on deactivate', () => { + const target = fakeTarget(); + const { off } = activate({ target }); + expect(target.count('wheel')).toBe(1); + expect(target.optionsFor('wheel')).toEqual({ capture: true, passive: true }); + off(); + expect(target.count('wheel')).toBe(0); + expect(target.removeOptionsFor('wheel')).toEqual({ capture: true }); + }); + + it('defaults scheduleFrame to requestAnimationFrame when the global exists', () => { + const target = fakeTarget(); + const raf = vi.fn((cb: FrameRequestCallback) => { + cb(0); + return 0; + }); + vi.stubGlobal('requestAnimationFrame', raf); + const { events } = activate({ target }); // no scheduleFrame injected → the real default + target.emit('wheel', wheel()); + expect(raf).toHaveBeenCalledTimes(1); + expect(events).toHaveLength(1); + }); + + it('falls back to a timeout when requestAnimationFrame does not exist', () => { + vi.useFakeTimers(); + try { + vi.stubGlobal('requestAnimationFrame', undefined); + const target = fakeTarget(); + const { events } = activate({ target }); + target.emit('wheel', wheel()); + expect(events).toStrictEqual([]); + vi.advanceTimersByTime(16); + expect(events).toHaveLength(1); + } finally { + vi.useRealTimers(); + } + }); + }); }); diff --git a/packages/browser/src/input-source.ts b/packages/browser/src/input-source.ts index 35acc2b7..fc7050e7 100644 --- a/packages/browser/src/input-source.ts +++ b/packages/browser/src/input-source.ts @@ -30,16 +30,23 @@ import { penAngles } from './pen-angles'; // Pointer Events are supported by every browser this SDK targets. `click` is deliberately dropped: it // is a synthesis of a down and an up we now record directly. // -// `pointermove` is captured for PEN gestures only. A move stream is orders of magnitude larger than the -// press stream and would dominate the capture ring, and for a mouse or a finger the viewer's gesture -// classification runs off the first and last event, which down/up already provide — so those drags still -// render as their two endpoints. A pen stroke is different: its pressure and tilt vary along the path, -// and that path is what the viewer's pen glyph draws. A pen move is recorded Android-canonically -// (`InputEventGenerationHelper.registerMoveEvent`): only while the gesture is in contact (never a -// hovering pen), only when something recorded changed, at the browser's own frame-aligned pointermove -// rate. Its identity lives on the gesture's `begin`, so a move carries no target and no button. A gesture -// that began on a MASKED target records no moves at all: a pen path there is handwriting — a signature, -// a PIN drawn on a pad. +// `pointermove` is captured for PEN gestures, plus (version 3) two MOUSE cases on the same change-gated, +// frame-aligned terms — everything else (finger, touchpad-as-mouse taps with no drag) still renders as its +// two endpoints, because a move stream is orders of magnitude larger than the press stream: +// - DRAG: `pointermove` while any button is held. Recorded Android-canonically +// (`InputEventGenerationHelper.registerMoveEvent`): only when something recorded (x/y/force) changed, +// at the browser's own frame-aligned pointermove rate. Its identity lives on the gesture's `begin`, so +// a move carries no target and no `button` — same as a pen move — but it DOES carry `buttonMask` (see +// MOUSE BUTTONS below). A gesture that began on a MASKED target records no moves at all (mouse or +// pen): a path there is handwriting — a signature, a PIN drawn on a pad — or, for a mouse, still +// traces exactly where the pointer went over that field. +// - HOVER: `pointermove` with no button held is sampled at about {@link HOVER_SAMPLE_INTERVAL_MS} — a +// GUESS (`recording-input-presentation` open item 31; the recording-size cost is unmeasured), so it is +// a named constant, never a bare literal. A hover move carries no target and no `button`, like a pen +// move, plus `buttonMask` (reads `0` — hover is exactly "no button held"); its `id` identifies one +// continuous hover run and is freed the moment a real press begins, so it can never collide with — +// or, Android's own documented bug, get reused by — a press id. +// A pen move: only while the gesture is in contact (never a hovering pen — that stays future work). // // `keyup` is a DELIBERATE divergence from the mobile SDKs, not an oversight: a press is recorded once, on // the way down. Android emits both edges because it has them for free; on the web a keyup doubles the @@ -48,16 +55,38 @@ import { penAngles } from './pen-angles'; // // Emitted per interaction: // pointerdown → { type:'begin', id, x, y, force, majorRadius?, minorRadius?, altitudeAngle?, -// azimuthAngle?, tool, button, view* } (the two angles: pen only — see pen-angles.ts) -// pointermove → { type:'move', id, x, y, force, majorRadius?, minorRadius?, altitudeAngle?, -// azimuthAngle?, tool:Pen } (pen gestures only — see above) +// azimuthAngle?, tool, button?, buttonMask?, metaState?, view* } +// (angles: pen only; button/buttonMask/metaState: mouse only — see below) +// pointermove → { type:'move', id?, x, y, force, majorRadius?, minorRadius?, altitudeAngle?, +// azimuthAngle?, tool:Pen|Mouse, buttonMask? } +// (pen: in contact, no buttonMask; mouse: drag or sampled hover, buttonMask but no +// `button` — a move never changes one) // pointerup → { type:'end', …the same, closing the gesture id } // pointercancel → { type:'end', …the gesture was aborted by the browser } +// wheel → { type:'scroll', x?, y?, scrollX, scrollY, scrollUnit, tool:Mouse, metaState, +// buttonMask? } +// (no `id` — a wheel occurrence opens/closes no gesture; same-frame deltas coalesced) // keydown → { type:'keydown', tool:Key, id, key, keyCode, metaState, view* } // `type:'keydown'` is Android's InputEventStage.KeyDown (interception/input/InputEventStage.java) — // distinct from the pointer 'begin'/'end' stages, so a consumer can tell a key press apart from a // pointer-down without inspecting `tool`. // +// MOUSE BUTTONS (version 3, `input.md`): `button` is the button that CHANGED, in the wire's shared +// cross-platform numbering — 0 primary, 1 secondary, 2 middle, 3 back, 4 forward — mouse `begin`/`end` +// only, and OMITTED (never `0`) for an unrecognised button or a pen/touch pointer. The DOM's own +// `MouseEvent.button` numbers middle and secondary the OTHER WAY ROUND (0 primary, 1 middle, 2 +// secondary), so it is mapped, never passed through. `buttonMask` is the buttons HELD, from +// `MouseEvent.buttons` — the DOM's bit layout already matches the wire's (1 primary, 2 secondary, 4 +// middle, 8 back, 16 forward), so it passes straight through — and, unlike `button`, it is written on +// EVERY mouse stage the platform can read it on: `begin`/`end`, a drag or hover `move`, and `scroll` (a +// native `WheelEvent` inherits `buttons` from `MouseEvent`). A consumer must not read a non-zero +// `buttonMask` on a `scroll` entry as a press — only as "a button happened to be down while the wheel +// turned"; only `begin`/`end` ever carries `button`, because only a press or release changes one. +// `metaState` is written on a mouse `begin`/`end` AND `scroll` (previously key entries only), through the +// same Android-metaState mapping the keydown path already uses — reused, not re-derived. Ctrl-plus-wheel +// is a zoom, not a scroll, which is worth being able to tell apart, so it stays on `scroll` too even +// though the contract's v3 prose did not explicitly call it out there. +// // DELIBERATELY NOT ON THIS STREAM: `change` / `submit` / `focus`. They are not device presses — they are // STATE-CHANGE DOM signals, and none of the three is (or structurally can be) a member of Android's // InputEventStage, which is exactly unknown|begin|move|end|scroll|keydown|keyup and is produced by SDKs @@ -72,6 +101,13 @@ import { penAngles } from './pen-angles'; // read. `describeTarget` masks anything the SHARED sensitive-input definition matches, or anything // under the app's mask selector, to `{ tag, masked }`. And a keystroke aimed at either is dropped // OUTRIGHT — see THE SECURE-FIELD EXCLUSION in the keydown handler. +// +// Mouse HOVER sampling is DELIBERATELY not masked-target-gated, unlike a drag move: a drag suppresses +// itself entirely for a gesture that began on a masked target (see the DRAG bullet above) because it +// carries no target of its own to check — its privacy rides on where it started. Hover has no "began" to +// key off (no press ever opens it), and it carries only a bare position, exactly like every other pointer +// coordinate this source already records over a masked field (see the secure-field pointer test below) — +// a screen recording already shows the cursor there. Withholding it would be inconsistent, not safer. /** A structural, PII-safe description of an interaction's target element. */ export interface TargetDescriptor { @@ -200,6 +236,14 @@ export interface BrowserInputEnv { target?: InputEventTarget; /** Elements (or subtrees) to fully mask to `{ tag, masked }`. Default `[data-bugsee-hidden]`. */ maskSelector?: string; + /** Wall-clock source for hover sampling. Injectable for tests. Default `Date.now`. */ + now?: () => number; + /** + * Schedules a callback for "the next animation frame" — used to coalesce same-frame wheel deltas into + * one `scroll` entry. Injectable for tests. Default `requestAnimationFrame`, falling back to a 16ms + * timeout where it does not exist (SSR / non-browser contexts this source may still be constructed in). + */ + scheduleFrame?: (callback: () => void) => void; } // Capture phase so we observe before the app's bubbling handlers; passive so the browser knows we never @@ -214,7 +258,9 @@ const INTERACTIONS = [ 'keydown', ] as const; -/** The fields whose change makes a pen move worth recording (everything a move carries but its identity). */ +/** The fields whose change makes a pen or mouse-drag move worth recording (everything a move carries but + * its identity). Irrelevant fields for a given tool (e.g. a mouse's geometry/angles) are `undefined` on + * both sides of the comparison, so they never force a spurious move. */ const MOVE_FIELDS = [ 'x', 'y', @@ -225,6 +271,62 @@ const MOVE_FIELDS = [ 'azimuthAngle', ] as const; +/** + * How often a hovering (no button held) mouse pointer is sampled into a `move` entry. A NAMED constant, + * not a literal: `recording-input-presentation`'s open item 31 flags the rate as a guess whose effect on + * recording size is unmeasured, so it is meant to be tuned (or dropped) from one place once it is. + */ +export const HOVER_SAMPLE_INTERVAL_MS = 100; // ~10/s + +/** + * DOM `MouseEvent.button` (the button that changed) → the wire's shared cross-platform numbering + * (`input.md` v3): 0 primary, 1 secondary, 2 middle, 3 back, 4 forward. The DOM numbers middle and + * secondary the other way round (1 middle, 2 secondary), so this is a MAP, not a passthrough. A DOM value + * outside this table (or non-numeric) has no entry, which is exactly "omit it" — never invent `0`. + */ +const MOUSE_BUTTON_MAP: Readonly> = { 0: 0, 1: 2, 2: 1, 3: 3, 4: 4 }; + +/** DOM `WheelEvent.deltaMode` → the wire's `scrollUnit`. Unrecognised/missing falls back to `'pixel'` — + * the DOM's own default (`WheelEvent.DOM_DELTA_PIXEL` is `0`). */ +const SCROLL_UNIT_BY_MODE: Readonly> = { 0: 'pixel', 1: 'line', 2: 'page' }; + +/** `buttonMask` alone — every mouse stage the platform can read `MouseEvent.buttons` on writes it: + * `begin`/`end`, `move` (drag and sampled hover alike) and `scroll`. `button` (the button that CHANGED) + * is narrower: only a press or release changes a button, so a move or a wheel turn never carries one — + * see {@link mouseButtonFields}, used on `begin`/`end` only. */ +function buttonMaskFor(e: ButtonsEventLike): Partial { + const mask = typeof e.buttons === 'number' && Number.isFinite(e.buttons) ? e.buttons : undefined; + return mask !== undefined ? { buttonMask: mask } : {}; +} + +/** `button`/`buttonMask` for a mouse pointer BEGIN/END — never emitted for any other tool (pen/touch + * carry no button fields at all, per the wire contract), and never for a move or a wheel turn (those get + * {@link buttonMaskFor} alone — no `button`, since neither changes one). */ +function mouseButtonFields(e: PointerEventLike): Partial { + const mapped = typeof e.button === 'number' ? MOUSE_BUTTON_MAP[e.button] : undefined; + return { + ...(mapped !== undefined ? { button: mapped } : {}), + ...buttonMaskFor(e), + }; +} + +/** The modifier-key surface both `androidMetaState` call sites (keydown, mouse begin/end) read. */ +interface ModifierEventLike { + ctrlKey?: unknown; + shiftKey?: unknown; + altKey?: unknown; + metaKey?: unknown; +} + +/** Normalise a possibly-`unknown`-typed modifier surface into the booleans `androidMetaState` expects. */ +const metaStateFor = (e: ModifierEventLike): number => + androidMetaState({ + ctrlKey: e.ctrlKey === true, + shiftKey: e.shiftKey === true, + altKey: e.altKey === true, + metaKey: e.metaKey === true, + }); + /** `pointerType` → the wire tool. An unrecognised (but present) type is a real device we cannot name. */ const TOOL_BY_POINTER_TYPE: Readonly> = { touch: InputTool.Touch, @@ -279,7 +381,15 @@ function contact(e: PointerEventLike, tool: InputTool): Partial void): void { + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(callback); + } else { + setTimeout(callback, 16); + } +} + class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { readonly name = 'browser-input'; readonly #target: InputEventTarget | undefined; readonly #mask: string; + readonly #now: () => number; + readonly #scheduleFrame: (callback: () => void) => void; /** Open gestures: pointerId → the wire `id` its stages share. */ readonly #openGestures = new Map(); /** Open PEN gestures whose moves are recorded: pointerId → the last entry emitted for that gesture. */ readonly #penPaths = new Map(); + /** Open MOUSE DRAGS whose moves are recorded (a button held on an unmasked target): pointerId → the + * last entry emitted for that gesture — same role as {@link #penPaths}, kept separate so a pen and a + * mouse sharing a pointerId (never happens in practice, but the ids are caller-controlled) can't + * collide. */ + readonly #mouseDragPaths = new Map(); + /** The `id` of the hover run in progress for a pointerId, if any. A press frees it (deleted on + * `begin`), so a hover id can never be reused across — or by — a press, unlike Android's mouse hover + * (`recording-input-presentation` open item 10, a documented bug this design deliberately avoids). */ + readonly #hoverIds = new Map(); + /** Wall-clock ms of the last emitted hover sample, per pointerId — the ~10/s throttle. */ + readonly #lastHoverSampleAt = new Map(); + /** Same-frame wheel deltas awaiting their coalesced `scroll` entry, or `null` between frames. */ + #wheelAccum: WheelAccumulator | null = null; + #wheelFrameScheduled = false; #nextGestureId = 1; // Each handler is wrapped by #dispatch: it builds the entry (returning undefined to skip) inside a @@ -311,7 +470,7 @@ class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { // app's own dispatch. A build throw drops the whole event — fail-safe, never a partial/unmasked leak. readonly #handlers: Record<(typeof INTERACTIONS)[number], (event: Event) => void> = { pointerdown: this.#dispatch((event) => this.#pointer(event, 'begin')), - pointermove: this.#dispatch((event) => this.#penMove(event)), + pointermove: this.#dispatch((event) => this.#move(event)), pointerup: this.#dispatch((event) => this.#pointer(event, 'end')), pointercancel: this.#dispatch((event) => this.#pointer(event, 'end')), keydown: this.#dispatch((event) => { @@ -342,10 +501,17 @@ class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { }), }; - constructor(target: InputEventTarget | undefined, mask: string) { + constructor( + target: InputEventTarget | undefined, + mask: string, + now: () => number, + scheduleFrame: (callback: () => void) => void, + ) { super(); this.#target = target; this.#mask = mask; + this.#now = now; + this.#scheduleFrame = scheduleFrame; } /** A fresh id for one key press — it shares the gesture-id sequence so no key entry can collide with @@ -367,28 +533,52 @@ class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { #pointer(event: Event, stage: 'begin' | 'end'): InputEventDetail { const e = event as PointerEventLike; const tool = toolFor(e.pointerType); + const isMouse = tool === InputTool.Mouse; const desc = describeTarget(e.target, this.#mask); const detail: InputEventDetail = { id: this.#gestureId(e.pointerId, stage), type: stage, ...contact(e, tool), tool, - ...(typeof e.button === 'number' ? { button: e.button } : {}), + // button/buttonMask/metaState: mouse begin/end only — pen and touch never carry a button field. + ...(isMouse ? mouseButtonFields(e) : {}), + ...(isMouse ? { metaState: metaStateFor(e) } : {}), ...targetFields(desc), }; - if (stage === 'end') this.#penPaths.delete(e.pointerId); - else if (tool === InputTool.Pen && desc.masked !== true) - this.#penPaths.set(e.pointerId, detail); + if (stage === 'end') { + this.#penPaths.delete(e.pointerId); + this.#mouseDragPaths.delete(e.pointerId); + } else { + if (tool === InputTool.Pen && desc.masked !== true) this.#penPaths.set(e.pointerId, detail); + if (isMouse) { + // A press ENDS whatever hover run was in progress for this pointer. Cleared here, at `begin` + // (not `end`): every move for the rest of the press goes through the drag path, never hover, so + // clearing once, up front, is enough for the next hover sample — after release — to always open + // a fresh id. + this.#hoverIds.delete(e.pointerId); + this.#lastHoverSampleAt.delete(e.pointerId); + if (desc.masked !== true) this.#mouseDragPaths.set(e.pointerId, detail); + } + } return detail; } + /** Routes a `pointermove` to the pen-stroke or mouse (drag/hover) path; every other tool (touch, + * unknown, other) records no moves, unchanged from before version 3. */ + #move(event: Event): InputEventDetail | undefined { + const e = event as PointerEventLike; + const tool = toolFor(e.pointerType); + if (tool === InputTool.Pen) return this.#penMove(e); + if (tool === InputTool.Mouse) return this.#mouseMove(e); + return undefined; + } + /** A move of an open, unmasked pen gesture — or `undefined` when there is none, or nothing changed. */ - #penMove(event: Event): InputEventDetail | undefined { - // Every mouse move in the document lands here, so the common case must cost one size check. + #penMove(e: PointerEventLike): InputEventDetail | undefined { + // Every pen move in the document lands here, so the common case must cost one size check. if (this.#penPaths.size === 0) return undefined; - const e = event as PointerEventLike; const last = this.#penPaths.get(e.pointerId); - if (last === undefined || toolFor(e.pointerType) !== InputTool.Pen) return undefined; + if (last === undefined) return undefined; const detail: InputEventDetail = { id: last.id, type: 'move', @@ -400,6 +590,110 @@ class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { return detail; } + /** A mouse `pointermove`: a DRAG move while any button is held, else a sampled HOVER move. */ + #mouseMove(e: PointerEventLike): InputEventDetail | undefined { + const buttonsHeld = typeof e.buttons === 'number' && e.buttons !== 0; + return buttonsHeld ? this.#mouseDragMove(e) : this.#mouseHoverMove(e); + } + + /** A move of an open, unmasked mouse drag — same change-gating as a pen move (see {@link #penMove}). + * Carries `buttonMask` (the platform can read it here) but no `button` — a move never changes one. */ + #mouseDragMove(e: PointerEventLike): InputEventDetail | undefined { + if (this.#mouseDragPaths.size === 0) return undefined; + const last = this.#mouseDragPaths.get(e.pointerId); + if (last === undefined) return undefined; + const detail: InputEventDetail = { + id: last.id, + type: 'move', + ...contact(e, InputTool.Mouse), + tool: InputTool.Mouse, + ...buttonMaskFor(e), + }; + if (MOVE_FIELDS.every((field) => detail[field] === last[field])) return undefined; + this.#mouseDragPaths.set(e.pointerId, detail); + return detail; + } + + /** A sampled hover move (no button held): throttled to ~{@link HOVER_SAMPLE_INTERVAL_MS}, carrying no + * target and no `button` — same shape as a pen move, plus `buttonMask` (reads as `0` when the event + * reports a usable mask — hover is exactly "no button held" — omitted like any other field the event + * simply didn't report). */ + #mouseHoverMove(e: PointerEventLike): InputEventDetail | undefined { + const now = this.#now(); + const lastSampleAt = this.#lastHoverSampleAt.get(e.pointerId); + if (lastSampleAt !== undefined && now - lastSampleAt < HOVER_SAMPLE_INTERVAL_MS) + return undefined; + this.#lastHoverSampleAt.set(e.pointerId, now); + const id = this.#hoverIds.get(e.pointerId) ?? String(this.#nextGestureId++); + this.#hoverIds.set(e.pointerId, id); + return { + id, + type: 'move', + ...contact(e, InputTool.Mouse), + tool: InputTool.Mouse, + ...buttonMaskFor(e), + }; + } + + /** The passive, capture-phase `wheel` listener: accumulates this frame's deltas (never emits directly) + * and schedules {@link #flushWheel} once per frame. Not routed through {@link #dispatch} — it never + * emits synchronously — but the same observe-only guarantee applies via its own try/catch. */ + readonly #onWheel = (event: Event): void => { + try { + const e = event as WheelEventLike; + const dx = typeof e.deltaX === 'number' && Number.isFinite(e.deltaX) ? e.deltaX : 0; + const dy = typeof e.deltaY === 'number' && Number.isFinite(e.deltaY) ? e.deltaY : 0; + if (dx === 0 && dy === 0) return; // nothing to coalesce — also guards a hostile/empty event + const unit = + typeof e.deltaMode === 'number' ? (SCROLL_UNIT_BY_MODE[e.deltaMode] ?? 'pixel') : 'pixel'; + const acc: WheelAccumulator = this.#wheelAccum ?? { + scrollX: 0, + scrollY: 0, + scrollUnit: unit, + metaState: 0, + }; + acc.scrollX += dx; + acc.scrollY += dy; + acc.scrollUnit = unit; + acc.metaState = metaStateFor(e); + const xField = coord(e.clientX, 'x'); + const yField = coord(e.clientY, 'y'); + if ('x' in xField) acc.x = xField.x; + if ('y' in yField) acc.y = yField.y; + // buttonMask: a native WheelEvent inherits `buttons` from MouseEvent, so a chorded wheel turn (a + // button happened to be down while it turned — never a press, which only begin/end carries) is + // told from a plain one. Same latest-wins, omit-if-unreported treatment as x/y. + const maskField = buttonMaskFor(e); + if ('buttonMask' in maskField) acc.buttonMask = maskField.buttonMask; + this.#wheelAccum = acc; + if (!this.#wheelFrameScheduled) { + this.#wheelFrameScheduled = true; + this.#scheduleFrame(this.#flushWheel); + } + } catch { + // Observe-only: a throwing event must never disrupt the application. + } + }; + + /** Emits one coalesced `scroll` entry for the frame's accumulated deltas, if any. */ + readonly #flushWheel = (): void => { + this.#wheelFrameScheduled = false; + const acc = this.#wheelAccum; + this.#wheelAccum = null; + if (acc === null) return; + this.emit('input', { + type: 'scroll', + ...(acc.x !== undefined ? { x: acc.x } : {}), + ...(acc.y !== undefined ? { y: acc.y } : {}), + scrollX: acc.scrollX, + scrollY: acc.scrollY, + scrollUnit: acc.scrollUnit, + tool: InputTool.Mouse, + metaState: acc.metaState, + ...(acc.buttonMask !== undefined ? { buttonMask: acc.buttonMask } : {}), + }); + }; + #dispatch(build: (event: Event) => InputEventDetail | undefined): (event: Event) => void { return (event) => { try { @@ -415,16 +709,24 @@ class BrowserInputSource extends InterceptorBase<{ input: InputEventDetail }> { for (const type of INTERACTIONS) { this.#target?.addEventListener(type, this.#handlers[type], ADD_OPTIONS); } + this.#target?.addEventListener('wheel', this.#onWheel, ADD_OPTIONS); } protected override onDeactivate(): void { for (const type of INTERACTIONS) { this.#target?.removeEventListener(type, this.#handlers[type], REMOVE_OPTIONS); } + this.#target?.removeEventListener('wheel', this.#onWheel, REMOVE_OPTIONS); // Gestures cannot span a deactivation: their `up` will never be observed, so holding the ids would - // leak one Map entry per pointer and let a stale id resurface on the next activation. + // leak one Map entry per pointer and let a stale id resurface on the next activation. A pending wheel + // accumulator is dropped the same way — no listener can feed it again until the next activate(), and + // a stale already-scheduled frame callback finds it `null` and flushes nothing. this.#openGestures.clear(); this.#penPaths.clear(); + this.#mouseDragPaths.clear(); + this.#hoverIds.clear(); + this.#lastHoverSampleAt.clear(); + this.#wheelAccum = null; } } @@ -433,5 +735,10 @@ export function createBrowserInputSource( ): Interceptor<{ input: InputEventDetail }> { const target = env.target ?? (typeof document !== 'undefined' ? (document as InputEventTarget) : undefined); - return new BrowserInputSource(target, env.maskSelector ?? '[data-bugsee-hidden]'); + return new BrowserInputSource( + target, + env.maskSelector ?? '[data-bugsee-hidden]', + env.now ?? (() => Date.now()), + env.scheduleFrame ?? defaultScheduleFrame, + ); } diff --git a/packages/core/src/bundle-assembler.test.ts b/packages/core/src/bundle-assembler.test.ts index 596af502..404b36f8 100644 --- a/packages/core/src/bundle-assembler.test.ts +++ b/packages/core/src/bundle-assembler.test.ts @@ -196,10 +196,15 @@ describe('assembleBundle — manifest.json & files', () => { const out = unzip(assembleBundle(request(), captured, context()).body); expect(out.names).toContain('input.json'); expect(out.names).toContain('events.user.json'); - expect(JSON.parse(out.text('input.json'))).toEqual([ - { timestamp: 2, id: 'g1', type: 'begin', x: 4, y: 5, tool: 1 }, - { timestamp: 3, id: 'g1', type: 'end', x: 4, y: 5, tool: 1 }, - ]); + // `input.json` v3 (specs/sdk/reporting/bundle/input.md): `{ version, events }`, not a bare array — a + // consumer keys its button/scroll decoding on `version`. + expect(JSON.parse(out.text('input.json'))).toEqual({ + version: 3, + events: [ + { timestamp: 2, id: 'g1', type: 'begin', x: 4, y: 5, tool: 1 }, + { timestamp: 3, id: 'g1', type: 'end', x: 4, y: 5, tool: 1 }, + ], + }); // The user stream carries the app's event and NOTHING the SDK observed. expect(JSON.parse(out.text('events.user.json'))).toEqual([ { timestamp: 1, name: 'checkout', params: { t: 9 } }, @@ -238,6 +243,23 @@ describe('assembleBundle — manifest.json & files', () => { expect(JSON.parse(out.text('network.json'))).toEqual([{ url: 'u' }]); }); + // input.md v3: input.json is `{ version, events }`; version 3 is stamped for THIS stream only — every + // other file type is unaffected, whether bare-array (network) or its own wrapper (performance). + it('stamps input.json with the input-stream version 3, and no other file type', () => { + const captured = new Map([ + [ + 'input', + [entry('input', 1, { timestamp: 1, id: 'g1', type: 'begin', x: 1, y: 2, tool: 2 })], + ], + ['performance', [entry('performance', 2, { traceId: 't', name: 'a' })]], + ['network', [entry('network', 3, { url: 'u' })]], + ]); + const out = unzip(assembleBundle(request(), captured, context()).body); + expect(JSON.parse(out.text('input.json'))).toMatchObject({ version: 3 }); + expect(JSON.parse(out.text('performance.json'))).not.toHaveProperty('version'); + expect(Array.isArray(JSON.parse(out.text('network.json')))).toBe(true); + }); + it('serializes the profile file type as the single bare V8 CPU profile object (not an array)', () => { const cpuProfile = { nodes: [{ id: 1 }], diff --git a/packages/core/src/bundle-assembler.ts b/packages/core/src/bundle-assembler.ts index d6af53b2..0685a248 100644 --- a/packages/core/src/bundle-assembler.ts +++ b/packages/core/src/bundle-assembler.ts @@ -4,6 +4,7 @@ import { DEFAULT_FILENAMES, type EnvironmentEnvelope, type FileType, + INPUT_STREAM_VERSION, MANIFEST_JSON_FILENAME, MANIFEST_VERSION, type ManifestFileEntry, @@ -81,11 +82,13 @@ function fileNameForType(type: FileType): string { return type === 'attachment' ? 'attachment' : DEFAULT_FILENAMES[type]; } -// Per-type JSON shape. Most file types serialize to a top-level ARRAY of payloads. Two exceptions: -// `performance.json` = `{ transactions: [...] }` (§711/§8.8), and `profile.json` = the SINGLE bare V8 CPU -// profile object (.cpuprofile — DevTools/speedscope-loadable). Binary streams (§8.4: replay/screenshot/ -// attachment) are platform-tier provider concerns; when those land the assembler branches further -// (passing a Uint8Array `data` through). +// Per-type JSON shape. Most file types serialize to a top-level ARRAY of payloads. Three exceptions: +// `performance.json` = `{ transactions: [...] }` (§711/§8.8); `profile.json` = the SINGLE bare V8 CPU +// profile object (.cpuprofile — DevTools/speedscope-loadable); and `input.json` = the wire spec's own +// versioned envelope, `{ version, events: [...] }` (`specs/sdk/reporting/bundle/input.md`) — a consumer +// keys its decoding of `button`/`buttonMask`/`metaState`/scroll on `version`, so the file must carry it. +// Binary streams (§8.4: replay/screenshot/attachment) are platform-tier provider concerns; when those +// land the assembler branches further (passing a Uint8Array `data` through). function serializeFileData(type: FileType, payloads: unknown[]): unknown { if (type === 'performance') { return { transactions: payloads }; @@ -96,6 +99,9 @@ function serializeFileData(type: FileType, payloads: unknown[]): unknown { // the key when 0), so payloads[0] is always present here. return payloads[0]; } + if (type === 'input') { + return { version: INPUT_STREAM_VERSION, events: payloads }; + } return payloads; } diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index 06dc55bf..01778c46 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -66,11 +66,17 @@ export type InputTool = (typeof InputTool)[keyof typeof InputTool]; * `tool`/`view`/`view_id`/`view_tag`, grouped into gestures by `id`, staged by `type`), so a web * interaction and a mobile touch render through ONE path. * - * SDK-AHEAD-OF-CONTRACT — `button`, `key` and `target` are NOT in the viewer's `RecordingTouchEvent` - * today. Desktop input has buttons and keyboards that the mobile-shaped contract - * has no room for, and JSON consumers ignore unknown keys, so the SDK emits them as an additive - * superset rather than waiting on backend/viewer adoption. Nothing may be moved OUT of the contract - * fields into these; they are additions only. + * VERSION 3 (`input.md`): `button` is the button that CHANGED, in the shared cross-platform numbering + * (0 primary, 1 secondary, 2 middle, 3 back, 4 forward — the browser tier maps the DOM's own numbering + * onto it, never passes it through); `buttonMask` is the buttons HELD, tool = mouse only (the DOM's + * `buttons` bitmask already matches this layout); `metaState` is now also written on mouse press/release + * (previously key entries only); `scrollX`/`scrollY`/`scrollUnit` carry a mouse wheel occurrence. + * + * SDK-AHEAD-OF-CONTRACT — `key` and `target` are NOT in the viewer's `RecordingTouchEvent` today. + * Desktop input has a keyboard that the mobile-shaped contract has no room for, and JSON consumers + * ignore unknown keys, so the SDK emits them as an additive superset rather than waiting on + * backend/viewer adoption. Nothing may be moved OUT of the contract fields into these; they are + * additions only. */ export interface InputEvent { /** Wall-clock ms. */ @@ -124,8 +130,26 @@ export interface InputEvent { * `@bugsee/browser`'s `keycodes.ts`. */ keyCode?: number; - /** Android `KeyEvent` modifier bitmask for a key entry (`META_SHIFT_ON` etc.); 0 when none are held. */ + /** + * Android `KeyEvent` modifier bitmask (`META_SHIFT_ON` etc.); 0 when none are held. A key entry always + * carries it. **Version 3:** also written on a mouse `begin`/`end`, from `ctrlKey`/`shiftKey`/ + * `altKey`/`metaKey` through the same mapping, so a shift-click is distinguishable from a plain one. + */ metaState?: number; + /** + * Which device button changed, in the SHARED cross-platform numbering (`input.md` v3): `0` primary, + * `1` secondary, `2` middle, `3` back, `4` forward. Mouse `begin`/`end` only; omitted when the button + * is unrecognised (never sent as `0`, which means primary) and always omitted for a pen or touch + * pointer. The browser tier MAPS the DOM's own numbering (0 primary, 1 **middle**, 2 **secondary** — + * the other way round) rather than passing it through. + */ + button?: number; + /** + * Buttons HELD after the event, as bits `1 << button` (1 primary, 2 secondary, 4 middle, 8 back, + * 16 forward). Mouse `begin`/`end` only (`input.md` v3). The DOM's `MouseEvent.buttons` is already + * this exact bit layout, so the browser tier passes it through unchanged. + */ + buttonMask?: number; /** * Which display the input happened on (Android multi-display). Declared for parity and DELIBERATELY * never set by the web tier: a document belongs to exactly one display and JS cannot observe which, @@ -133,10 +157,18 @@ export interface InputEvent { * WebView receiver, which does know, is the tier that can fill it. */ displayId?: number; + /** + * Signed scroll amount from a mouse wheel occurrence (`input.md` v3, `type: 'scroll'`), the browser's + * own `deltaX`/`deltaY` convention (positive Y = content scrolled down, positive X = right) — the wire + * already matches it, so no sign flip. Coalesced: the browser tier sums same-frame wheel deltas into + * one entry rather than one per DOM event. + */ + scrollX?: number; + scrollY?: number; + /** Unit of {@link scrollX}/{@link scrollY}: `'pixel'` \| `'line'` \| `'page'` (from `WheelEvent.deltaMode`). */ + scrollUnit?: string; // ---- SDK-ahead-of-contract (see the note above) ---- - /** Which device button was pressed (DOM `MouseEvent.button`: 0 primary, 1 middle, 2 secondary…). */ - button?: number; /** * The key's IDENTITY as the DOM names it — only ever a NAMED key, or a character reached as a * shortcut; typed characters are never recorded. Kept ALONGSIDE the mobile-shaped `keyCode` because it diff --git a/packages/protocol/src/constants.ts b/packages/protocol/src/constants.ts index b3de551f..5a568be0 100644 --- a/packages/protocol/src/constants.ts +++ b/packages/protocol/src/constants.ts @@ -3,6 +3,16 @@ /** Manifest schema version emitted by the JS SDK (§0.16 / §8). Backend accepts v1 (mobile) + v2. */ export const MANIFEST_VERSION = 2; +/** + * `input.json`'s OWN stream version (`specs/sdk/reporting/bundle/input.md`), independent of + * {@link MANIFEST_VERSION}: the wire spec is cross-platform and a consumer keys its decoding of the + * file's `button`/`buttonMask`/`metaState`/scroll fields on THIS number, not the manifest's. Version 3 + * fixes the mouse `button` numbering across platforms, adds `buttonMask`, writes `metaState` on a mouse + * press/release, and adds `scrollX`/`scrollY`/`scrollUnit` — all landing together in the JS SDK's mouse + * capture, so this stream alone moves from unversioned to 3. + */ +export const INPUT_STREAM_VERSION = 3; + // Root bundle files (§8.4). export const REQUEST_JSON_FILENAME = 'request.json'; export const MANIFEST_JSON_FILENAME = 'manifest.json'; diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index dcf2d37e..57ac055d 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -7,6 +7,7 @@ export { BUNDLE_FILE_SUFFIX, DEFAULT_FILENAMES, type FileType, + INPUT_STREAM_VERSION, MANIFEST_JSON_FILENAME, MANIFEST_VERSION, NAME_SOURCE_ATTRIBUTE,