diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a20ea2..defc257 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: "20.x" + node-version: "24.x" cache: npm - name: Install system test dependencies diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8ab38b1..9953aa1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: "20.x" + node-version: "24.x" cache: "npm" - name: Install dependencies @@ -63,7 +63,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: "20.x" + node-version: "24.x" cache: "npm" registry-url: "https://registry.npmjs.org" diff --git a/CLAUDE.md b/CLAUDE.md index 6409a2f..83b4b67 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ src/ ├── commands/ # One file per CLI command (install, start, stop, exec, diff, pr, clean) ├── browser/ # agent-browser CLI wrappers (session, capture, interact, navigate) ├── server/ # Dev server detection, startup, port waiting -├── session/state.ts # .session.json lifecycle (save/load/clear) +├── session/ # Global registry, per-session leases, selection, manifests ├── session/metadata.ts # Persistent per-session metadata (branch, commit) for PR matching ├── artifacts/ # Output generation (viewer.html, SUMMARY.md, PR format) └── utils/ # Config, exec helpers, port utils, error patterns, GitHub API @@ -30,16 +30,16 @@ src/ - **ESM only** — all imports MUST use `.js` extensions: `import { foo } from '../utils/config.js'` - **Build before test** — CLI runs from `dist/`, always `npm run build` after code changes -- **agent-browser** — external peer dependency (Rust CLI + Node daemon). All browser commands go through `ab()` in `utils/exec.ts` which calls `agent-browser ` via `execSync` -- **Session state** — `start` writes `.session.json`, `exec` and `stop` read it. `stop` clears it. Don't assume session exists without checking +- **agent-browser** — exact 0.34.0 runtime dependency. ProofShot preflights version/capabilities and uses a dedicated Node 24 launcher without changing the project's runtime +- **Session state** — registry files live under the user state directory; every session has an immutable ID, private browser namespace, evidence directory, and operation lease. `exec`/`stop` auto-select only one live match or require `--session` - **Session metadata** — `start` writes `metadata.json` inside each session folder with git branch/commit. This persists after `stop` and is used by `pr` to match sessions to branches - **Per-session subfolders** — artifacts go in `proofshot-artifacts/YYYY-MM-DD_HH-mm-ss_slug/` ## Command lifecycle -1. `proofshot start` — spawns dev server, opens browser, starts recording, saves session state + writes `metadata.json` with git branch/commit -2. `proofshot exec ` — logs action to `session-log.json`, forwards to `agent-browser` -3. `proofshot stop` — collects errors, stops recording, trims video, generates SUMMARY.md + viewer.html, clears session +1. `proofshot start` — validates a local isolated browser, claims a start lease, opens a unique agent-browser namespace, starts recording/network capture, and writes metadata +2. `proofshot exec [--session ID] ` — claims an exec lease, logs sanitized provenance/private structured output, and forwards to the exact browser +3. `proofshot stop [--session ID]` — claims a stop lease, atomically finalizes private evidence, performs exact cleanup, and generates canonical artifacts 4. `proofshot pr [number]` — finds sessions for current branch, uploads artifacts to GitHub, posts PR comment ## Adding a new command @@ -69,8 +69,10 @@ Edit `src/utils/error-patterns.ts` — add a new entry to the `PATTERNS` array: | `metadata.json` | `start` | Git branch, commit SHA, timestamp (persists after stop) | | `session.mp4` | `stop` | Finalized H.264 recording (`start` captures temporary `session.webm`) | | `session-log.json` | `exec` (appended each call) | Action timeline with relative timestamps | -| `server.log` | `start` (piped stdout+stderr) | All dev server output | -| `console-output.log` | `stop` | Browser console output | +| `private/server.log` | `start` (piped stdout+stderr) | Private raw dev server output | +| `private/browser/console-output.log` | `stop` | Private raw browser console output | +| `network-summary.json` | `stop` | Sanitized endpoint/method/status/timing/error metadata | +| `private/agent-browser/` | `exec` / `stop` | User-only raw JSON/HAR; excluded from manifests and publication | | `step-*.png` | `exec screenshot` | Screenshots at key moments | | `SUMMARY.md` | `stop` | Markdown report with errors and screenshots | | `viewer.html` | `stop` | Standalone HTML viewer with video + timeline | @@ -90,6 +92,8 @@ Edit `src/utils/error-patterns.ts` — add a new entry to the `PATTERNS` array: ## Gotchas - `proofshot exec` has special shell quoting logic (`buildShellCommand` in exec.ts) — `eval` commands get single-quoted, args with special chars get auto-quoted +- Direct DOM mutation is diagnostic only; synthetic actions force `INCOMPLETE`. Preserve category/redaction behavior when adding commands +- Every browser subprocess strips provider/CDP/profile/state/session/proxy inheritance and uses the persisted private config, namespace, allowlist, and socket root - Video trimming adjusts session-log.json timestamps to match the trimmed video (see `trimOffsetSec` in stop.ts) - Server log capture only works when proofshot starts the server itself — if the port is already occupied, we skip spawning and get no server logs - The `consoleErrors`/`consoleOutput` from agent-browser are point-in-time snapshots collected at stop time diff --git a/PROOFSHOT.md b/PROOFSHOT.md index a4d6746..aed6bc9 100644 --- a/PROOFSHOT.md +++ b/PROOFSHOT.md @@ -1,26 +1,18 @@ -# ProofShot Visual Verification +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets any AI coding agent verify its own work in a real browser — no vendor lock-in required. +Explore UI behavior with the approved local agent-browser launcher, maintain User Stories/Use Cases/User Flows and iteration feedback under the task's persistent `~/data` directory, then record one fresh ProofShot session per finalized Use Case. -After building or modifying UI features, verify with this workflow: +```bash +proofshot start --run "your-dev-command" --port PORT --description "UC-ID: behavior" +proofshot exec snapshot -i +proofshot exec click @e3 +proofshot exec assert-visible "#expected-result" +proofshot exec screenshot step-result.png +proofshot stop +``` -1. Start: `proofshot start --run "your-dev-command" --port PORT --description "what you are verifying"` - If the server is already running, omit --run. -2. Test: Use `proofshot exec` to navigate, click, fill forms, take screenshots -3. Stop: `proofshot stop` — bundles video, screenshots, and error report +With multiple sessions, pass the printed ID to `proofshot exec --session ID ...` and `proofshot stop --session ID`. Prefer pointer/keyboard actions, disclose hybrid input, and treat synthetic DOM mutation as diagnostic only. `FAIL`, `INCOMPLETE`, and `BLOCKED` are unsuccessful. -ProofShot keeps all `proofshot exec` commands inside the same isolated `agent-browser` session that was created by `proofshot start`, so recording, screenshots, and browser actions stay aligned. +Keep raw JSON, HAR, response bodies, credentials, and customer data private under the task's `~/data` evidence root. Report the local path without copying raw artifacts into prompts, scratch, PRs, or uploads. Copy only exercised final flows into PR User Testing instructions, then dry-run ordered explicit publication with repeated `--session` flags. -Use `--url` on `start` when verification must begin on a specific target. In an isolated HOME, ProofShot discovers executable-only Chrome/Chromium installs from system/account locations; use `--browser-executable /absolute/path/to/chrome` to select one explicitly. - -Key proofshot exec commands: -- `proofshot exec snapshot -i` — see interactive elements -- `proofshot exec click @e3` — click an element -- `proofshot exec fill @e2 "text"` — fill a form field -- `proofshot exec screenshot step.png` — capture a moment - -Artifacts saved to ./proofshot-artifacts/ including video, screenshots, errors, and summary. -Custom `--output` paths do not move active control state, so a separate `proofshot stop` still finds the session. `stop` is idempotent; after `stop --no-close`, run a later plain `stop` to close that exact retained browser without rebundling. -You can customize browser launch behavior in `proofshot.config.json`, including HTTPS error ignoring, a custom browser executable path, and a project-specific `agent-browser` config path. - -Use `proofshot doctor` when the local setup looks wrong. It prints the current config path, browser mode, viewport, installed binaries, and any active ProofShot session. +Browser tooling requires agent-browser 0.34.0 through its Node 24 launcher; project commands continue to honor the repository runtime. Recover with `proofshot session list` and `proofshot session clean --session ID`, never broad process cleanup. diff --git a/README.md b/README.md index 29d5b7a..cd8ecbd 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ npm install -g proofshot proofshot install ``` -The first command installs the CLI and [agent-browser](https://github.com/vercel-labs/agent-browser) (with headless Chromium). The second detects your AI coding tools and installs the ProofShot skill at user level — works across all your projects automatically. +The first command installs the CLI and the exact agent-browser 0.34.0 package. agent-browser requires Node 24; managed environments should expose it through a dedicated Node 24 launcher so project servers can continue using their repository runtime (for example, Node 22 from `.nvmrc`). The second command installs the ProofShot skill at user level. + +Starting a dev server with `--run` also requires `lsof` on Linux/macOS; Windows uses PowerShell's `Get-NetTCPConnection`. Run `proofshot doctor` to verify the listener-inspection prerequisite. ## How It Works @@ -102,11 +104,12 @@ Each session produces a timestamped folder in `./proofshot-artifacts/`: | `SUMMARY.md` | Markdown report with errors, screenshots, and video | | `step-*.png` | Screenshots captured at key moments | | `session-log.json` | Action timeline with timestamps and element data | -| `server.log` | Dev server stdout/stderr (when using `--run`) | -| `console-output.log` | Browser console output | | `evidence.json` | Canonical browser/environment events, incidents, source integrity, and media timing | | `verdict.json` | Structured `PASS`, `FAIL`, `INCOMPLETE`, or `BLOCKED` verdict | -| `artifact-manifest.json` | Finalized repository/commit provenance and ordered artifact hashes | +| `artifact-manifest.json` | Finalized repository/commit/runtime provenance and ordered artifact hashes | +| `network-summary.json` | Sanitized request method/status/timing/error metadata | + +Raw agent-browser JSON, HAR, browser-console, and ProofShot-owned server evidence stays under the session's `private/` directory with user-only permissions. It is excluded from manifests and publication; only curated media and sanitized summaries are reviewable artifacts.

ProofShot artifacts folder @@ -161,11 +164,13 @@ You can also configure browser launch behavior in `proofshot.config.json`: } ``` -Set `browser.configPath` when you need ProofShot to run `agent-browser` against a project-specific config instead of inheriting `~/.agent-browser/config.json`. Relative paths are resolved from the directory that contains `proofshot.config.json`. +Set `browser.configPath` for a project-specific agent-browser config. ProofShot validates it, rejects provider/CDP/profile/state/proxy and default JSON-output modes, and copies it into private per-session state. Without this option, ProofShot uses an empty per-session config instead of inheriting user or project agent-browser defaults. + +ProofShot rejects inherited `AGENT_BROWSER_JSON` at session preflight and strips it from every agent-browser subprocess. ProofShot enables JSON only for individual internal calls whose structured output it parses; a global JSON mode would change normal command output and break evidence capture. -ProofShot discovers system and account-level Chrome/Chromium installs even when the command runs with an isolated `HOME`. If no runnable browser is found, `start` prints the exact `agent-browser install` action. An explicit `--browser-executable` takes precedence for one run. +ProofShot requires agent-browser 0.34.0, a local Chrome/Chromium executable, an explicit target-domain allowlist, and a fresh isolated session. Cloud providers, shared CDP browsers, persistent profiles/state, and auto-connect are refused. If no runnable browser is found, `start` prints the exact `agent-browser install` action. An explicit `--browser-executable` takes precedence for one run. -`--output` changes only where evidence is written. Active control state stays in the configured/default output directory, so later `proofshot exec` and `proofshot stop` processes can find the same session. +`--output` changes only where evidence is written. Registry-backed control state remains discoverable across processes. Concurrent sessions in one worktree are supported; commands resolve a single live session automatically and require `--session ` when several live sessions match. `--force` recovers proven stale state only and refuses live processes or lifecycle operations. ### `proofshot stop` @@ -173,6 +178,7 @@ Stop recording, collect errors, generate proof artifacts. ```bash proofshot stop # Stop session and close browser +proofshot stop --session ID # Stop one exact session proofshot stop --no-close # Stop but keep browser open ``` @@ -186,11 +192,13 @@ When a ProofShot session is active, `proofshot exec` reuses the same isolated `a ```bash proofshot exec click @e3 +proofshot exec --session ID click @e3 proofshot exec assert-visible "#checkout-complete" proofshot exec screenshot step-checkout.png ``` Failed `assert-visible` checks are recorded in `session-log.json` and contribute to the structured verdict. +Action receipts include sanitized intent, interaction category, timing, outcome, and page context. Synthetic DOM actions make final proof incomplete; they remain useful only for diagnosis. ### `proofshot diff` @@ -227,6 +235,7 @@ Inspect and clean durable recovery records after an interrupted or incomplete cl ```bash proofshot session list proofshot session clean --session +proofshot session clean --all ``` Cleanup validates persisted process identities and never widens to a name-, port-, or default-socket kill. @@ -239,11 +248,11 @@ Remove the `./proofshot-artifacts/` directory. proofshot clean ``` -`clean` refuses while active or retained session control state exists. Run `proofshot stop` first so ProofShot does not discard exact process ownership metadata. +`clean` refuses while any matching registry state exists, including stale, starting, stopping, or recovery records. Run `proofshot stop` or `proofshot session clean` first so ProofShot does not discard evidence or exact process ownership metadata. ### `proofshot doctor` -Print the current ProofShot environment, including config path, browser mode, viewport, installed binaries, and any active session. +Print the current ProofShot environment, including config path, browser mode, viewport, installed binaries, TCP listener-inspection support, and any active session. ```bash proofshot doctor diff --git a/content/docs/concepts/how-it-works.mdx b/content/docs/concepts/how-it-works.mdx index ab24f84..bf707ca 100644 --- a/content/docs/concepts/how-it-works.mdx +++ b/content/docs/concepts/how-it-works.mdx @@ -47,9 +47,10 @@ ProofShot uses a three-phase model. 1. Check if the port is available (fail fast on conflicts) 2. Spawn an isolated dev-server process session if `--run` is provided, pipe timestamped output to `server.log`, and persist its immutable PID/process-group identity 3. Wait for the port to respond (polls every 500ms, 30s timeout) -4. Open the requested URL in a short, collision-safe agent-browser session and persist its daemon identity -5. Start video recording -6. Write `.session.json` to the configured/default control directory and `metadata.json` beside the evidence (git branch/commit, persists after stop) +4. Verify agent-browser 0.34.0, reject inherited provider/CDP/profile/state modes, and copy a safe config into private per-session state +5. Open the requested URL in a unique namespace with a target-domain allowlist and persist the daemon identity +6. Start private HAR capture and video recording +7. Register the immutable session ID and operation lease under the user state directory; write `metadata.json` beside the evidence Recording is mandatory. If it fails after 3 retries, the session aborts. @@ -57,31 +58,31 @@ Recording is mandatory. If it fails after 3 retries, the session aborts. Each `proofshot exec` call: -1. Loads `.session.json` and validates the session is active -2. For `click`/`fill`/`type` targeting `@eN` refs: captures the element's bounding box and label *before* execution -3. Appends an entry to `session-log.json` with a relative timestamp -4. Forwards the command to agent-browser -5. Returns the output +1. Selects the single addressable recording or requires `--session ` when several live sessions match +2. Claims an immutable exec lease +3. Captures pre-action page context and element data where available +4. Appends sanitized intent, interaction category, timing, and outcome to `session-log.json`; raw structured output stays private +5. Forwards the command to the exact agent-browser namespace and releases the lease ### Phase 3: Stop `proofshot stop` collects and bundles everything: -1. Collects browser console errors and output (point-in-time snapshot) -2. Stops video recording -3. Closes the exact owned browser session -4. Stops only the dev-server process session created by this run -5. Trims video dead time using ffmpeg (5s buffer before first action, 3s after last), converts the finalized recording to H.264 `session.mp4`, and adjusts all `session-log.json` timestamps by the trim offset. -6. Scans `server.log` with multi-language regex patterns for errors -7. Generates `SUMMARY.md` and `viewer.html` -8. Clears `.session.json` (or retains exact browser ownership after `--no-close`) +1. Claims a stop lease and collects console evidence +2. Atomically finalizes private HAR evidence and writes a metadata-only network summary +3. Stops video recording and closes only the exact owned browser/environment/server processes +4. Trims video dead time using ffmpeg, converts to H.264 `session.mp4`, and adjusts action timestamps +5. Generates canonical evidence, verdict, summary, viewer, and provenance manifest +6. Unregisters completed ownership, or retains the exact browser record after `--no-close` ## Design principles **Graceful degradation.** Missing ffmpeg? Video trimming and MP4 conversion are skipped, preserving the original `session.webm`. Element data capture fails? Overlays are skipped. Browser already closed? Silent catch. Non-critical failures never abort a session. -**Minimal dependencies.** Three production dependencies: `commander`, `chalk`, `detect-port`. agent-browser is an optional peer dependency. Small install, small supply chain. +**Pinned browser contract.** ProofShot pins agent-browser 0.34.0 and fails before launch on older versions. Managed launchers keep its Node 24 runtime separate from project runtimes. -**Session isolation.** Each project keeps control state in its configured/default output directory. A CLI-only custom evidence path cannot hide the session from a later process, while different projects still run independently. +**Session isolation.** Registry-backed immutable IDs, namespaces, sockets, operation leases, and evidence directories allow concurrent sessions in one worktree without cross-session cleanup. + +**Private evidence boundary.** Raw JSON, HAR, response bodies, credentials, and customer data remain in a user-only private directory excluded from manifests and publication. **ESM-only.** All imports use explicit `.js` extensions for correct resolution after TypeScript compilation. diff --git a/content/docs/faq.mdx b/content/docs/faq.mdx index 79d46e6..09370aa 100644 --- a/content/docs/faq.mdx +++ b/content/docs/faq.mdx @@ -14,7 +14,7 @@ If your agent can run shell commands, it works with ProofShot. The CLI is agent- No. Skills install at user level (your home directory), so one `proofshot install` works across all projects on your machine. The optional `proofshot.config.json` is only needed if you want to customize defaults like port or viewport size. **What does the agent-browser dependency do?** -[agent-browser](https://github.com/vercel-labs/agent-browser) is a Rust CLI + Node.js daemon that manages a headless Chromium browser. ProofShot delegates all browser work to it — opening pages, clicking elements, taking screenshots, recording video. It's installed automatically with ProofShot. +[agent-browser](https://github.com/vercel-labs/agent-browser) is a Rust CLI + Node.js daemon that manages Chromium. ProofShot pins 0.34.0, requires its Node 24 runtime, and delegates browser interaction/recording to it. A dedicated launcher lets the project continue using its own runtime, such as Node 22 from `.nvmrc`. **Do I need ffmpeg?** No. ffmpeg is optional. Without it, ProofShot skips video trimming (you get the full uncut recording) and `.webm` to `.mp4` conversion (GitHub PR comments may not display `.webm` inline). Everything else works the same. @@ -28,7 +28,7 @@ The skill file installed by `proofshot install` teaches your agent the three-ste Stable handles to interactive elements on a page. When your agent runs `proofshot exec snapshot -i`, it gets a list like `@e1: button "Submit"`, `@e2: input "Email"`. These references persist across commands within a session, so the agent can target elements reliably without CSS selectors. **Can I run multiple sessions at the same time?** -Yes. Session state (`.session.json`) lives in each project's configured/default output directory, not globally. Different projects can run concurrently. A one-run `--output` override moves evidence without changing where that project finds active control state. +Yes. Registry-backed session IDs, browser namespaces, sockets, operation leases, and evidence directories are independent even in one worktree. Commands auto-select one live match; with several, pass the printed ID using `--session`. **What languages does error detection support?** JavaScript/Node.js, Python, Ruby/Rails, Go, Java/Kotlin, Rust, PHP, C#/.NET, Elixir/Phoenix, plus generic patterns for `FATAL`, `CRITICAL`, and segfaults. See [How to add error patterns](/docs/guides/add-error-pattern) to extend support. @@ -41,9 +41,18 @@ When ffmpeg is available, `proofshot stop` cuts dead time from the video — kee **"No active session" when running exec or stop** You need to run `proofshot start` before `exec`. `stop` is idempotent, so it succeeds without changing artifacts when the session is already stopped. +**"Multiple active sessions" when running exec or stop** +Run `proofshot session list`, then pass the exact ID with `proofshot exec --session ID ...` or `proofshot stop --session ID`. + +**agent-browser version or Node errors** +Check the repository runtime (`.nvmrc` and `node --version`) separately from the configured agent-browser launcher. ProofShot requires the launcher to report agent-browser 0.34.0 under Node 24; do not change the project's runtime to fix the browser tool. + **Chrome is installed, but an isolated HOME cannot find it** ProofShot checks system paths and executable-only browser caches under the real account home without reusing a browser profile or storage. You can also pass one exact path with `proofshot start --browser-executable /absolute/path/to/chrome`. If nothing is runnable, run the `agent-browser install` command printed by `proofshot start`. +**Can ProofShot use my Chrome profile, saved state, cloud provider, or CDP browser?** +No. Final proof uses a fresh local owned Chromium session. ProofShot rejects provider/CDP/auto-connect/profile/state/proxy modes and copies a validated config into private per-session state. + **Server errors aren't being detected** Server log capture only works when ProofShot starts the server itself via `--run`. If your server was already running on the port, ProofShot skips spawning and gets no logs. diff --git a/content/docs/guides/verify-feature.mdx b/content/docs/guides/verify-feature.mdx index 2bb6d0b..f113c0a 100644 --- a/content/docs/guides/verify-feature.mdx +++ b/content/docs/guides/verify-feature.mdx @@ -125,7 +125,7 @@ Every `proofshot exec` or `agent-browser` command operates on the same browser s ## Troubleshooting **"No active session" error when running exec commands** -You need to run `proofshot start` first. The session state is stored in `.session.json` inside the artifacts directory. +Run `proofshot start` first. If several live sessions match the worktree, inspect `proofshot session list` and pass the exact ID with `--session`. **Video has long pauses with no activity** Install [ffmpeg](https://ffmpeg.org/) and ProofShot will automatically trim dead time from the video (5s buffer before the first action, 3s buffer after the last) and convert the finalized recording to H.264 MP4. diff --git a/content/docs/quick-start.mdx b/content/docs/quick-start.mdx index 04260ad..f506024 100644 --- a/content/docs/quick-start.mdx +++ b/content/docs/quick-start.mdx @@ -34,7 +34,7 @@ yarn global add proofshot -This installs the CLI and downloads [agent-browser](https://github.com/vercel-labs/agent-browser) with headless Chromium. +This installs the CLI and exact agent-browser 0.34.0 package. agent-browser requires Node 24; use the configured browser-tool launcher so project commands continue to honor their repository runtime. ### Install the skill for your AI agent @@ -64,7 +64,7 @@ In your project directory, tell your AI agent to verify a feature. For example: > Verify the login page with proofshot. Start the dev server, check that the form renders, fill in credentials, and submit. Screenshot each step. -The agent will run: +Explore and refine the User Flow first, then record one fresh session for the finalized Use Case. The agent will run: ```sh proofshot start --run "npm run dev" --port 3000 --description "Login page verification" diff --git a/content/docs/reference/cli.mdx b/content/docs/reference/cli.mdx index 4b5401d..8d87d0b 100644 --- a/content/docs/reference/cli.mdx +++ b/content/docs/reference/cli.mdx @@ -49,7 +49,7 @@ proofshot start [options] | `--headed` | Show the browser window (visible Chromium) | `false` | | `--output

` | Custom output directory for artifacts | `./proofshot-artifacts` | | `--browser-executable ` | Use an exact Chrome/Chromium executable | auto-discovered | -| `--force` | Clean up and replace an active session | `false` | +| `--force` | Recover proven stale state; refuses live sessions/operations | `false` | **Examples:** @@ -64,10 +64,10 @@ proofshot start --headed # Show the browser w **What happens:** 1. Starts either `--run` or the configured owned `environment` (the two are mutually exclusive), attaches every configured log source, and completes readiness checks -2. Opens headless Chromium via agent-browser -3. Navigates to `--url` (or `http://localhost:`) -4. Starts video recording (retries up to 3 times) -5. Writes control `.session.json` to the configured/default output and durable `metadata.json` beside the evidence. A CLI-only `--output` changes evidence placement, not control discovery. +2. Requires agent-browser 0.34.0, a local Chromium executable, a safe per-session config, and a target-domain allowlist +3. Creates a unique browser namespace/private evidence directory and registers an immutable session ID +4. Navigates to `--url`, starts private HAR capture, and starts video recording +5. Persists exact process identities and lifecycle operation ownership for cross-process recovery --- @@ -81,17 +81,18 @@ proofshot stop [options] | Flag | Description | Default | |------|-------------|---------| +| `--session ` | Stop one exact registered session | auto-select one live match | | `--no-close` | Keep the browser open after stopping | `false` | **What happens:** -1. Collects console errors and output from the browser -2. Stops video recording +1. Claims the exact session operation and collects console errors/output +2. Atomically finalizes private HAR evidence and a sanitized network summary, then stops video recording 3. Closes the exact owned browser session (unless `--no-close`) 4. Stops only the tmux session/server, direct processes, capture helpers, and dev-server process identities created by this ProofShot start 5. Trims video dead time and converts the finalized recording to H.264 `session.mp4` (requires ffmpeg): 5s buffer before first action, 3s after last 6. Scans `server.log` for errors across 10+ languages 7. Generates `SUMMARY.md`, canonical `evidence.json`, structured `verdict.json`, `viewer.html`, and a hashed provenance manifest -8. Clears `.session.json`, or retains it after `--no-close` until a later plain `stop` closes that exact browser +8. Unregisters completed ownership, or retains the exact session after `--no-close` until a later plain `stop` Repeated `stop` calls are successful no-ops. If bundling fails, control state remains retryable; a later `stop` reuses already-collected artifacts instead of widening process cleanup. @@ -102,7 +103,7 @@ Repeated `stop` calls are successful no-ops. If bundling fails, control state re Pass-through to agent-browser with automatic session logging. Captures timestamps, element bounding boxes, and resolves screenshot paths into the session directory. ```sh -proofshot exec [args...] +proofshot exec [--session ] [args...] ``` **Examples:** @@ -116,14 +117,15 @@ proofshot exec scroll down proofshot exec press Enter ``` -Every `exec` call appends an entry to `session-log.json` with: +With one live session, `exec` selects it automatically. Multiple live sessions require `--session `. Every call appends an entry to `session-log.json` with: - Relative timestamp from session start -- The command and arguments +- Sanitized command intent and interaction category - Passed/failed outcome and error text +- Sanitized pre/post page context and duration - Expected selector for `assert-visible` - Element bounding box and label (for `click`, `fill`, `type` targeting `@eN` refs) -This data powers the interactive viewer's timeline and overlays. +This data powers the interactive viewer's timeline and overlays. Synthetic DOM actions are diagnostic only and force an incomplete final verdict. Raw structured agent-browser output stays in private local evidence. --- @@ -201,6 +203,7 @@ Inspect durable recovery records and retry exact cleanup after an interrupted ru ```sh proofshot session list proofshot session clean --session +proofshot session clean --all ``` `clean` validates persisted process/session identities. Identity mismatches remain in recovery state and never widen cleanup to a port, process name, or default tmux socket. @@ -215,4 +218,4 @@ Remove the entire artifacts directory. proofshot clean ``` -Deletes `./proofshot-artifacts/` (or the configured output directory). No flags. If active or retained control state exists, `clean` refuses and asks you to run `proofshot stop` first so exact process ownership metadata is not discarded. +Deletes `./proofshot-artifacts/` (or the configured output directory). No flags. If any matching registry state exists—including starting, stopping, stale, or recovery state—`clean` refuses until `proofshot stop` or `proofshot session clean` preserves the evidence/ownership invariant. diff --git a/docs/architecture.md b/docs/architecture.md index 5f530d8..7fffd3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,15 +34,14 @@ The choice of agent-browser as the browser automation layer is the most importan **Built-in recording.** Playwright's screencast API is exposed directly, so video capture works without ffmpeg. Finalization uses ffmpeg to trim the WebM capture and convert it to H.264 MP4. -All browser commands in ProofShot go through a single function: +All browser commands pass through one wrapper that injects the exact session, private config, namespace, socket root, allowlist, and a cleaned environment: ```typescript // src/utils/exec.ts -export function ab(command: string, timeoutMs = 30000): string { - return execSync(`agent-browser ${command}`, { - encoding: 'utf-8', - timeout: timeoutMs, - stdio: ['pipe', 'pipe', 'pipe'], +export function ab(command: string, options: AgentBrowserCommandOptions): string { + return execSync(buildAgentBrowserCommand(command, options), { + env: getAgentBrowserEnvironment(options), + timeout: options.timeoutMs ?? 30000, }).trim(); } ``` @@ -57,12 +56,12 @@ ProofShot uses a three-phase model: **start**, **exec** (repeated), **stop**. proofshot start --run "npm run dev" --port 3000 --description "Login flow" ``` -1. Check if port is already occupied (fail fast if `--run` conflicts) -2. Spawn dev server as a detached process, pipe stdout/stderr to `server.log` -3. Wait for port to become available (polls every 500ms, 30s timeout) -4. Open headless Chromium via agent-browser -5. Start video recording (Playwright screencast → temporary `.webm`) -6. Write `.session.json` to the artifacts directory +1. Verify agent-browser 0.34.0, reject inherited provider/CDP/profile/state/default-JSON modes, and copy a safe config into private per-session state +2. Create a collision-safe session ID, namespace, socket root, domain allowlist, evidence directory, and immutable start-operation lease +3. Spawn an exact-owned dev server/environment when configured and persist process identities before waiting for readiness +4. Open fresh local Chromium and persist the daemon identity +5. Start private HAR capture and video recording +6. Register the session under the user state directory and write durable Git metadata beside its evidence Recording is mandatory. If it fails after 3 retries, the session aborts. Video proof is the whole point. @@ -75,11 +74,11 @@ proofshot exec screenshot step-login.png Each `exec` call: -1. Loads `.session.json`, validates recording is active -2. For click/fill/type actions targeting `@eN` refs: captures element bounding box and label *before* execution (used for viewer overlays) -3. Appends an entry to `session-log.json` with relative timestamp and element data -4. Forwards the command to agent-browser -5. Returns agent-browser's output +1. Selects the single addressable recording or requires `--session ` when several live sessions match +2. Claims an immutable exec-operation lease +3. Captures pre-action page context plus element bounds/label when available +4. Appends sanitized intent, interaction category, timing, outcome, and page context to `session-log.json`; raw structured output stays private +5. Forwards the command to the exact agent-browser namespace and releases the lease The element data capture uses a multi-strategy approach because agent-browser's `get box` command doesn't accept refs directly: - Try to get the element's `id` attribute, then query by `#id` @@ -92,14 +91,12 @@ The element data capture uses a multi-strategy approach because agent-browser's proofshot stop ``` -1. Collect console errors and console output from the browser (point-in-time snapshots) -2. Stop video recording -3. Close browser (unless `--no-close`) -4. **Finalize video** — remove dead time before the first action (5s buffer) and after the last action (3s buffer), convert to H.264 `session.mp4`, and adjust all `session-log.json` timestamps by the trim offset. -5. **Extract server errors** — scan `server.log` with multi-language regex patterns -6. Generate `SUMMARY.md` — markdown report with description, video link, screenshots, and errors -7. Generate `viewer.html` — standalone interactive viewer -8. Clear `.session.json` +1. Claim a stop-operation lease and collect console evidence +2. Flush HAR capture to a private pending file, validate it, atomically adopt it, and write a metadata-only network summary +3. Stop video and close only exact owned browser/environment/server processes +4. **Finalize video** — trim dead time, convert to H.264 `session.mp4`, and adjust action timestamps +5. Generate canonical evidence, verdict, summary, viewer, and provenance manifest +6. Unregister completed ownership, or retain the exact browser record after `--no-close` ## Interactive viewer @@ -176,6 +173,9 @@ src/ │ ├── pr.ts # GitHub PR description formatting │ └── clean.ts # Artifact directory removal ├── browser/ +│ ├── isolation.ts # Version gate, clean config/env, local-runtime policy +│ ├── provenance.ts # Interaction classification and secret-safe intent +│ ├── evidence.ts # Private structured/HAR evidence + sanitized summary │ ├── session.ts # Browser open/close, console collection │ ├── capture.ts # Recording start/stop, screenshots, diffs │ ├── navigate.ts # URL navigation, snapshot @@ -183,7 +183,10 @@ src/ ├── server/ │ └── start.ts # Dev server spawn + port waiting ├── session/ -│ └── state.ts # .session.json read/write/clear +│ ├── registry.ts # Global registry and per-session operation leases +│ ├── selection.ts # Operation-specific implicit/explicit resolution +│ ├── lifecycle.ts # Exact owned-process cleanup and recovery +│ └── state.ts # Session identity and evidence/process state ├── artifacts/ │ ├── viewer.ts # Interactive HTML viewer generation │ ├── summary.ts # Markdown summary generation @@ -203,10 +206,16 @@ src/ **tsup with two entry points.** The CLI binary (`bin/proofshot.ts`) builds to a single file with a shebang. The library entry (`src/index.ts`) builds with code splitting and `.d.ts` generation. This supports both CLI usage and programmatic import by other tools. -**Minimal dependencies.** Only three production dependencies: `commander` (CLI framework), `chalk` (terminal colors), `detect-port` (port checking). agent-browser is an optional peer dependency. This keeps the install fast and the supply chain small. +**Pinned browser contract.** agent-browser 0.34.0 is an exact runtime dependency and requires Node 24. Managed installations use a dedicated launcher so project commands keep their repository runtime. -**Session state in the output directory.** `.session.json` lives alongside artifacts, not in a global location. This allows parallel sessions in different projects and ensures `proofshot clean` removes everything. +Finalized evidence records the managed launcher contract and the hashes of the launcher and executed native artifact. This receipt identifies the runtime used for the session; it does not attest how the package dependency was installed. + +ProofShot rejects inherited `AGENT_BROWSER_JSON` and the equivalent config key, then strips the environment variable from every subprocess. Structured output is enabled only on the internal calls that parse it; ordinary passthrough commands retain their expected text output. + +**Registry-backed session control.** User-state registry files point to per-run evidence while immutable IDs, namespaces, sockets, and operation leases allow concurrent sessions in the same worktree. `clean` refuses while any matching registry record remains. + +**Private evidence boundary.** Raw agent-browser JSON, HAR, response bodies, credentials, and customer data remain in a `0700` private tree with `0600` files. Manifests and PR publication exclude that tree. **Config file walk-up.** `proofshot.config.json` is searched from cwd upward to filesystem root, supporting monorepo layouts where config lives at the repo root. -**Graceful degradation everywhere.** Missing ffmpeg skips video trimming and MP4 conversion, preserving the original WebM capture. Failed element data capture skips overlays. Browser already closed gets a silent catch. Console errors unavailable shows "0 errors". Non-critical failures never abort a session. +**Fail closed at trust boundaries.** Wrong browser versions, shared/cloud browser modes, ownership mismatches, corrupt operation locks, synthetic final proof, and invalid private evidence cannot be presented as successful proof. Missing ffmpeg still preserves the original WebM. diff --git a/package-lock.json b/package-lock.json index 9b2ad5e..01ea6b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "agent-browser": "^0.27.0", + "agent-browser": "0.34.0", "chalk": "^5.4.0", "commander": "^12.0.0", "detect-port": "^2.1.0", @@ -27,7 +27,7 @@ "vitest": "^2.1.0" }, "engines": { - "node": ">=18" + "node": ">=24" } }, "node_modules/@esbuild/aix-ppc64": { @@ -1038,13 +1038,17 @@ } }, "node_modules/agent-browser": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/agent-browser/-/agent-browser-0.27.0.tgz", - "integrity": "sha512-mmHzVsYFVA6nshNNGJzg83aVMgKpf4h98ytY3pvtJB1Cot0ZyA2bfnkbSngGD56Azkj+GlhVH6qx9DfKOVE0yg==", + "version": "0.34.0", + "resolved": "https://registry.npmjs.org/agent-browser/-/agent-browser-0.34.0.tgz", + "integrity": "sha512-eR6Ey4I/DMs9zZ60b3ziV6pgLIgpxXWzggr3dfFbtskLmeXPJAgXCIIwVL4PihVYJqEUpvWgUKlZ2CIjY1u44g==", "hasInstallScript": true, "license": "Apache-2.0", "bin": { "agent-browser": "bin/agent-browser.js" + }, + "engines": { + "node": ">=24.0.0", + "pnpm": ">=11.0.0" } }, "node_modules/any-promise": { diff --git a/package.json b/package.json index 382c1d0..239bb6d 100644 --- a/package.json +++ b/package.json @@ -15,10 +15,10 @@ "test:ffmpeg": "node test/ffmpeg-trim.mjs", "test:watch": "vitest", "prepare": "node prepare.js", - "postinstall": "node -e \"try { require('child_process').execSync('agent-browser install', { stdio: 'inherit' }) } catch(e) { console.log('Note: Run agent-browser install to set up Chromium') }\"" + "postinstall": "node -e \"try { require('child_process').execFileSync('agent-browser', ['install'], { stdio: 'inherit' }) } catch(e) { console.log('Note: Run agent-browser install to set up Chromium') }\"" }, "dependencies": { - "agent-browser": "^0.27.0", + "agent-browser": "0.34.0", "chalk": "^5.4.0", "commander": "^12.0.0", "detect-port": "^2.1.0", @@ -32,7 +32,7 @@ "vitest": "^2.1.0" }, "engines": { - "node": ">=18" + "node": ">=24" }, "files": [ "dist", diff --git a/skills/claude/SKILL.md b/skills/claude/SKILL.md index e99f2ca..8c16622 100644 --- a/skills/claude/SKILL.md +++ b/skills/claude/SKILL.md @@ -1,79 +1,48 @@ --- name: proofshot -description: Visual verification of UI features. Use after building or modifying any - UI component, page, or visual feature. Starts a verification session with video - recording and error capture, then you drive the browser to test, then stop to - bundle proof artifacts for the human. +description: Visual verification of UI features. Explore and refine real user flows with agent-browser, then record one fresh isolated ProofShot session per reviewable use case and retain local evidence. allowed-tools: Bash(proofshot:*), Bash(agent-browser:*) --- -# ProofShot — Visual Verification Workflow +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets you verify your own work in a real browser — video proof, screenshots, and error reports, no vendor lock-in. +Use this workflow after a UI or visual behavior changes. -## When to use +## 1. Explore and write the brief -Use ProofShot after: -- Building a new UI feature or page -- Modifying existing UI components -- Fixing a visual bug -- Any change that affects what the user sees +Use the approved local `agent-browser` launcher before recording. Keep project commands on the repository runtime; browser tooling uses its separate Node 24 launcher and agent-browser 0.34.0. -## The workflow (always follow these 3 steps) +Maintain one Markdown brief in the task's persistent `~/data` directory. Define User Stories (actor, goal, benefit), stable Use Cases (preconditions, starting state, test data, success criteria), user-level User Flows, Iteration Feedback, and final User Testing instructions. -### Step 1: Start the session +## 2. Refine every flow -```bash -proofshot start --run "your-dev-command" --port PORT --description "what you are about to verify" -``` - -This opens a browser and begins recording. If the port is already in use, ProofShot leaves that unowned listener alone and asks you to choose another port or stop it explicitly. - -**Always use `--run`** to let proofshot start and capture your dev server output (server logs appear in the proof report). -Only omit `--run` if the server was explicitly started by the user or another process — without it, no server logs are captured. +Exercise each flow repeatedly until it is correct, stable, and concise. Prefer pointer/keyboard-backed actions and disclose hybrid fill/select/checkbox behavior. `eval`, direct DOM clicks, setters, and dispatched events are diagnostics only and cannot be final proof. -If a previous session was not stopped cleanly, add `--force` to override it. -For configured multi-service projects, let `environment` start the declared tmux panes or direct processes; do not launch duplicate services separately. Use `proofshot doctor` when browser, FFmpeg, or recovery state is unclear. - -### Step 2: Drive the browser and test - -Use proofshot exec to navigate, interact, and verify: +## 3. Record one fresh session per Use Case ```bash -proofshot exec snapshot -i # See interactive elements -proofshot exec open http://localhost:PORT/page # Navigate to a page -proofshot exec click @e3 # Click a button -proofshot exec fill @e2 "test@example.com" # Fill a form field -proofshot exec assert-visible "#expected-result" # Record an explicit assertion -proofshot exec screenshot step-NAME.png # Capture key moments +proofshot start --run "your-dev-command" --port PORT --description "UC-ID: behavior" +proofshot exec snapshot -i +proofshot exec click @e3 +proofshot exec assert-visible "#expected-result" +proofshot exec screenshot step-result.png +proofshot stop ``` -Take screenshots at important moments — these become the visual proof. -Verify expected outcomes with assertion commands before taking screenshots. A screenshot alone is not a passing assertion. +Omit `--run` only for an explicitly external server. With multiple sessions, use the printed ID in `proofshot exec --session ID ...` and `proofshot stop --session ID`. Record the final session ID and local evidence pointer in the brief. -### Step 3: Stop and bundle the proof +## 4. Inspect and retain -```bash -proofshot stop -``` +`FAIL`, `INCOMPLETE`, and `BLOCKED` are unsuccessful. Keep raw JSON, HAR, response bodies, credentials, and customer data private under the task's `~/data` evidence root. Report paths without copying raw artifacts into prompts, scratch, PRs, or uploads. -This stops recording, collects console + server errors, and generates -a SUMMARY.md with video, screenshots, and error report. -Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` verdicts as unsuccessful verification. If cleanup was interrupted, inspect `proofshot session list` and retry only the exact session with `proofshot session clean --session ID`. +Recover interrupted cleanup with `proofshot session list` and `proofshot session clean --session ID`; never kill by process name or port. `--force` only recovers proven stale state. -### Step 4 (optional): Post proof to the PR +## 5. Publish curated proof + +Copy the exercised final flows into PR User Testing instructions. Dry-run explicit ordered publication, repeating `--session` in review order: ```bash -proofshot pr # Auto-detect PR from current branch -proofshot pr 42 # Target a specific PR number -proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID +proofshot pr 42 --session SESSION_ID --screenshot step-result.png --dry-run ``` -This uploads screenshots and video to GitHub and posts a formatted comment on the PR. By default it uses the official GitHub contents API on a `proofshot-artifacts` branch. Use `--upload-provider github-web-attachments` if you specifically want GitHub attachment URLs. - -## Tips - -- Always include a meaningful --description so the human knows what was tested -- Take screenshots before AND after key actions (e.g., before form submit, after redirect) -- If you find errors during verification, fix them and re-run the workflow -- Use `proofshot pr` after stopping to attach proof directly to the pull request +Publish only curated media, sanitized summaries, and interaction disclosure. diff --git a/skills/codex/AGENTS.md b/skills/codex/AGENTS.md index 5b7afc4..ad2500d 100644 --- a/skills/codex/AGENTS.md +++ b/skills/codex/AGENTS.md @@ -1,22 +1,3 @@ -# ProofShot Visual Verification +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets any AI coding agent verify its own work in a real browser — no vendor lock-in required. - -After building or modifying UI features, verify with this workflow: - -1. Start: `proofshot start --run "your-dev-command" --port PORT --description "what you are verifying"` - If the server is already running, omit --run. - For configured multi-service projects, let `environment` launch its tmux panes or processes. Run `proofshot doctor` for dependency or recovery diagnostics. -2. Test: Use `proofshot exec` to navigate, click, fill forms, take screenshots - Record an explicit assertion such as `proofshot exec assert-visible "#expected-result"` before screenshots. -3. Stop: `proofshot stop` — bundles canonical evidence, video, screenshots, and a structured verdict. `FAIL`, `INCOMPLETE`, and `BLOCKED` are unsuccessful. - After an interrupted cleanup, use `proofshot session list` and `proofshot session clean --session ID`; never kill by port or process name. -4. Publish only finalized matching evidence: `proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID`. Repeat either flag for multiple explicit selections. - -Key proofshot exec commands: -- `proofshot exec snapshot -i` — see interactive elements -- `proofshot exec click @e3` — click an element -- `proofshot exec fill @e2 "text"` — fill a form field -- `proofshot exec screenshot step.png` — capture a moment - -Artifacts saved to ./proofshot-artifacts/ including video, screenshots, errors, and summary. +Follow [SKILL.md](./SKILL.md) for the complete exploration-to-proof workflow. The completion boundary is one fresh isolated session per finalized Use Case, retained local evidence, and PR User Testing instructions copied from flows that were actually exercised. diff --git a/skills/codex/SKILL.md b/skills/codex/SKILL.md index 2dd6659..abdb7ea 100644 --- a/skills/codex/SKILL.md +++ b/skills/codex/SKILL.md @@ -1,78 +1,61 @@ --- name: proofshot -description: Visual verification of UI features. Use after building or modifying any - UI component, page, or visual feature. Starts a verification session with video - recording and error capture, then you drive the browser to test, then stop to - bundle proof artifacts for the human. +description: Visual verification of UI features. Explore and refine real user flows with agent-browser, then record one fresh isolated ProofShot session per reviewable use case and retain local evidence. --- -# ProofShot — Visual Verification Workflow +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets you verify your own work in a real browser — video proof, screenshots, and error reports, no vendor lock-in. +Use this workflow after a UI or visual behavior changes. -## When to use +## 1. Explore -Use ProofShot after: -- Building a new UI feature or page -- Modifying existing UI components -- Fixing a visual bug -- Any change that affects what the user sees +Use the approved local `agent-browser` launcher to understand the feature before recording proof. Keep the project on its repository runtime; browser tooling uses its separate Node 24 launcher and agent-browser 0.34.0. -## The workflow (always follow these 3 steps) +Maintain one Markdown brief in the task's persistent `~/data` directory: -### Step 1: Start the session +- **User Story**: actor, goal, and benefit. +- **Use Case**: one reviewable behavior with a stable ID, preconditions, starting state, test-data references, and success criteria. +- **User Flow**: ordered user-level actions and observable outcomes. Exclude shell commands and ephemeral element refs. +- **Iteration Feedback**: what was wrong, brittle, redundant, or slow and what the next revision changes. +- **User Testing instructions**: the concise final flow copied to the PR after it was exercised. -```bash -proofshot start --run "your-dev-command" --port PORT --description "what you are about to verify" -``` - -This opens a browser and begins recording. If the port is already in use, ProofShot leaves that unowned listener alone and asks you to choose another port or stop it explicitly. +## 2. Refine every flow -**Always use `--run`** to let proofshot start and capture your dev server output (server logs appear in the proof report). -Only omit `--run` if the server was explicitly started by the user or another process — without it, no server logs are captured. +Exercise each User Flow with `agent-browser`. Append feedback and revise until the flow is correct, stable, and concise. Exploration can use repeated passes. -If a previous session was not stopped cleanly, add `--force` to override it. -For configured multi-service projects, let `environment` start the declared tmux panes or direct processes; do not launch duplicate services separately. Use `proofshot doctor` when browser, FFmpeg, or recovery state is unclear. +- Prefer pointer/keyboard-backed interactions. +- Disclose hybrid operations such as fill, select, and checkbox actions. +- Treat `eval`, direct DOM `.click()`, setters, and dispatched events as diagnostics only. A synthetic DOM action cannot be final behavioral proof. -### Step 2: Drive the browser and test +## 3. Record fresh proof -Use proofshot exec to navigate, interact, and verify: +For each finalized Use Case, start a fresh ProofShot session and rerun only its optimized flow: ```bash -proofshot exec snapshot -i # See interactive elements -proofshot exec open http://localhost:PORT/page # Navigate to a page -proofshot exec click @e3 # Click a button -proofshot exec fill @e2 "test@example.com" # Fill a form field -proofshot exec assert-visible "#expected-result" # Record an explicit assertion -proofshot exec screenshot step-NAME.png # Capture key moments +proofshot start --run "your-dev-command" --port PORT --description "UC-ID: behavior" +proofshot exec snapshot -i +proofshot exec click @e3 +proofshot exec assert-visible "#expected-result" +proofshot exec screenshot step-result.png +proofshot stop ``` -Take screenshots at important moments — these become the visual proof. -Verify expected outcomes with assertion commands before taking screenshots. A screenshot alone is not a passing assertion. +Omit `--run` only when the server is explicitly owned elsewhere. Record one short session per Use Case. When several sessions match the worktree, pass the printed ID with `proofshot exec --session ID ...` and `proofshot stop --session ID`. -### Step 3: Stop and bundle the proof +Completion requires an explicit assertion, a fresh-session replay, and a retained session ID/evidence pointer in the brief. -```bash -proofshot stop -``` +## 4. Inspect and retain -This stops recording, collects console + server errors, and generates -a SUMMARY.md with video, screenshots, and error report. -Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` verdicts as unsuccessful verification. If cleanup was interrupted, inspect `proofshot session list` and retry only the exact session with `proofshot session clean --session ID`. +Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` as unsuccessful verification. Raw agent-browser JSON, HAR, response bodies, credentials, and customer data remain private under the task's `~/data` evidence root. Report the local path; do not copy raw evidence into prompts, scratch, PRs, or uploads. -### Step 4 (optional): Post proof to the PR +If cleanup is interrupted, use `proofshot session list` and `proofshot session clean --session ID`. Never clean by process name or port. `--force` recovers proven stale state only and refuses a verified live session or operation. -```bash -proofshot pr # Auto-detect PR from current branch -proofshot pr 42 # Target a specific PR number -proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID -``` +## 5. Publish curated proof -This uploads screenshots and video to GitHub and posts a formatted comment on the PR. By default it uses the official GitHub contents API on a `proofshot-artifacts` branch. Use `--upload-provider github-web-attachments` if you specifically want GitHub attachment URLs. +Copy only exercised final flows into PR User Testing instructions. Publish explicit ordered sessions and curated screenshots: -## Tips +```bash +proofshot pr 42 --session SESSION_ID --screenshot step-result.png --dry-run +``` -- Always include a meaningful --description so the human knows what was tested -- Take screenshots before AND after key actions (e.g., before form submit, after redirect) -- If you find errors during verification, fix them and re-run the workflow -- Use `proofshot pr` after stopping to attach proof directly to the pull request +Repeat `--session` in review order. Publish only curated media, sanitized summaries, and interaction disclosure; keep raw private evidence local. diff --git a/skills/cursor/proofshot.mdc b/skills/cursor/proofshot.mdc index 0e9dcb7..fcf3cda 100644 --- a/skills/cursor/proofshot.mdc +++ b/skills/cursor/proofshot.mdc @@ -1,24 +1,15 @@ --- -description: Visual verification of UI changes using ProofShot +description: Explore UI behavior, refine user flows, then record fresh isolated ProofShot evidence globs: ["**/*.tsx", "**/*.jsx", "**/*.vue", "**/*.svelte", "**/*.html"] --- -ProofShot is an open-source, agent-agnostic CLI for visual verification — works with any agent, no vendor lock-in. +After a UI change: -After modifying UI files, visually verify changes with this workflow: +1. Explore with the approved local agent-browser launcher. Keep project commands on the repository runtime and browser tooling on its separate Node 24/agent-browser 0.34.0 launcher. +2. Maintain a brief under the task's persistent `~/data` directory: User Stories; stable Use Cases with preconditions, starting state, test data, and success criteria; user-level User Flows; Iteration Feedback; and final User Testing instructions. +3. Exercise and simplify every flow. Prefer pointer/keyboard actions, disclose hybrid fill/select/checkbox behavior, and use synthetic DOM mutation only for diagnosis. +4. Rerun each finalized Use Case in one fresh ProofShot session. Include an assertion and screenshot. With multiple sessions, pass the printed ID to `proofshot exec --session ID ...` and `proofshot stop --session ID`. +5. Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` as unsuccessful. Record the session ID and local evidence pointer. Keep raw JSON/HAR/bodies/credentials/customer data local under `~/data` and out of prompts, scratch, PRs, and uploads. +6. Copy only exercised flows into PR User Testing instructions. Dry-run explicit ordered publication with repeated `--session` flags and curated screenshots. -1. Start session: `proofshot start --run "your-dev-command" --port PORT --description "what you are verifying"` - Always use --run so proofshot captures server logs. Only omit --run if the server was started by the user. - If the port is in use, ProofShot leaves that unowned listener alone; choose another port or stop it explicitly. Add --force only if a stale ProofShot session blocks startup. - For configured multi-service projects, let `environment` launch its tmux panes or processes instead of starting duplicate services. Run `proofshot doctor` for dependency or recovery diagnostics. -2. Drive browser: Use `proofshot exec` commands to navigate, click, fill forms, and take screenshots - Record an explicit assertion such as `proofshot exec assert-visible "#expected-result"` before screenshots. -3. Stop session: `proofshot stop` to bundle canonical evidence, video, screenshots, and a structured verdict. `FAIL`, `INCOMPLETE`, and `BLOCKED` are unsuccessful. - After an interrupted cleanup, use `proofshot session list` and `proofshot session clean --session ID`; never kill by port or process name. -4. Publish only finalized matching evidence: `proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID`. Repeat either flag for multiple explicit selections. - -Key proofshot exec commands: -- `proofshot exec snapshot -i` — see interactive elements -- `proofshot exec click @e3` — click an element -- `proofshot exec fill @e2 "text"` — fill a form field -- `proofshot exec screenshot step.png` — capture a moment +Recover only with `proofshot session list` and `proofshot session clean --session ID`; never kill by process name or port. `--force` refuses live sessions and operations. diff --git a/skills/generic/PROOFSHOT.md b/skills/generic/PROOFSHOT.md index 4655d77..9ab45a2 100644 --- a/skills/generic/PROOFSHOT.md +++ b/skills/generic/PROOFSHOT.md @@ -1,23 +1,10 @@ -# ProofShot Visual Verification +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets any AI coding agent verify its own work in a real browser — no vendor lock-in required. +1. Explore with the approved local agent-browser launcher before recording. Keep project commands on the repository runtime and browser tooling on its separate Node 24/agent-browser 0.34.0 launcher. +2. Maintain a brief under the task's persistent `~/data` directory: User Stories; stable Use Cases with preconditions, starting state, test data, and success criteria; user-level User Flows; Iteration Feedback; and final User Testing instructions. +3. Exercise and simplify every flow. Prefer pointer/keyboard actions, disclose hybrid fill/select/checkbox behavior, and use synthetic DOM mutation only for diagnosis. +4. Record one fresh ProofShot session per finalized Use Case. Include an assertion and screenshot. With multiple sessions, use `proofshot exec --session ID ...` and `proofshot stop --session ID`. +5. Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` as unsuccessful. Record the session ID and local evidence pointer. Keep raw JSON/HAR/bodies/credentials/customer data local under `~/data` and out of prompts, scratch, PRs, and uploads. +6. Copy only exercised flows into PR User Testing instructions and dry-run explicit ordered publication with curated screenshots. -After building or modifying UI features, verify with this workflow: - -1. Start: `proofshot start --run "your-dev-command" --port PORT --description "what you are verifying"` - Always use --run so proofshot captures server logs. Only omit --run if the server was started by the user. - If the port is in use, ProofShot leaves that unowned listener alone; choose another port or stop it explicitly. Add --force only if a stale ProofShot session blocks startup. - For configured multi-service projects, let `environment` launch its tmux panes or processes instead of starting duplicate services. Run `proofshot doctor` for dependency or recovery diagnostics. -2. Test: Use `proofshot exec` to navigate, click, fill forms, take screenshots - Record an explicit assertion such as `proofshot exec assert-visible "#expected-result"` before screenshots. -3. Stop: `proofshot stop` — bundles canonical evidence, video, screenshots, and a structured verdict. `FAIL`, `INCOMPLETE`, and `BLOCKED` are unsuccessful. - After an interrupted cleanup, use `proofshot session list` and `proofshot session clean --session ID`; never kill by port or process name. -4. Publish only finalized matching evidence: `proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID`. Repeat either flag for multiple explicit selections. - -Key proofshot exec commands: -- `proofshot exec snapshot -i` — see interactive elements -- `proofshot exec click @e3` — click an element -- `proofshot exec fill @e2 "text"` — fill a form field -- `proofshot exec screenshot step.png` — capture a moment - -Artifacts saved to ./proofshot-artifacts/ including video, screenshots, errors, and summary. +Recover with `proofshot session list` and `proofshot session clean --session ID`; never kill by process name or port. `--force` refuses live sessions and operations. diff --git a/skills/opencode/SKILL.md b/skills/opencode/SKILL.md index 7268484..6cdc0d3 100644 --- a/skills/opencode/SKILL.md +++ b/skills/opencode/SKILL.md @@ -1,79 +1,16 @@ --- name: proofshot -description: Visual verification of UI features. Use after building or modifying any - UI component, page, or visual feature. Starts a verification session with video - recording and error capture, then you drive the browser to test, then stop to - bundle proof artifacts for the human. +description: Explore UI behavior, refine user flows, then record one fresh isolated ProofShot session per reviewable use case. compatibility: opencode --- -# ProofShot - Visual Verification Workflow +# ProofShot visual verification -ProofShot is an open-source, agent-agnostic CLI that lets you verify your own work in a real browser - video proof, screenshots, and error reports, no vendor lock-in. +1. Explore with the approved local agent-browser launcher before recording. Keep project commands on the repository runtime and browser tooling on its separate Node 24/agent-browser 0.34.0 launcher. +2. Maintain a brief under the task's persistent `~/data` directory: User Stories; stable Use Cases with preconditions, starting state, test data, and success criteria; user-level User Flows; Iteration Feedback; and final User Testing instructions. +3. Exercise and simplify every flow. Prefer pointer/keyboard actions, disclose hybrid fill/select/checkbox behavior, and use synthetic DOM mutation only for diagnosis. +4. Record one fresh session per finalized Use Case with an explicit assertion and screenshot. With multiple sessions, use `proofshot exec --session ID ...` and `proofshot stop --session ID`. +5. Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` as unsuccessful. Retain the session ID and local evidence pointer. Keep raw JSON/HAR/bodies/credentials/customer data local under `~/data` and out of prompts, scratch, PRs, and uploads. +6. Copy only exercised flows into PR User Testing instructions and dry-run explicit ordered publication with curated screenshots. -## When to use - -Use ProofShot after: -- Building a new UI feature or page -- Modifying existing UI components -- Fixing a visual bug -- Any change that affects what the user sees - -## The workflow (always follow these 3 steps) - -### Step 1: Start the session - -```bash -proofshot start --run "your-dev-command" --port PORT --description "what you are about to verify" -``` - -This opens a browser and begins recording. If the port is already in use, ProofShot leaves that unowned listener alone and asks you to choose another port or stop it explicitly. - -**Always use `--run`** to let proofshot start and capture your dev server output (server logs appear in the proof report). -Only omit `--run` if the server was explicitly started by the user or another process - without it, no server logs are captured. - -If a previous session was not stopped cleanly, add `--force` to override it. -For configured multi-service projects, let `environment` start the declared tmux panes or direct processes; do not launch duplicate services separately. Use `proofshot doctor` when browser, FFmpeg, or recovery state is unclear. - -### Step 2: Drive the browser and test - -Use proofshot exec to navigate, interact, and verify: - -```bash -proofshot exec snapshot -i # See interactive elements -proofshot exec open http://localhost:PORT/page # Navigate to a page -proofshot exec click @e3 # Click a button -proofshot exec fill @e2 "test@example.com" # Fill a form field -proofshot exec assert-visible "#expected-result" # Record an explicit assertion -proofshot exec screenshot step-NAME.png # Capture key moments -``` - -Take screenshots at important moments - these become the visual proof. -Verify expected outcomes with assertion commands before taking screenshots. A screenshot alone is not a passing assertion. - -### Step 3: Stop and bundle the proof - -```bash -proofshot stop -``` - -This stops recording, collects console + server errors, and generates -a SUMMARY.md with video, screenshots, and error report. -Treat `FAIL`, `INCOMPLETE`, and `BLOCKED` verdicts as unsuccessful verification. If cleanup was interrupted, inspect `proofshot session list` and retry only the exact session with `proofshot session clean --session ID`. - -### Step 4 (optional): Post proof to the PR - -```bash -proofshot pr # Auto-detect PR from current branch -proofshot pr 42 # Target a specific PR number -proofshot pr 42 --session SESSION_ID --screenshot ARTIFACT_ID -``` - -This uploads screenshots and video to GitHub and posts a formatted comment on the PR. By default it uses the official GitHub contents API on a `proofshot-artifacts` branch. Use `--upload-provider github-web-attachments` if you specifically want GitHub attachment URLs. - -## Tips - -- Always include a meaningful --description so the human knows what was tested -- Take screenshots before AND after key actions (e.g., before form submit, after redirect) -- If you find errors during verification, fix them and re-run the workflow -- Use `proofshot pr` after stopping to attach proof directly to the pull request +Recover with `proofshot session list` and `proofshot session clean --session ID`; never kill by process name or port. `--force` refuses live sessions and operations. diff --git a/specs/visual-overlays.md b/specs/visual-overlays.md index c323032..fd52e08 100644 --- a/specs/visual-overlays.md +++ b/specs/visual-overlays.md @@ -103,7 +103,7 @@ For ref-targeted actions (`click @e3`, `fill @e5 "text"`, `type @e4 "hello"`): - Toast works from the action string alone - No ripple rendered (except scroll arrow for `scroll`) -**Viewport caching:** Query viewport once at session start (in `startCommand`) and store in session state. In `execCommand()`, if the action starts with `set viewport`, re-query viewport via eval after execution and update the cached value in session state (call `saveSession()` with the new viewport). +**Viewport caching:** Query viewport once at session start (in `startCommand`) and store it in the registered session state. In `execCommand()`, if the action starts with `set viewport`, re-query viewport via eval after execution and update that exact registry record. ### Failure handling diff --git a/src/artifacts/bundle.test.ts b/src/artifacts/bundle.test.ts new file mode 100644 index 0000000..e051b5d --- /dev/null +++ b/src/artifacts/bundle.test.ts @@ -0,0 +1,242 @@ +import fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { syncBuiltinESMExports } from 'module'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ensureOutputDir } from './bundle.js'; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + syncBuiltinESMExports(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('ensureOutputDir', () => { + it('creates private directories without changing an existing unsafe directory', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const outputDirectory = path.join(root, 'proofshot-artifacts'); + const nestedDirectory = path.join(outputDirectory, 'private', 'environment'); + + expect(ensureOutputDir(outputDirectory)).toBe(outputDirectory); + expect(fs.statSync(outputDirectory).mode & 0o777).toBe(0o700); + expect(ensureOutputDir(nestedDirectory)).toBe(nestedDirectory); + expect(fs.statSync(nestedDirectory).mode & 0o777).toBe(0o700); + expect(() => ensureOutputDir(outputDirectory)).not.toThrow(); + + fs.chmodSync(outputDirectory, 0o755); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(/private/); + expect(fs.statSync(outputDirectory).mode & 0o777).toBe(0o755); + }); + + it('rejects a symlink without changing its target permissions', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const targetDirectory = path.join(root, 'target'); + const outputDirectory = path.join(root, 'proofshot-artifacts'); + fs.mkdirSync(targetDirectory, { mode: 0o700 }); + fs.symlinkSync(targetDirectory, outputDirectory); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(/real directory/); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o700); + }); + + it('rejects a new directory below a symlinked ancestor without creating it', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const targetDirectory = path.join(root, 'target'); + const linkedAncestor = path.join(root, 'linked-output'); + const targetOutputDirectory = path.join(targetDirectory, 'session'); + const outputDirectory = path.join(linkedAncestor, 'session'); + fs.mkdirSync(targetDirectory, { mode: 0o755 }); + fs.chmodSync(targetDirectory, 0o755); + fs.symlinkSync(targetDirectory, linkedAncestor); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(/real directory/); + expect(fs.existsSync(targetOutputDirectory)).toBe(false); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o755); + }); + + it('rejects an existing directory below a symlinked ancestor without changing it', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const targetDirectory = path.join(root, 'target'); + const linkedAncestor = path.join(root, 'linked-output'); + const targetOutputDirectory = path.join(targetDirectory, 'session'); + const outputDirectory = path.join(linkedAncestor, 'session'); + fs.mkdirSync(targetDirectory, { mode: 0o755 }); + fs.chmodSync(targetDirectory, 0o755); + fs.mkdirSync(targetOutputDirectory, { mode: 0o700 }); + fs.symlinkSync(targetDirectory, linkedAncestor); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(/real directory/); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o755); + expect(fs.statSync(targetOutputDirectory).mode & 0o777).toBe(0o700); + }); + + it('rejects a non-directory ancestor without changing it', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const fileAncestor = path.join(root, 'not-a-directory'); + const outputDirectory = path.join(fileAncestor, 'session'); + fs.writeFileSync(fileAncestor, 'unchanged'); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(/real directory/); + expect(fs.readFileSync(fileAncestor, 'utf8')).toBe('unchanged'); + }); + + it('does not create through an ancestor swapped during mkdir', () => { + if (process.platform !== 'linux') { + return; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const outputParent = path.join(root, 'output'); + const displacedOutputParent = path.join(root, 'output-before-race'); + const targetDirectory = path.join(root, 'target'); + const targetOutputDirectory = path.join(targetDirectory, 'session'); + const outputDirectory = path.join(outputParent, 'session'); + fs.mkdirSync(outputParent, { mode: 0o700 }); + fs.mkdirSync(targetDirectory, { mode: 0o755 }); + fs.chmodSync(targetDirectory, 0o755); + + const mkdirSync = fs.mkdirSync; + let swapped = false; + vi.spyOn(fs, 'mkdirSync').mockImplementation((directory, options) => { + if (!swapped && path.basename(String(directory)) === 'session') { + fs.renameSync(outputParent, displacedOutputParent); + fs.symlinkSync(targetDirectory, outputParent); + swapped = true; + } + return mkdirSync(directory, options); + }); + syncBuiltinESMExports(); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(); + expect(fs.existsSync(targetOutputDirectory)).toBe(false); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o755); + }); + + it('keeps descriptor chmod bound to a leaf swapped during validation', () => { + if (process.platform === 'win32') { + return; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const outputDirectory = path.join(root, 'session'); + const displacedOutputDirectory = path.join(root, 'session-before-race'); + const targetDirectory = path.join(root, 'target'); + fs.mkdirSync(targetDirectory, { mode: 0o755 }); + fs.chmodSync(targetDirectory, 0o755); + + const chmodSync = fs.chmodSync; + const fchmodSync = fs.fchmodSync; + let swapped = false; + function swapOutputDirectory(): void { + if (swapped) { + return; + } + fs.renameSync(outputDirectory, displacedOutputDirectory); + fs.symlinkSync(targetDirectory, outputDirectory); + swapped = true; + } + const chmodSpy = vi.spyOn(fs, 'chmodSync').mockImplementation((filePath, mode) => { + swapOutputDirectory(); + chmodSync(filePath, mode); + }); + vi.spyOn(fs, 'fchmodSync').mockImplementation((fileDescriptor, mode) => { + swapOutputDirectory(); + fchmodSync(fileDescriptor, mode); + }); + syncBuiltinESMExports(); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(); + expect(chmodSpy).not.toHaveBeenCalled(); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o755); + }); + + it('does not accept a leaf swapped during descriptor validation', () => { + if (process.platform === 'win32') { + return; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const outputDirectory = path.join(root, 'session'); + const displacedOutputDirectory = path.join(root, 'session-before-race'); + const targetDirectory = path.join(root, 'target'); + fs.mkdirSync(outputDirectory, { mode: 0o700 }); + fs.mkdirSync(targetDirectory, { mode: 0o700 }); + + const accessSync = fs.accessSync; + const fstatSync = fs.fstatSync; + let swapped = false; + function swapOutputDirectory(): void { + if (swapped) { + return; + } + fs.renameSync(outputDirectory, displacedOutputDirectory); + fs.symlinkSync(targetDirectory, outputDirectory); + swapped = true; + } + const accessSpy = vi.spyOn(fs, 'accessSync').mockImplementation((filePath, mode) => { + swapOutputDirectory(); + accessSync(filePath, mode); + }); + vi.spyOn(fs, 'fstatSync').mockImplementation((fileDescriptor, options) => { + swapOutputDirectory(); + return fstatSync(fileDescriptor, options); + }); + syncBuiltinESMExports(); + + expect(() => ensureOutputDir(outputDirectory)).toThrow(); + expect(accessSpy).toHaveBeenCalledOnce(); + expect(fs.lstatSync(outputDirectory).isSymbolicLink()).toBe(true); + expect(fs.statSync(targetDirectory).mode & 0o777).toBe(0o700); + }); + + it('canonicalizes an immutable root-owned system alias ancestor', () => { + if (process.platform === 'win32') { + return; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-output-')); + temporaryDirectories.push(root); + const canonicalTarget = path.join(root, 'canonical-target'); + const realAlias = path.join(root, 'real-alias'); + const virtualAlias = `/proofshot-system-alias-${path.basename(root)}`; + fs.mkdirSync(canonicalTarget, { mode: 0o700 }); + fs.symlinkSync(canonicalTarget, realAlias); + + const lstatSync = fs.lstatSync; + const realpathSync = fs.realpathSync; + const aliasStat = lstatSync(realAlias); + aliasStat.uid = 0; + vi.spyOn(fs, 'lstatSync').mockImplementation((filePath, options) => { + const resolvedPath = path.resolve(String(filePath)); + if (resolvedPath === virtualAlias || resolvedPath === realAlias) { + return aliasStat; + } + return lstatSync(filePath, options); + }); + vi.spyOn(fs, 'realpathSync').mockImplementation((filePath, options) => { + if (path.resolve(String(filePath)) === virtualAlias) { + return canonicalTarget; + } + return realpathSync(filePath, options); + }); + syncBuiltinESMExports(); + + const outputDirectory = path.join(virtualAlias, 'session'); + const canonicalOutputDirectory = path.join(canonicalTarget, 'session'); + expect(ensureOutputDir(outputDirectory)).toBe(canonicalOutputDirectory); + expect(fs.statSync(canonicalOutputDirectory).mode & 0o777).toBe(0o700); + expect(() => ensureOutputDir(virtualAlias)).toThrow(/real directory/); + expect(() => ensureOutputDir(path.join(realAlias, 'unsafe'))).toThrow( + /real directory/, + ); + }); +}); diff --git a/src/artifacts/bundle.ts b/src/artifacts/bundle.ts index cab5e40..15edd30 100644 --- a/src/artifacts/bundle.ts +++ b/src/artifacts/bundle.ts @@ -22,11 +22,193 @@ export interface VerificationResult { durationMs: number; } +type ResolvedDirectoryPath = { + canonicalPath: string; + existingPath: string; + existingStat: fs.Stats; + missingComponents: string[]; +}; + /** - * Ensure the output directory exists. + * Ensure the output directory exists and return its canonical safe path. */ -export function ensureOutputDir(outputDir: string): void { - fs.mkdirSync(outputDir, { recursive: true }); +export function ensureOutputDir(outputDir: string): string { + const resolvedPath = resolveDirectoryPath(outputDir); + let directoryPath = resolvedPath.existingPath; + let directoryDescriptor = openStableDirectory( + resolvedPath.existingPath, + resolvedPath.existingStat, + ); + + try { + for (const component of resolvedPath.missingComponents) { + const childPath = descriptorChildPath( + directoryDescriptor, + directoryPath, + component, + ); + fs.mkdirSync(childPath, { mode: 0o700 }); + const childDescriptor = fs.openSync(childPath, directoryOpenFlags()); + try { + if (process.platform !== 'win32') { + fs.fchmodSync(childDescriptor, 0o700); + } + assertPrivateDirectory( + fs.fstatSync(childDescriptor), + path.join(directoryPath, component), + ); + } catch (error) { + fs.closeSync(childDescriptor); + throw error; + } + + fs.closeSync(directoryDescriptor); + directoryDescriptor = childDescriptor; + directoryPath = path.join(directoryPath, component); + } + + const directoryStat = fs.fstatSync(directoryDescriptor); + assertPrivateDirectory(directoryStat, directoryPath); + fs.accessSync( + directoryPath, + fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK, + ); + assertStableDirectoryPath(directoryPath, directoryStat); + return directoryPath; + } finally { + fs.closeSync(directoryDescriptor); + } +} + +function resolveDirectoryPath(outputDir: string): ResolvedDirectoryPath { + const requestedPath = path.resolve(outputDir); + const root = path.parse(requestedPath).root; + const relativePath = path.relative(root, requestedPath); + const components = relativePath === '' ? [] : relativePath.split(path.sep); + let existingPath = root; + let existingStat = fs.lstatSync(root); + + for (const [index, component] of components.entries()) { + const componentPath = path.join(existingPath, component); + const componentStat = fs.lstatSync(componentPath, { + throwIfNoEntry: false, + }); + if (componentStat === undefined) { + const missingComponents = components.slice(index); + return { + canonicalPath: path.join(existingPath, ...missingComponents), + existingPath, + existingStat, + missingComponents, + }; + } + if (componentStat.isSymbolicLink()) { + const isLeaf = index === components.length - 1; + if (isLeaf || !isTrustedSystemAlias(componentStat, existingStat)) { + throw new Error( + `ProofShot output path component must be a real directory: ${componentPath}`, + ); + } + const canonicalAliasPath = fs.realpathSync(componentPath); + const canonicalAliasStat = fs.lstatSync(canonicalAliasPath); + if ( + !path.isAbsolute(canonicalAliasPath) || + !canonicalAliasStat.isDirectory() || + canonicalAliasStat.isSymbolicLink() + ) { + throw new Error( + `ProofShot output system alias must resolve to a real directory: ${componentPath}`, + ); + } + existingPath = canonicalAliasPath; + existingStat = canonicalAliasStat; + continue; + } + if (!componentStat.isDirectory()) { + throw new Error( + `ProofShot output path component must be a real directory: ${componentPath}`, + ); + } + existingPath = componentPath; + existingStat = componentStat; + } + + return { + canonicalPath: existingPath, + existingPath, + existingStat, + missingComponents: [], + }; +} + +function isTrustedSystemAlias(aliasStat: fs.Stats, parentStat: fs.Stats): boolean { + return ( + aliasStat.uid === 0 && + parentStat.uid === 0 && + (parentStat.mode & 0o022) === 0 + ); +} + +function openStableDirectory(directoryPath: string, expectedStat: fs.Stats): number { + const directoryDescriptor = fs.openSync(directoryPath, directoryOpenFlags()); + const directoryStat = fs.fstatSync(directoryDescriptor); + if ( + !directoryStat.isDirectory() || + directoryStat.dev !== expectedStat.dev || + directoryStat.ino !== expectedStat.ino + ) { + fs.closeSync(directoryDescriptor); + throw new Error(`ProofShot output directory changed during validation: ${directoryPath}`); + } + return directoryDescriptor; +} + +function directoryOpenFlags(): number { + return ( + fs.constants.O_RDONLY | + (fs.constants.O_DIRECTORY ?? 0) | + (fs.constants.O_NOFOLLOW ?? 0) + ); +} + +function descriptorChildPath( + directoryDescriptor: number, + directoryPath: string, + component: string, +): string { + if (process.platform === 'linux') { + return path.join('/proc/self/fd', String(directoryDescriptor), component); + } + // Node exposes descriptor-relative traversal only on Linux. Darwin and Windows + // can detect, but cannot prevent, a same-UID parent replacement during mkdir. + return path.join(directoryPath, component); +} + +function assertPrivateDirectory(directoryStat: fs.Stats, outputDir: string): void { + if (!directoryStat.isDirectory()) { + throw new Error(`ProofShot output must be a real directory: ${outputDir}`); + } + const uid = process.getuid?.(); + if (uid !== undefined && directoryStat.uid !== uid) { + throw new Error(`ProofShot output must be owned by the current user: ${outputDir}`); + } + if (uid !== undefined && (directoryStat.mode & 0o777) !== 0o700) { + throw new Error( + `ProofShot output must already be private; refusing to change permissions: ${outputDir}`, + ); + } +} + +function assertStableDirectoryPath(outputDir: string, descriptorStat: fs.Stats): void { + const resolvedPath = resolveDirectoryPath(outputDir); + if ( + resolvedPath.missingComponents.length > 0 || + resolvedPath.canonicalPath !== outputDir || + resolvedPath.existingStat.dev !== descriptorStat.dev || + resolvedPath.existingStat.ino !== descriptorStat.ino + ) { + throw new Error(`ProofShot output directory changed during validation: ${outputDir}`); + } } /** diff --git a/src/artifacts/evidence.test.ts b/src/artifacts/evidence.test.ts index f02a933..0b4391f 100644 --- a/src/artifacts/evidence.test.ts +++ b/src/artifacts/evidence.test.ts @@ -46,6 +46,10 @@ describe('canonical evidence and verdicts', () => { environmentEvent('Error: Vite stack exploded', 10), environmentEvent('GET /health', 11), environmentEvent('Error: Vite stack exploded', 12), + environmentEvent( + 'Authorization: Basic cHJpdmF0ZTpzZWNyZXQ=', + 12.5, + ), environmentEvent('[process stopped by ProofShot]', 13), ]; fs.writeFileSync( @@ -137,7 +141,7 @@ describe('canonical evidence and verdicts', () => { evidence.sources.find((source) => source.origin === 'browser'), ).toEqual( expect.objectContaining({ - title: 'https://example.test', + title: 'https://example.test/', group: 'browser', }), ); @@ -149,6 +153,9 @@ describe('canonical evidence and verdicts', () => { expect(verdict.mediaTruncated).toBe(true); expect(fs.existsSync(path.join(sessionDir, 'evidence.json'))).toBe(true); expect(fs.existsSync(path.join(sessionDir, 'verdict.json'))).toBe(true); + expect( + fs.readFileSync(path.join(sessionDir, 'evidence.json'), 'utf-8'), + ).not.toContain('cHJpdmF0ZTpzZWNyZXQ='); }); it('does not treat pre-recording lifecycle time as truncated media', () => { @@ -238,6 +245,73 @@ describe('canonical evidence and verdicts', () => { }); expect(verdict.status).toBe('INCOMPLETE'); }); + + it('does not accept synthetic DOM mutation as final behavioral proof', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-synthetic-action-'), + ); + temporaryDirectories.push(sessionDir); + + const { verdict } = writeCanonicalEvidence({ + sessionId: 'synthetic-action', + sessionDir, + durationSec: 1, + videoPath: path.join(sessionDir, 'unused.webm'), + recordingWasActive: false, + consoleEvidenceAvailable: true, + actions: [ + { + action: 'eval [REDACTED_SCRIPT]', + category: 'synthetic-dom', + intent: { + command: 'eval', + summary: 'eval [REDACTED_SCRIPT]', + }, + relativeTimeSec: 0, + timestamp: '2026-08-09T00:00:00.000Z', + outcome: 'passed', + }, + ], + consoleEntries: [], + serverEntries: [], + }); + + expect(verdict.status).toBe('INCOMPLETE'); + expect(verdict.reasons).toContain( + 'Synthetic DOM mutation is diagnostic-only and cannot serve as final behavioral proof.', + ); + }); + + it('marks interrupted browser actions without an outcome as incomplete', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-interrupted-action-'), + ); + temporaryDirectories.push(sessionDir); + + const { verdict } = writeCanonicalEvidence({ + sessionId: 'interrupted-action', + sessionDir, + durationSec: 1, + videoPath: path.join(sessionDir, 'unused.webm'), + recordingWasActive: false, + consoleEvidenceAvailable: true, + actions: [ + { + action: 'click @e1', + category: 'pointer-keyboard', + relativeTimeSec: 0, + timestamp: '2026-08-09T00:00:00.000Z', + }, + ], + consoleEntries: [], + serverEntries: [], + }); + + expect(verdict.status).toBe('INCOMPLETE'); + expect(verdict.reasons).toContain( + '1 browser action(s) had no recorded outcome.', + ); + }); }); function environmentEvent( diff --git a/src/artifacts/evidence.ts b/src/artifacts/evidence.ts index 03840ad..3b32694 100644 --- a/src/artifacts/evidence.ts +++ b/src/artifacts/evidence.ts @@ -3,7 +3,13 @@ import * as path from 'path'; import { createHash, randomUUID } from 'crypto'; import { execFileSync } from 'child_process'; import { PNG } from 'pngjs'; -import type { SessionLogEntry } from '../commands/exec.js'; +import type { SanitizedNetworkSummary } from '../browser/evidence.js'; +import { + sanitizeSessionLogEntry, + type SessionLogEntry, +} from '../session/action-log.js'; +import { sanitizeDiagnosticMessage } from '../browser/provenance.js'; +import type { AgentBrowserRuntimeReceipt } from '../browser/isolation.js'; import { loadEvidenceEvents } from '../environment/evidence.js'; import type { EnvironmentState, @@ -59,6 +65,8 @@ export type CanonicalEvidence = { sources: EvidenceSourceSummary[]; incidents: EvidenceIncident[]; screenshots: ScreenshotIntegrity[]; + network?: SanitizedNetworkSummary | null; + runtime?: AgentBrowserRuntimeReceipt; }; export type Verdict = { @@ -85,17 +93,28 @@ export type EvidenceBuildOptions = { consoleEntries: TimestampedLogEntry[]; serverEntries: TimestampedLogEntry[]; environment?: EnvironmentState | null; + networkSummary?: SanitizedNetworkSummary | null; + networkEvidenceRequired?: boolean; + runtime?: AgentBrowserRuntimeReceipt; }; export function writeCanonicalEvidence( options: EvidenceBuildOptions, ): { evidence: CanonicalEvidence; verdict: Verdict } { - const events = collectEvents(options); + const sanitizedOptions = { + ...options, + actions: options.actions.map(sanitizeSessionLogEntry), + }; + const events = collectEvents(sanitizedOptions); applyPresentationFilters(events, options.environment?.sources || []); - const incidents = buildIncidents(events); - const screenshots = inspectScreenshots(options.sessionDir, options.actions); + const sanitizedEvents = events.map(sanitizeEvidenceEvent); + const incidents = buildIncidents(sanitizedEvents); + const screenshots = inspectScreenshots( + sanitizedOptions.sessionDir, + sanitizedOptions.actions, + ); const mediaDurationSec = probeMediaDuration(options.videoPath); - const actionDuration = options.actions + const actionDuration = sanitizedOptions.actions .map((entry) => entry.relativeTimeSec) .filter(Number.isFinite) .reduce((maximum, current) => Math.max(maximum, current), 0); @@ -106,10 +125,7 @@ export function writeCanonicalEvidence( : Math.max(0, actionDuration - mediaDurationSec); const mediaTruncated = mediaDivergenceSec !== null && mediaDivergenceSec > 1; - const sources = buildSourceSummaries( - events, - incidents, - ); + const sources = buildSourceSummaries(sanitizedEvents, incidents); const evidence: CanonicalEvidence = { version: 1, sessionId: options.sessionId, @@ -118,13 +134,15 @@ export function writeCanonicalEvidence( mediaDurationSec, mediaDivergenceSec, mediaTruncated, - actions: options.actions, - events, + actions: sanitizedOptions.actions, + events: sanitizedEvents, sources, incidents, screenshots, + network: options.networkSummary, + ...(options.runtime ? { runtime: options.runtime } : {}), }; - const verdict = buildVerdict(options, evidence); + const verdict = buildVerdict(sanitizedOptions, evidence); writeJsonAtomically( path.join(options.sessionDir, 'evidence.json'), evidence, @@ -136,6 +154,18 @@ export function writeCanonicalEvidence( return { evidence, verdict }; } +function sanitizeEvidenceEvent(event: EvidenceEvent): EvidenceEvent { + return { + ...event, + group: sanitizeDiagnosticMessage(event.group) || 'environment', + sourceId: sanitizeDiagnosticMessage(event.sourceId) || 'source', + sourceTitle: sanitizeDiagnosticMessage(event.sourceTitle) || 'Source', + text: sanitizeDiagnosticMessage(event.text) || '[REDACTED]', + navigationId: sanitizeDiagnosticMessage(event.navigationId), + pageUrl: sanitizeDiagnosticMessage(event.pageUrl), + }; +} + function writeJsonAtomically(filePath: string, value: unknown): void { const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; try { @@ -528,6 +558,9 @@ function buildVerdict( evidence: CanonicalEvidence, ): Verdict { const missingArtifacts: string[] = []; + if (options.networkEvidenceRequired && !options.networkSummary) { + missingArtifacts.push('network-summary.json'); + } if (options.recordingWasActive && !fs.existsSync(options.videoPath)) { missingArtifacts.push(path.basename(options.videoPath)); } else if ( @@ -579,8 +612,12 @@ function buildVerdict( action.expectedSelector && action.outcome === 'failed', ) .map((action) => action.expectedSelector!); - const pendingExpectedSelectors = options.actions.filter( - (action) => action.expectedSelector && action.outcome === undefined, + const pendingActions = options.actions.filter( + (action) => action.outcome === undefined, + ); + const hasSyntheticDomMutation = options.actions.some( + (action) => + action.category === 'synthetic-dom' || /^eval\b/i.test(action.action), ); const fatalIncidentCount = evidence.incidents.filter( (incident) => incident.severity === 'fatal', @@ -609,14 +646,19 @@ function buildVerdict( ...(evidence.sources.some((source) => source.truncationCount > 0) ? ['One or more evidence sources were truncated.'] : []), - ...(pendingExpectedSelectors.length > 0 + ...(pendingActions.length > 0 ? [ - `${pendingExpectedSelectors.length} expected selector assertion(s) had no recorded outcome.`, + `${pendingActions.length} browser action(s) had no recorded outcome.`, ] : []), ...(reusedScreenshotPaths > 0 ? ['One or more screenshot paths were reused by multiple actions.'] : []), + ...(hasSyntheticDomMutation + ? [ + 'Synthetic DOM mutation is diagnostic-only and cannot serve as final behavioral proof.', + ] + : []), ]; const status: VerdictStatus = blockingReasons.length > 0 diff --git a/src/artifacts/viewer.ts b/src/artifacts/viewer.ts index 6942193..b3ef399 100644 --- a/src/artifacts/viewer.ts +++ b/src/artifacts/viewer.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import type { SessionLogEntry } from '../commands/exec.js'; +import type { SessionLogEntry } from '../session/action-log.js'; import type { CanonicalEvidence, EvidenceSourceSummary, @@ -383,24 +383,24 @@ export function generateViewer(data: ViewerData): string { let consoleLogBodyHtml: string; if (data.consoleEntries && data.consoleEntries.length > 0) { const built = buildTimestampedLogLines(data.consoleEntries); - consoleLogBodyHtml = `
${built.html}
${built.truncated ? '

Log truncated at 2000 entries. See console-output.log for full output.

' : ''}`; + consoleLogBodyHtml = `
${built.html}
${built.truncated ? '

Log truncated at 2000 entries. Raw output is retained privately.

' : ''}`; } else { const consoleTrunc = truncateLog(data.consoleOutput ?? '', MAX_LOG_BYTES); const consoleLogLines = buildLogLines(consoleTrunc.text); consoleLogBodyHtml = consoleLogLines - ? `
${consoleLogLines}
${consoleTrunc.truncated ? '

Log truncated at 50 KB. See console-output.log for full output.

' : ''}` + ? `
${consoleLogLines}
${consoleTrunc.truncated ? '

Log truncated at 50 KB. Raw output is retained privately.

' : ''}` : '

No console output captured

'; } let serverLogBodyHtml: string; if (data.serverEntries && data.serverEntries.length > 0) { const built = buildTimestampedLogLines(data.serverEntries); - serverLogBodyHtml = `
${built.html}
${built.truncated ? '

Log truncated at 2000 entries. See server.log for full output.

' : ''}`; + serverLogBodyHtml = `
${built.html}
${built.truncated ? '

Log truncated at 2000 entries. Raw output is retained privately.

' : ''}`; } else { const serverTrunc = truncateLog(data.serverLog ?? '', MAX_LOG_BYTES); const serverLogLines = buildLogLines(serverTrunc.text); serverLogBodyHtml = serverLogLines - ? `
${serverLogLines}
${serverTrunc.truncated ? '

Log truncated at 50 KB. See server.log for full output.

' : ''}` + ? `
${serverLogLines}
${serverTrunc.truncated ? '

Log truncated at 50 KB. Raw output is retained privately.

' : ''}` : '

No server log captured

'; } diff --git a/src/browser/capture.ts b/src/browser/capture.ts index 64c9be2..a2138bb 100644 --- a/src/browser/capture.ts +++ b/src/browser/capture.ts @@ -4,7 +4,7 @@ import { ab } from '../utils/exec.js'; * Start video recording to the given file path. */ export function startRecording(outputPath: string, sessionName?: string): void { - ab(`record start ${outputPath}`, { timeoutMs: 10000, session: sessionName }); + ab(['record', 'start', outputPath], { timeoutMs: 10000, session: sessionName }); } /** @@ -12,7 +12,7 @@ export function startRecording(outputPath: string, sessionName?: string): void { */ export function stopRecording(sessionName?: string): void { try { - ab('record stop', { timeoutMs: 15000, session: sessionName }); + ab(['record', 'stop'], { timeoutMs: 15000, session: sessionName }); } catch { // Recording may not have started — that's fine } @@ -22,15 +22,20 @@ export function stopRecording(sessionName?: string): void { * Take a screenshot and save to the given path. */ export function takeScreenshot(outputPath: string, fullPage = true, sessionName?: string): void { - const fullFlag = fullPage ? ' --full' : ''; - ab(`screenshot ${outputPath}${fullFlag}`, { timeoutMs: 15000, session: sessionName }); + ab(['screenshot', outputPath, ...(fullPage ? ['--full'] : [])], { + timeoutMs: 15000, + session: sessionName, + }); } /** * Take an annotated screenshot (labels interactive elements). */ export function takeAnnotatedScreenshot(outputPath: string, sessionName?: string): void { - ab(`screenshot ${outputPath} --annotate`, { timeoutMs: 15000, session: sessionName }); + ab(['screenshot', outputPath, '--annotate'], { + timeoutMs: 15000, + session: sessionName, + }); } /** @@ -44,7 +49,7 @@ export function diffScreenshots( sessionName?: string, ): number | null { try { - const result = ab(`diff screenshot ${baseline} ${current} ${outputPath}`, { + const result = ab(['diff', 'screenshot', baseline, current, outputPath], { timeoutMs: 15000, session: sessionName, }); diff --git a/src/browser/command-policy.ts b/src/browser/command-policy.ts new file mode 100644 index 0000000..5420fc0 --- /dev/null +++ b/src/browser/command-policy.ts @@ -0,0 +1,299 @@ +import * as path from 'path'; +import { parseAgentBrowserBatchCommands } from './provenance.js'; + +// The runtime pin makes this a closed 0.34.0 contract. Re-audit the complete +// command surface before changing the version or adding an entry here. +const ALLOWED_COMMANDS = new Set([ + 'back', + 'batch', + 'check', + 'click', + 'console', + 'cookies', + 'dblclick', + 'drag', + 'errors', + 'eval', + 'fill', + 'find', + 'focus', + 'forward', + 'get', + 'goto', + 'hover', + 'is', + 'key', + 'keyboard', + 'keydown', + 'keyup', + 'mouse', + 'navigate', + 'network', + 'open', + 'press', + 'reload', + 'screenshot', + 'scroll', + 'scrollinto', + 'scrollintoview', + 'select', + 'set', + 'snapshot', + 'storage', + 'tab', + 'type', + 'uncheck', + 'upload', + 'wait', +]); +const PROOFSHOT_OWNED_COMMANDS = new Set([ + 'auth', + 'close', + 'connect', + 'record', + 'state', +]); +const RESERVED_AGENT_BROWSER_FLAGS = new Set([ + '--action-policy', + '--allow-file-access', + '--allowed-domains', + '--args', + '--auto-connect', + '--cdp', + '--config', + '--confirm-actions', + '--confirm-interactive', + '--device', + '--download-path', + '--enable', + '--engine', + '--executable-path', + '--extension', + '--headed', + '--ignore-https-errors', + '--init-script', + '--namespace', + '--no-auto-dialog', + '--profile', + '--provider', + '--proxy', + '--proxy-bypass', + '--session', + '--session-name', + '--socket-dir', + '--state', + '--user-agent', + '-p', +]); +const ALLOWED_GET_SUBCOMMANDS = new Set([ + 'attr', + 'box', + 'count', + 'html', + 'styles', + 'text', + 'title', + 'url', + 'value', +]); +const ALLOWED_COOKIE_OPERATIONS = new Set(['clear', 'get', 'set']); +const ALLOWED_IS_SUBCOMMANDS = new Set(['checked', 'enabled', 'visible']); +const ALLOWED_KEYBOARD_SUBCOMMANDS = new Set(['inserttext', 'type']); +const ALLOWED_MOUSE_SUBCOMMANDS = new Set(['down', 'move', 'up', 'wheel']); +const ALLOWED_SET_SUBCOMMANDS = new Set([ + 'credentials', + 'device', + 'geo', + 'headers', + 'media', + 'offline', + 'viewport', +]); +const ALLOWED_STORAGE_TYPES = new Set(['local', 'session']); +const ALLOWED_STORAGE_OPERATIONS = new Set(['clear', 'get', 'set']); +const SCREENSHOT_FLAGS = new Set(['--annotate', '--full', '-f']); +const MAX_BATCH_DEPTH = 8; + +export function assertControlledAgentBrowserCommand(args: string[]): void { + assertControlledAgentBrowserCommandAtDepth(args, 0); +} + +export function prepareControlledAgentBrowserCommand( + args: string[], + sessionDir: string, +): string[] { + assertControlledAgentBrowserCommand(args); + if (args[0]?.toLowerCase() !== 'screenshot') { + return args; + } + + const filename = args.at(-1); + assertScreenshotFilename(filename, args); + return [...args.slice(0, -1), path.join(sessionDir, filename)]; +} + +function assertControlledAgentBrowserCommandAtDepth( + args: string[], + batchDepth: number, +): void { + const command = args[0]?.toLowerCase(); + if (!command) { + throw new Error('An agent-browser command is required.'); + } + if (PROOFSHOT_OWNED_COMMANDS.has(command)) { + throw new Error( + `agent-browser ${command} cannot override ProofShot-owned browser state.`, + ); + } + if (!ALLOWED_COMMANDS.has(command)) { + throw new Error( + `agent-browser ${command} is not permitted inside a ProofShot session.`, + ); + } + if (batchDepth > 0 && command === 'screenshot') { + throw new Error( + 'agent-browser screenshot must run as a separate ProofShot action so its output remains session-local.', + ); + } + + const reservedFlag = args + .map(normalizeFlag) + .find((argument) => RESERVED_AGENT_BROWSER_FLAGS.has(argument)); + if (reservedFlag) { + throw new Error( + `${reservedFlag} is owned by ProofShot and cannot be passed through proofshot exec.`, + ); + } + + switch (command) { + case 'batch': + assertBatchCommands(args, batchDepth); + return; + case 'console': + case 'errors': + if (args.length > 1) { + throw new Error( + `agent-browser ${command} options are not permitted because they can destroy ProofShot evidence.`, + ); + } + return; + case 'cookies': + assertOptionalSubcommand( + command, + args[1], + ALLOWED_COOKIE_OPERATIONS, + ); + return; + case 'get': + assertAllowedSubcommand(command, args[1], ALLOWED_GET_SUBCOMMANDS); + return; + case 'is': + assertAllowedSubcommand(command, args[1], ALLOWED_IS_SUBCOMMANDS); + return; + case 'keyboard': + assertAllowedSubcommand(command, args[1], ALLOWED_KEYBOARD_SUBCOMMANDS); + return; + case 'mouse': + assertAllowedSubcommand(command, args[1], ALLOWED_MOUSE_SUBCOMMANDS); + return; + case 'network': + assertNetworkCommand(args); + return; + case 'screenshot': + assertScreenshotFilename(args.at(-1), args); + return; + case 'set': + assertAllowedSubcommand(command, args[1], ALLOWED_SET_SUBCOMMANDS); + return; + case 'storage': + assertAllowedSubcommand(command, args[1], ALLOWED_STORAGE_TYPES); + assertOptionalSubcommand( + command, + args[2], + ALLOWED_STORAGE_OPERATIONS, + ); + return; + default: + return; + } +} + +function assertBatchCommands(args: string[], batchDepth: number): void { + if (batchDepth >= MAX_BATCH_DEPTH) { + throw new Error('Nested agent-browser batch commands exceed the safe depth.'); + } + const nestedCommands = parseAgentBrowserBatchCommands(args); + if (nestedCommands.length === 0) { + throw new Error('agent-browser batch requires parseable commands.'); + } + for (const nestedArgs of nestedCommands) { + assertControlledAgentBrowserCommandAtDepth(nestedArgs, batchDepth + 1); + } +} + +function assertAllowedSubcommand( + command: string, + subcommand: string | undefined, + allowed: Set, +): void { + const normalized = subcommand?.toLowerCase(); + if (!normalized || !allowed.has(normalized)) { + throw new Error( + `agent-browser ${command} ${subcommand || '(missing)'} is not permitted inside a ProofShot session.`, + ); + } +} + +function assertOptionalSubcommand( + command: string, + subcommand: string | undefined, + allowed: Set, +): void { + if (subcommand && !allowed.has(subcommand.toLowerCase())) { + throw new Error( + `agent-browser ${command} ${subcommand} is not permitted inside a ProofShot session.`, + ); + } +} + +function assertNetworkCommand(args: string[]): void { + const subcommand = args[1]?.toLowerCase(); + if (subcommand === 'har') { + throw new Error('agent-browser network HAR capture is owned by ProofShot.'); + } + if (subcommand !== 'requests') { + throw new Error( + `agent-browser network ${subcommand || '(missing)'} is not permitted inside a ProofShot session.`, + ); + } + if (args.slice(2).some((argument) => normalizeFlag(argument) === '--clear')) { + throw new Error( + 'agent-browser network requests --clear is not permitted because it destroys ProofShot evidence.', + ); + } +} + +function assertScreenshotFilename( + filename: string | undefined, + args: string[], +): asserts filename is string { + const flags = args + .slice(1, -1) + .filter((argument) => argument.startsWith('-')); + if ( + !filename || + filename.startsWith('-') || + path.isAbsolute(filename) || + path.basename(filename) !== filename || + path.extname(filename).toLowerCase() !== '.png' || + flags.some((flag) => !SCREENSHOT_FLAGS.has(normalizeFlag(flag))) + ) { + throw new Error( + 'ProofShot screenshots require one PNG filename directly inside the active session.', + ); + } +} + +function normalizeFlag(argument: string): string { + return argument.toLowerCase().split('=', 1)[0]; +} diff --git a/src/browser/diagnostic-redaction.ts b/src/browser/diagnostic-redaction.ts new file mode 100644 index 0000000..cff8311 --- /dev/null +++ b/src/browser/diagnostic-redaction.ts @@ -0,0 +1,242 @@ +import { isHighConfidenceSecretField } from './redaction-policy.js'; + +const REDACTED = '[REDACTED]'; + +type Assignment = { + replacement: string; + valueEnd: number; + valueStart: number; +}; + +export function redactDiagnosticAssignments(value: string): string { + let copyStart = 0; + let cursor = 0; + let sanitized = ''; + while (cursor < value.length) { + const assignment = readSensitiveAssignment(value, cursor); + if (!assignment) { + cursor += 1; + continue; + } + sanitized += value.slice(copyStart, assignment.valueStart); + sanitized += assignment.replacement; + copyStart = assignment.valueEnd; + cursor = assignment.valueEnd; + } + return sanitized + value.slice(copyStart); +} + +function readSensitiveAssignment( + value: string, + start: number, +): Assignment | null { + const key = readKey(value, start); + if (!key || !isSensitiveKey(key.value)) { + return null; + } + + let separator = key.end; + while (isHorizontalWhitespace(value[separator])) { + separator += 1; + } + if (![':', '='].includes(value[separator] || '')) { + return null; + } + + let valueStart = separator + 1; + while (isWhitespace(value[valueStart])) { + valueStart += 1; + } + if (valueStart >= value.length) { + return null; + } + + const first = value[valueStart]; + if ( + first === '\\' && + (value[valueStart + 1] === '"' || value[valueStart + 1] === "'") + ) { + const quote = value[valueStart + 1]; + return { + replacement: `\\${quote}${REDACTED}\\${quote}`, + valueEnd: findEscapedQuotedEnd(value, valueStart), + valueStart, + }; + } + if (first === '"' || first === "'") { + return { + replacement: `${first}${REDACTED}${first}`, + valueEnd: findQuotedEnd(value, valueStart), + valueStart, + }; + } + if (first === '{' || first === '[') { + return { + replacement: JSON.stringify(REDACTED), + valueEnd: findCompositeEnd(value, valueStart), + valueStart, + }; + } + + // Free-form values have no reliable intra-line boundary. Folded headers and + // diagnostic continuations remain part of the field while they are indented. + return { + replacement: REDACTED, + valueEnd: findFreeFormEnd(value, valueStart), + valueStart, + }; +} + +function readKey( + value: string, + start: number, +): { end: number; value: string } | null { + const first = value[start]; + if ( + first === '\\' && + (value[start + 1] === '"' || value[start + 1] === "'") + ) { + const end = findEscapedQuotedEnd(value, start); + const hasClosingQuote = + end > start + 3 && + value[end - 2] === '\\' && + value[end - 1] === value[start + 1]; + return hasClosingQuote + ? { end, value: value.slice(start + 2, end - 2) } + : null; + } + if (first === '"' || first === "'") { + if (start > 0 && value[start - 1] === '\\') { + return null; + } + const end = findQuotedEnd(value, start); + return end > start + 1 && value[end - 1] === first + ? { end, value: value.slice(start + 1, end - 1) } + : null; + } + if (!/[A-Za-z0-9_-]/.test(first || '')) { + return null; + } + if (start > 0 && /[A-Za-z0-9_-]/.test(value[start - 1])) { + return null; + } + let end = start; + while (/[A-Za-z0-9_-]/.test(value[end] || '')) { + end += 1; + } + return { end, value: value.slice(start, end) }; +} + +function isSensitiveKey(key: string): boolean { + return isHighConfidenceSecretField(key); +} + +function isHorizontalWhitespace(value: string | undefined): boolean { + return value !== undefined && /\s/.test(value) && !/[\r\n]/.test(value); +} + +function isWhitespace(value: string | undefined): boolean { + return value !== undefined && /\s/.test(value); +} + +function findQuotedEnd(value: string, start: number): number { + const quote = value[start]; + let escaped = false; + for (let index = start + 1; index < value.length; index += 1) { + const character = value[index]; + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === quote) { + return index + 1; + } + } + return value.length; +} + +function findEscapedQuotedEnd(value: string, start: number): number { + const quote = value[start + 1]; + for (let index = start + 2; index < value.length - 1; index += 1) { + if ( + value[index] === '\\' && + value[index + 1] === quote && + value[index - 1] !== '\\' + ) { + return index + 2; + } + } + return value.length; +} + +function findFreeFormEnd(value: string, start: number): number { + let end = findLineEnd(value, start); + while (end < value.length) { + let probe = end; + while (probe < value.length) { + const nextLineStart = skipLineBreak(value, probe); + let contentStart = nextLineStart; + while (isHorizontalWhitespace(value[contentStart])) { + contentStart += 1; + } + const nextLineEnd = findLineEnd(value, contentStart); + if (contentStart === nextLineEnd) { + probe = nextLineEnd; + continue; + } + if (contentStart === nextLineStart) { + return end; + } + end = nextLineEnd; + break; + } + if (probe >= value.length) { + return end; + } + } + return end; +} + +function findLineEnd(value: string, start: number): number { + let end = start; + while (end < value.length && !/[\r\n]/.test(value[end])) { + end += 1; + } + return end; +} + +function skipLineBreak(value: string, start: number): number { + return value[start] === '\r' && value[start + 1] === '\n' + ? start + 2 + : start + 1; +} + +function findCompositeEnd(value: string, start: number): number { + const closers = [value[start] === '{' ? '}' : ']']; + let quote: '"' | "'" | null = null; + let escaped = false; + for (let index = start + 1; index < value.length; index += 1) { + const character = value[index]; + if (quote) { + if (escaped) { + escaped = false; + } else if (character === '\\') { + escaped = true; + } else if (character === quote) { + quote = null; + } + continue; + } + if (character === '"' || character === "'") { + quote = character; + } else if (character === '{' || character === '[') { + closers.push(character === '{' ? '}' : ']'); + } else if (character === closers[closers.length - 1]) { + closers.pop(); + if (closers.length === 0) { + return index + 1; + } + } + } + return value.length; +} diff --git a/src/browser/evidence.test.ts b/src/browser/evidence.test.ts new file mode 100644 index 0000000..49588b0 --- /dev/null +++ b/src/browser/evidence.test.ts @@ -0,0 +1,530 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + buildSanitizedNetworkSummary, + finalizePrivateNetworkCapture, + formatAgentBrowserOutputForDisplay, + sanitizeAgentBrowserError, + writePrivateAgentBrowserResult, +} from './evidence.js'; +import { + CONVENTIONAL_FUSED_SECRET_FIELDS, + PUBLIC_FUSED_FIELDS, +} from './redaction-policy.test-data.js'; + +const mocks = vi.hoisted(() => ({ ab: vi.fn() })); + +vi.mock('../utils/exec.js', async (importOriginal) => ({ + ...(await importOriginal()), + ab: mocks.ab, +})); + +afterEach(() => { + mocks.ab.mockReset(); +}); + +describe('private browser evidence', () => { + it('produces a deterministic metadata-only network summary', () => { + const entries = [ + { + time: 20.12345, + request: { + method: 'POST', + url: 'https://example.com/z?token=private-token', + headers: [{ name: 'authorization', value: 'private-auth' }], + }, + response: { + status: 500, + content: { text: 'private-response-body' }, + _error: 'request failed with token=private-error', + }, + }, + { + time: 5, + request: { + method: 'GET', + url: 'https://example.com/a?customer=private-customer', + }, + response: { status: 200 }, + }, + ]; + + const summary = buildSanitizedNetworkSummary({ log: { entries } }); + const reversed = buildSanitizedNetworkSummary({ + log: { entries: [...entries].reverse() }, + }); + + expect(summary).toEqual(reversed); + expect(summary.requests).toEqual([ + { + endpoint: 'https://example.com/a', + method: 'GET', + status: 200, + durationMs: 5, + error: null, + }, + { + endpoint: 'https://example.com/z', + method: 'POST', + status: 500, + durationMs: 20.123, + error: 'request failed with token=[REDACTED]', + }, + ]); + const serialized = JSON.stringify(summary); + expect(serialized).not.toContain('private-token'); + expect(serialized).not.toContain('private-auth'); + expect(serialized).not.toContain('private-response-body'); + expect(serialized).not.toContain('private-customer'); + expect(serialized).not.toContain('private-error'); + }); + + it('redacts signed URL keys and sensitive endpoint path values', () => { + const summary = buildSanitizedNetworkSummary({ + log: { + entries: [ + { + time: 1, + request: { + method: 'GET', + url: 'https://example.test/download/token/private-path?signature=private-signature', + }, + response: { status: 200 }, + }, + ], + }, + }); + + const serialized = JSON.stringify(summary); + expect(serialized).not.toContain('private-path'); + expect(serialized).not.toContain('private-signature'); + expect(summary.requests[0]?.endpoint).toBe( + 'https://example.test/download/token/%5BREDACTED%5D', + ); + }); + + it('keeps raw structured results only in private action evidence', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-private-result-'), + ); + try { + const receipt = writePrivateAgentBrowserResult({ + args: ['cookies', 'get'], + sessionDir, + rawOutput: JSON.stringify({ + success: true, + data: { + cookies: [{ name: 'session', value: 'private-cookie' }], + }, + }), + success: true, + }); + const evidencePath = path.join(sessionDir, receipt.evidencePath); + const evidence = fs.readFileSync(evidencePath, 'utf-8'); + expect(evidence).toContain('private-cookie'); + expect(receipt.evidencePath).toMatch(/^private\/agent-browser\/actions\//); + expect(fs.statSync(evidencePath).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); + + it('keeps safe observations useful while redacting secret-bearing results', () => { + expect( + formatAgentBrowserOutputForDisplay({ + args: ['snapshot', '-i'], + rawOutput: JSON.stringify({ + success: true, + data: { snapshot: 'button "Submit" [ref=e1]' }, + }), + success: true, + }), + ).toBe('button "Submit" [ref=e1]'); + expect( + formatAgentBrowserOutputForDisplay({ + args: ['fill', '@e1', 'private-value'], + rawOutput: JSON.stringify({ + success: true, + data: { value: 'private-value' }, + }), + success: true, + }), + ).toBe('[REDACTED]'); + expect( + sanitizeAgentBrowserError( + ['type', '@e1', 'private-value'], + 'failed to type private-value', + ), + ).not.toContain('private-value'); + }); + + it('applies the shared secret policy to structured browser results', () => { + const display = formatAgentBrowserOutputForDisplay({ + args: ['snapshot', '-i'], + rawOutput: JSON.stringify({ + success: true, + data: { + auth: 'private-auth', + cookies: [{ name: 'theme', value: 'private-cookie' }], + session: 'private-session', + signature: 'private-signature', + code: 'visible-code', + key: 'visible-key', + status: 'visible', + }, + }), + success: true, + }); + const result = JSON.parse(display) as { + data: Record; + }; + + expect(result.data).toMatchObject({ + auth: '[REDACTED]', + cookies: '[REDACTED]', + session: '[REDACTED]', + signature: '[REDACTED]', + code: 'visible-code', + key: 'visible-key', + status: 'visible', + }); + }); + + it.each([ + ['tokens', 'private-tokens'], + ['accessTokens', 'private-access-tokens'], + ['sessions', 'private-sessions'], + ['signatures', 'private-signatures'], + ['passwords', 'private-passwords'], + ['secrets', 'private-secrets'], + ])('redacts plural structured field %s', (field, secret) => { + const display = formatAgentBrowserOutputForDisplay({ + args: ['snapshot', '-i'], + rawOutput: JSON.stringify({ + success: true, + data: { + [field]: secret, + code: 'visible-code', + codes: 'visible-codes', + key: 'visible-key', + keys: 'visible-keys', + }, + }), + success: true, + }); + const result = JSON.parse(display) as { + data: Record; + }; + + expect(result.data[field]).toBe('[REDACTED]'); + expect(result.data.code).toBe('visible-code'); + expect(result.data.codes).toBe('visible-codes'); + expect(result.data.key).toBe('visible-key'); + expect(result.data.keys).toBe('visible-keys'); + }); + + it.each([...CONVENTIONAL_FUSED_SECRET_FIELDS, 'AwsAccessKeyId'])( + 'redacts fused structured field %s', + (field) => { + const secret = `private-${field.toLowerCase()}`; + + const display = formatAgentBrowserOutputForDisplay({ + args: ['snapshot', '-i'], + rawOutput: JSON.stringify({ + success: true, + data: { + [field]: secret, + monkey: 'visible', + decode: 'visible', + bodyguard: 'visible', + tokenizer: 'visible', + }, + }), + success: true, + }); + const result = JSON.parse(display) as { + data: Record; + }; + + expect(result.data[field]).toBe('[REDACTED]'); + expect(result.data.monkey).toBe('visible'); + expect(result.data.decode).toBe('visible'); + expect(result.data.bodyguard).toBe('visible'); + expect(result.data.tokenizer).toBe('visible'); + }, + ); + + it.each([ + 'value', + 'data-access-token', + 'AwsAccessKeyId', + ...CONVENTIONAL_FUSED_SECRET_FIELDS, + ])( + 'redacts get attr output for secret attribute %s', + (attribute) => { + expect( + formatAgentBrowserOutputForDisplay({ + args: ['get', 'attr', '@e1', attribute], + rawOutput: JSON.stringify({ + success: true, + data: { value: 'private-value', origin: 'attribute' }, + }), + success: true, + }), + ).toBe('[REDACTED]'); + }, + ); + + it.each([ + ['aria-label', 'Submit'] as const, + ['data-testid', 'submit-button'] as const, + ...[...PUBLIC_FUSED_FIELDS, 'code', 'codes', 'key', 'keys'].map( + (attribute) => [attribute, `visible-${attribute}`] as const, + ), + ])('keeps safe get attr output useful for %s', (attribute, value) => { + expect( + formatAgentBrowserOutputForDisplay({ + args: ['get', 'attr', '@e1', attribute], + rawOutput: JSON.stringify({ + success: true, + data: { value, origin: 'attribute' }, + }), + success: true, + }), + ).toBe(value); + }); + + it.each([ + '--auth=private-auth', + '--cookies=private-cookie', + '--session=private-session', + '--signature=private-signature', + ])('redacts output from shared secret-bearing flag %s', (flag) => { + expect( + formatAgentBrowserOutputForDisplay({ + args: ['snapshot', flag], + rawOutput: JSON.stringify({ + success: true, + data: { status: 'private-output' }, + }), + success: true, + }), + ).toBe('[REDACTED]'); + }); + + it('still stops HAR capture when the request inventory command fails', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-network-finalize-'), + ); + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + const paths = { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; + mocks.ab.mockImplementation((command: string[]) => { + if (command.slice(0, 3).join(' ') === 'network har stop') { + fs.writeFileSync( + `${paths.harPath}.pending`, + JSON.stringify({ log: { entries: [] } }), + ); + return JSON.stringify({ success: true }); + } + if (command.join(' ') === 'network requests --json') { + throw new Error('request inventory failed'); + } + throw new Error(`unexpected command: ${command.join(' ')}`); + }); + + try { + const summary = finalizePrivateNetworkCapture('ps-test', paths); + + expect(summary.requestCount).toBe(0); + expect(mocks.ab).toHaveBeenCalledTimes(2); + expect(mocks.ab.mock.calls[0][0]).toContain(`${paths.harPath}.pending`); + expect(mocks.ab.mock.calls[1][0]).toEqual(['network', 'requests', '--json']); + expect(fs.existsSync(`${paths.harPath}.pending`)).toBe(false); + expect(fs.existsSync(paths.harPath)).toBe(true); + expect(fs.readFileSync(paths.requestsPath, 'utf-8')).toContain( + 'request inventory failed', + ); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); + + it('adopts a valid pending HAR when the stop command reports failure', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-network-stop-error-'), + ); + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + const paths = { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; + mocks.ab.mockImplementation((command: string[]) => { + if (command.slice(0, 3).join(' ') === 'network har stop') { + fs.writeFileSync( + `${paths.harPath}.pending`, + JSON.stringify({ log: { entries: [] } }), + ); + throw new Error('HAR stop response was lost'); + } + if (command.join(' ') === 'network requests --json') { + return JSON.stringify({ success: true, data: [] }); + } + throw new Error(`unexpected command: ${command.join(' ')}`); + }); + + try { + expect(finalizePrivateNetworkCapture('ps-test', paths)).toMatchObject({ + requestCount: 0, + }); + expect(fs.existsSync(paths.harPath)).toBe(true); + expect(fs.existsSync(`${paths.harPath}.pending`)).toBe(false); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); + + it('adopts a complete pending HAR after interrupted finalization', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-network-recovery-'), + ); + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + fs.mkdirSync(privateDirectory, { recursive: true }); + const paths = { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; + fs.writeFileSync( + `${paths.harPath}.pending`, + JSON.stringify({ + log: { + entries: [ + { + time: 4, + request: { method: 'GET', url: 'https://example.com/recovered' }, + response: { status: 200 }, + }, + ], + }, + }), + ); + mocks.ab.mockImplementation((command: string[]) => { + if (command.join(' ') === 'network requests --json') { + return JSON.stringify({ success: true, data: [] }); + } + throw new Error(`unexpected command: ${command.join(' ')}`); + }); + + try { + const summary = finalizePrivateNetworkCapture('ps-test', paths); + + expect(summary.requests).toEqual([ + { + endpoint: 'https://example.com/recovered', + method: 'GET', + status: 200, + durationMs: 4, + error: null, + }, + ]); + expect(mocks.ab).toHaveBeenCalledTimes(1); + expect(fs.existsSync(`${paths.harPath}.pending`)).toBe(false); + expect(fs.statSync(paths.harPath).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); + + it('adopts existing HAR evidence without browser commands after browser loss', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-network-offline-recovery-'), + ); + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + fs.mkdirSync(privateDirectory, { recursive: true }); + const paths = { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; + fs.writeFileSync( + paths.harPath, + JSON.stringify({ + log: { + entries: [ + { + time: 9, + request: { method: 'GET', url: 'https://example.com/offline' }, + response: { status: 204 }, + }, + ], + }, + }), + ); + fs.writeFileSync(paths.requestsPath, 'retained request evidence\n'); + + try { + const summary = finalizePrivateNetworkCapture('ps-test', paths, { + allowBrowserCommands: false, + }); + + expect(summary.requests).toEqual([ + { + endpoint: 'https://example.com/offline', + method: 'GET', + status: 204, + durationMs: 9, + error: null, + }, + ]); + expect(mocks.ab).not.toHaveBeenCalled(); + expect(fs.readFileSync(paths.requestsPath, 'utf-8')).toBe( + 'retained request evidence\n', + ); + expect(fs.statSync(paths.harPath).mode & 0o777).toBe(0o600); + expect(fs.statSync(paths.summaryPath).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); + + it('refuses an invalid HAR without adopting a partial final file', () => { + const sessionDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'proofshot-network-invalid-'), + ); + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + fs.mkdirSync(privateDirectory, { recursive: true }); + const paths = { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; + mocks.ab.mockImplementation((command: string[]) => { + if (command.slice(0, 3).join(' ') === 'network har stop') { + fs.writeFileSync(`${paths.harPath}.pending`, '{'); + return JSON.stringify({ success: true }); + } + throw new Error(`unexpected command: ${command.join(' ')}`); + }); + + try { + expect(() => finalizePrivateNetworkCapture('ps-test', paths)).toThrow( + 'agent-browser did not write valid HAR evidence', + ); + expect(fs.existsSync(paths.harPath)).toBe(false); + } finally { + fs.rmSync(sessionDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/browser/evidence.ts b/src/browser/evidence.ts new file mode 100644 index 0000000..379007b --- /dev/null +++ b/src/browser/evidence.ts @@ -0,0 +1,530 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { randomUUID } from 'crypto'; +import { + sanitizeDiagnosticMessage, + sanitizePageUrl, +} from './provenance.js'; +import { + isHighConfidenceSecretField, + isSecretBearingCommandArgument, +} from './redaction-policy.js'; +import { ab } from '../utils/exec.js'; + +export type AgentBrowserResultReceipt = { + success: boolean; + evidencePath: string; + error?: string; +}; + +type AgentBrowserOutputOptions = { + args: string[]; + rawOutput: string; + success: boolean; + error?: string; +}; + +export type PrivateNetworkEvidencePaths = { + privateDirectory: string; + harPath: string; + requestsPath: string; + summaryPath: string; +}; + +export type FinalizePrivateNetworkCaptureOptions = { + allowBrowserCommands?: boolean; +}; + +export type SanitizedNetworkRequest = { + endpoint: string; + method: string; + status: number; + durationMs: number | null; + error: string | null; +}; + +export type SanitizedNetworkSummary = { + version: 1; + requestCount: number; + requests: SanitizedNetworkRequest[]; +}; + +export function preparePrivateNetworkEvidence( + sessionDir: string, +): PrivateNetworkEvidencePaths { + const privateDirectory = path.join(sessionDir, 'private', 'agent-browser'); + preparePrivateDirectory(privateDirectory); + return { + privateDirectory, + harPath: path.join(privateDirectory, 'network.har'), + requestsPath: path.join(privateDirectory, 'requests.json'), + summaryPath: path.join(sessionDir, 'network-summary.json'), + }; +} + +export function startPrivateNetworkCapture(sessionName: string): void { + ab(['network', 'har', 'start', '--content', 'text'], { + session: sessionName, + }); +} + +export function finalizePrivateNetworkCapture( + sessionName: string, + paths: PrivateNetworkEvidencePaths, + options: FinalizePrivateNetworkCaptureOptions = {}, +): SanitizedNetworkSummary { + preparePrivateDirectory(paths.privateDirectory); + const allowBrowserCommands = options.allowBrowserCommands !== false; + const har = finalizePrivateHarCapture( + sessionName, + paths.harPath, + allowBrowserCommands, + ); + if (fs.existsSync(paths.requestsPath)) { + fs.chmodSync(paths.requestsPath, 0o600); + } else if (allowBrowserCommands) { + let requests: string; + try { + requests = ab(['network', 'requests', '--json'], { + session: sessionName, + }); + } catch (error) { + requests = JSON.stringify({ + success: false, + data: null, + error: sanitizeDiagnosticMessage( + error instanceof Error ? error.message : String(error), + ), + }); + } + writePrivateTextFile( + paths.requestsPath, + requests.trim() ? `${requests.trim()}\n` : '{"success":true,"data":[]}\n', + ); + } + const summary = buildSanitizedNetworkSummary(har); + writeJsonFile(paths.summaryPath, summary); + return summary; +} + +function finalizePrivateHarCapture( + sessionName: string, + harPath: string, + allowBrowserCommands: boolean, +): unknown { + const finalizedHar = readValidHarFile(harPath); + if (finalizedHar !== null) { + fs.chmodSync(harPath, 0o600); + return finalizedHar; + } + + const pendingPath = `${harPath}.pending`; + const pendingHar = readValidHarFile(pendingPath); + if (pendingHar !== null) { + adoptPendingHar(pendingPath, harPath); + return pendingHar; + } + if (!allowBrowserCommands) { + throw new Error('No valid local HAR evidence was available for recovery.'); + } + removeInvalidHarFile(pendingPath); + + let stopError: unknown; + try { + ab(['network', 'har', 'stop', pendingPath, '--json'], { + session: sessionName, + }); + } catch (error) { + stopError = error; + } + + const capturedHar = readValidHarFile(pendingPath); + if (capturedHar === null) { + if (stopError) { + throw stopError; + } + throw new Error('agent-browser did not write valid HAR evidence.'); + } + adoptPendingHar(pendingPath, harPath); + return capturedHar; +} + +function readValidHarFile(filePath: string): unknown | null { + if (!fs.existsSync(filePath)) { + return null; + } + try { + const value = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as unknown; + buildSanitizedNetworkSummary(value); + return value; + } catch { + return null; + } +} + +function adoptPendingHar(pendingPath: string, harPath: string): void { + fs.chmodSync(pendingPath, 0o600); + if (fs.existsSync(harPath)) { + fs.unlinkSync(harPath); + } + fs.renameSync(pendingPath, harPath); + fs.chmodSync(harPath, 0o600); +} + +function removeInvalidHarFile(filePath: string): void { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } +} + +export function loadSanitizedNetworkSummary( + summaryPath: string | undefined, +): SanitizedNetworkSummary | null { + if (!summaryPath || !fs.existsSync(summaryPath)) { + return null; + } + try { + const value = JSON.parse(fs.readFileSync(summaryPath, 'utf-8')) as unknown; + return isSanitizedNetworkSummary(value) ? value : null; + } catch { + return null; + } +} + +export function buildSanitizedNetworkSummary( + value: unknown, +): SanitizedNetworkSummary { + const entries = readHarEntries(value); + const requests = entries.map(toSanitizedRequest).sort(compareRequests); + return { + version: 1, + requestCount: requests.length, + requests, + }; +} + +export function writePrivateAgentBrowserResult(options: { + args: string[]; + sessionDir: string; + rawOutput: string; + success: boolean; + error?: string; +}): AgentBrowserResultReceipt { + const actionsDirectory = path.join( + options.sessionDir, + 'private', + 'agent-browser', + 'actions', + ); + preparePrivateDirectory(actionsDirectory); + const filePath = path.join( + actionsDirectory, + `${Date.now()}-${randomUUID()}.json`, + ); + const result = parseStructuredResult( + options.rawOutput, + options.success, + options.error, + ); + writeJsonFile(filePath, result); + const evidencePath = path + .relative(options.sessionDir, filePath) + .split(path.sep) + .join(path.posix.sep); + return { + success: readSuccess(result, options.success), + evidencePath, + ...(options.error + ? { error: sanitizeAgentBrowserError(options.args, options.error) } + : {}), + }; +} + +export function formatAgentBrowserOutputForDisplay( + options: AgentBrowserOutputOptions, +): string { + const result = sanitizeStructuredResult( + parseStructuredResult(options.rawOutput, options.success, options.error), + options.args, + ); + if (typeof result !== 'object' || result === null) { + return String(result); + } + const data = (result as { data?: unknown }).data; + if ( + typeof data === 'string' || + typeof data === 'number' || + typeof data === 'boolean' + ) { + return String(data); + } + if (typeof data !== 'object' || data === null) { + return JSON.stringify(result); + } + const record = data as Record; + for (const key of ['url', 'title', 'text', 'value', 'snapshot']) { + if (typeof record[key] === 'string') { + return record[key]; + } + } + return JSON.stringify(result); +} + +export function sanitizeAgentBrowserError( + args: string[], + error: string | undefined, +): string { + if (commandResultMayContainSecrets(args)) { + return 'agent-browser command failed; sensitive details were redacted'; + } + return sanitizeDiagnosticMessage(error) || 'agent-browser failed'; +} + +function sanitizeStructuredResult(value: unknown, args: string[]): unknown { + if (commandResultMayContainSecrets(args)) { + const record = readRecord(value); + return { + success: + typeof record.success === 'boolean' ? record.success : true, + data: '[REDACTED]', + ...(typeof record.error === 'string' + ? { error: sanitizeAgentBrowserError(args, record.error) } + : {}), + }; + } + return sanitizeStructuredValue(value); +} + +function commandResultMayContainSecrets(args: string[]): boolean { + const command = args[0]?.toLowerCase(); + if (!command) { + return true; + } + if (args.slice(1).some(isSecretBearingCommandArgument)) { + return true; + } + if ( + [ + 'auth', + 'batch', + 'cookies', + 'eval', + 'fill', + 'network', + 'select', + 'storage', + 'type', + 'upload', + ].includes(command) + ) { + return true; + } + if ( + command === 'keyboard' && + ['inserttext', 'type'].includes(args[1]?.toLowerCase()) + ) { + return true; + } + if ( + command === 'set' && + ['credentials', 'headers'].includes(args[1]?.toLowerCase()) + ) { + return true; + } + if (command === 'get') { + const getAction = args[1]?.toLowerCase(); + if (getAction === 'value') { + return true; + } + if (getAction === 'attr') { + const attribute = args[3]; + return ( + typeof attribute === 'string' && + (attribute.toLowerCase() === 'value' || + isHighConfidenceSecretField(attribute)) + ); + } + } + if (command === 'find') { + return ['fill', 'select', 'type'].some((action) => + args.some((argument) => argument.toLowerCase() === action), + ); + } + return false; +} + +function sanitizeStructuredValue(value: unknown): unknown { + if (typeof value === 'string') { + return sanitizeDiagnosticMessage(value); + } + if (Array.isArray(value)) { + return value.map(sanitizeStructuredValue); + } + if (typeof value !== 'object' || value === null) { + return value; + } + const record = value as Record; + const namedSecret = + typeof record.name === 'string' && + isHighConfidenceSecretField(record.name); + return Object.fromEntries( + Object.entries(record).map(([key, entry]) => { + const isSensitive = + isHighConfidenceSecretField(key) || (namedSecret && key === 'value'); + return [key, isSensitive ? '[REDACTED]' : sanitizeStructuredValue(entry)]; + }), + ); +} + +function parseStructuredResult( + rawOutput: string, + success: boolean, + error?: string, +): unknown { + if (rawOutput.trim()) { + try { + return JSON.parse(rawOutput); + } catch { + return { + success, + data: { text: rawOutput }, + ...(error ? { error } : {}), + }; + } + } + return { + success, + data: null, + ...(error ? { error } : {}), + }; +} + +function readSuccess(result: unknown, fallback: boolean): boolean { + if (typeof result !== 'object' || result === null) { + return fallback; + } + const success = (result as { success?: unknown }).success; + return typeof success === 'boolean' ? success : fallback; +} + +function preparePrivateDirectory(directory: string): void { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + let current = directory; + while (current !== path.dirname(current)) { + fs.chmodSync(current, 0o700); + if (path.basename(current) === 'private') { + break; + } + current = path.dirname(current); + } +} + +function readHarEntries(value: unknown): Record[] { + if (typeof value !== 'object' || value === null) { + throw new Error('HAR evidence root must be an object.'); + } + const log = (value as { log?: unknown }).log; + if (typeof log !== 'object' || log === null) { + throw new Error('HAR evidence is missing its log object.'); + } + const entries = (log as { entries?: unknown }).entries; + if (!Array.isArray(entries)) { + throw new Error('HAR evidence is missing its entries array.'); + } + return entries.filter( + (entry): entry is Record => + typeof entry === 'object' && entry !== null, + ); +} + +function toSanitizedRequest(entry: Record): SanitizedNetworkRequest { + const request = readRecord(entry.request); + const response = readRecord(entry.response); + const rawUrl = typeof request.url === 'string' ? request.url : ''; + const rawDuration = typeof entry.time === 'number' ? entry.time : Number.NaN; + const rawError = + typeof entry._error === 'string' + ? entry._error + : typeof response._error === 'string' + ? response._error + : null; + return { + endpoint: sanitizeEndpoint(rawUrl), + method: typeof request.method === 'string' ? request.method : '', + status: typeof response.status === 'number' ? response.status : 0, + durationMs: Number.isFinite(rawDuration) + ? parseFloat(rawDuration.toFixed(3)) + : null, + error: rawError ? sanitizeDiagnosticMessage(rawError) || 'request failed' : null, + }; +} + +function sanitizeEndpoint(value: string): string { + try { + const url = new URL(sanitizePageUrl(value) || value); + return `${url.origin}${url.pathname}`; + } catch { + return sanitizeDiagnosticMessage(value) || ''; + } +} + +function compareRequests( + left: SanitizedNetworkRequest, + right: SanitizedNetworkRequest, +): number { + return ( + left.endpoint.localeCompare(right.endpoint) || + left.method.localeCompare(right.method) || + left.status - right.status || + (left.durationMs ?? -1) - (right.durationMs ?? -1) || + (left.error || '').localeCompare(right.error || '') + ); +} + +function readRecord(value: unknown): Record { + return typeof value === 'object' && value !== null + ? (value as Record) + : {}; +} + +function isSanitizedNetworkSummary( + value: unknown, +): value is SanitizedNetworkSummary { + if (typeof value !== 'object' || value === null) { + return false; + } + const summary = value as Partial; + return ( + summary.version === 1 && + typeof summary.requestCount === 'number' && + Array.isArray(summary.requests) + ); +} + +function writePrivateTextFile(filePath: string, contents: string): void { + const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(temporaryPath, contents, { mode: 0o600 }); + fs.renameSync(temporaryPath, filePath); + fs.chmodSync(filePath, 0o600); + } finally { + if (fs.existsSync(temporaryPath)) { + fs.unlinkSync(temporaryPath); + } + } +} + +function writeJsonFile(filePath: string, value: unknown): void { + const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(value, null, 2) + '\n', { + mode: 0o600, + }); + fs.renameSync(temporaryPath, filePath); + fs.chmodSync(filePath, 0o600); + } finally { + if (fs.existsSync(temporaryPath)) { + fs.unlinkSync(temporaryPath); + } + } +} diff --git a/src/browser/interact.ts b/src/browser/interact.ts index e1ae392..544ba7f 100644 --- a/src/browser/interact.ts +++ b/src/browser/interact.ts @@ -4,33 +4,33 @@ import { ab } from '../utils/exec.js'; * Click an element by its ref (e.g., @e3). */ export function click(ref: string, sessionName?: string): void { - ab(`click ${ref}`, { timeoutMs: 10000, session: sessionName }); + ab(['click', ref], { timeoutMs: 10000, session: sessionName }); } /** * Fill a form field by its ref. */ export function fill(ref: string, value: string, sessionName?: string): void { - ab(`fill ${ref} "${value.replace(/"/g, '\\"')}"`, { timeoutMs: 10000, session: sessionName }); + ab(['fill', ref, value], { timeoutMs: 10000, session: sessionName }); } /** * Type text (keyboard input, not targeting a specific element). */ export function type(text: string, sessionName?: string): void { - ab(`type "${text.replace(/"/g, '\\"')}"`, { timeoutMs: 10000, session: sessionName }); + ab(['type', text], { timeoutMs: 10000, session: sessionName }); } /** * Press a key (e.g., Enter, Tab, Escape). */ export function press(key: string, sessionName?: string): void { - ab(`press ${key}`, { timeoutMs: 5000, session: sessionName }); + ab(['press', key], { timeoutMs: 5000, session: sessionName }); } /** * Scroll the page in a direction. */ export function scroll(direction: 'up' | 'down' = 'down', amount = 3, sessionName?: string): void { - ab(`scroll ${direction} ${amount}`, { timeoutMs: 5000, session: sessionName }); + ab(['scroll', direction, String(amount)], { timeoutMs: 5000, session: sessionName }); } diff --git a/src/browser/isolation.test.ts b/src/browser/isolation.test.ts new file mode 100644 index 0000000..cf07f5b --- /dev/null +++ b/src/browser/isolation.test.ts @@ -0,0 +1,298 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { createHash } from 'crypto'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + execFileSync: vi.fn(), + findExecutablePath: vi.fn(), +})); + +vi.mock('child_process', async (importOriginal) => ({ + ...(await importOriginal()), + execFileSync: mocks.execFileSync, +})); +vi.mock('../utils/process.js', () => ({ + findExecutablePath: mocks.findExecutablePath, +})); + +import { + assertAgentBrowserRuntime, + getIsolatedAgentBrowserEnvironment, + loadIsolatedAgentBrowserConfig, + parseAgentBrowserVersion, + resolveAgentBrowserRuntime, + writeIsolatedAgentBrowserConfig, +} from './isolation.js'; + +const createdDirectories: string[] = []; +const RUNTIME_SOURCE = Buffer.from([0x7f, 0x45, 0x4c, 0x46, 0x01]); +let runtimeExecutablePath: string; + +beforeEach(() => { + mocks.execFileSync.mockReset(); + mocks.findExecutablePath.mockReset(); + const directory = createDirectory(); + runtimeExecutablePath = path.join(directory, 'agent-browser'); + fs.writeFileSync(runtimeExecutablePath, RUNTIME_SOURCE, { mode: 0o755 }); + mocks.findExecutablePath.mockReturnValue(runtimeExecutablePath); +}); + +afterEach(() => { + for (const directory of createdDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +describe('agent-browser isolation', () => { + it('rejects inherited provider, connection, and persistent-state modes', () => { + for (const key of [ + 'AGENT_BROWSER_PROVIDER', + 'AGENT_BROWSER_CDP', + 'AGENT_BROWSER_AUTO_CONNECT', + 'AGENT_BROWSER_JSON', + 'AGENT_BROWSER_PROFILE', + 'AGENT_BROWSER_PLUGINS', + 'AGENT_BROWSER_STATE', + 'AGENT_BROWSER_SESSION_NAME', + ]) { + expect(() => + loadIsolatedAgentBrowserConfig(undefined, { [key]: 'configured' }), + ).toThrow(key); + } + }); + + it('rejects incompatible modes in an explicit config', () => { + const directory = createDirectory(); + const configPath = path.join(directory, 'agent-browser.json'); + fs.writeFileSync(configPath, JSON.stringify({ profile: './shared-profile' })); + + expect(() => + loadIsolatedAgentBrowserConfig(configPath, {}), + ).toThrow('config key "profile"'); + }); + + it('rejects plugin loading from an explicit config', () => { + const directory = createDirectory(); + const configPath = path.join(directory, 'agent-browser.json'); + fs.writeFileSync(configPath, JSON.stringify({ plugins: ['./plugin.js'] })); + + expect(() => loadIsolatedAgentBrowserConfig(configPath, {})).toThrow( + 'config key "plugins"', + ); + }); + + it('rejects JSON output mode from an explicit config', () => { + const directory = createDirectory(); + const configPath = path.join(directory, 'agent-browser.json'); + fs.writeFileSync(configPath, JSON.stringify({ json: true })); + + expect(() => loadIsolatedAgentBrowserConfig(configPath, {})).toThrow( + 'config key "json"', + ); + }); + + it('copies a safe config into private per-session state', () => { + const directory = createDirectory(); + const sourcePath = path.join(directory, 'source.json'); + const privateDirectory = path.join(directory, 'private', 'agent-browser'); + fs.writeFileSync( + sourcePath, + JSON.stringify({ ignoreHttpsErrors: true, userAgent: 'proofshot-test' }), + ); + + const config = loadIsolatedAgentBrowserConfig(sourcePath, {}); + const configPath = writeIsolatedAgentBrowserConfig(privateDirectory, config); + + expect(JSON.parse(fs.readFileSync(configPath, 'utf-8'))).toEqual(config); + expect(fs.statSync(privateDirectory).mode & 0o777).toBe(0o700); + expect(fs.statSync(configPath).mode & 0o777).toBe(0o600); + }); + + it('strips incompatible modes from every agent-browser subprocess', () => { + const environment = getIsolatedAgentBrowserEnvironment({ + AGENT_BROWSER_JSON: '1', + AGENT_BROWSER_PROVIDER: 'browserbase', + AGENT_BROWSER_PROFILE: './shared', + PATH: '/usr/bin', + }); + + expect(environment).toEqual({ PATH: '/usr/bin' }); + }); + + it('parses the supported semantic version envelope', () => { + expect(parseAgentBrowserVersion('agent-browser 0.34.0')).toEqual([0, 34, 0]); + expect(parseAgentBrowserVersion('0.35.2')).toEqual([0, 35, 2]); + expect(parseAgentBrowserVersion('unknown')).toBeNull(); + }); + + it('resolves one executable and requires exactly agent-browser 0.34.0', () => { + mockVersions('agent-browser 0.33.1\n'); + expect(() => resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toThrow( + 'requires exactly agent-browser 0.34.0', + ); + + mockVersions('agent-browser 0.35.0\n'); + expect(() => resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toThrow( + 'requires exactly agent-browser 0.34.0', + ); + + mockVersions('agent-browser 0.34.0-beta.1\n'); + expect(() => resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toThrow( + 'requires exactly agent-browser 0.34.0', + ); + + mockVersions('agent-browser 0.34.0\n'); + expect(resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toEqual({ + contract: 'direct-native-v1', + executablePath: runtimeExecutablePath, + nativePath: runtimeExecutablePath, + nativeSha256: createHash('sha256').update(RUNTIME_SOURCE).digest('hex'), + sha256: createHash('sha256').update(RUNTIME_SOURCE).digest('hex'), + version: '0.34.0', + }); + expect(mocks.execFileSync).toHaveBeenLastCalledWith( + runtimeExecutablePath, + ['--version'], + expect.objectContaining({ env: { PATH: '/usr/bin' } }), + ); + }); + + it('normalizes a discovered executable to an absolute path before probing it', () => { + const relativeExecutablePath = path.relative( + process.cwd(), + runtimeExecutablePath, + ); + mocks.findExecutablePath.mockReturnValue(relativeExecutablePath); + mockVersions('0.34.0\n'); + + expect(resolveAgentBrowserRuntime({ PATH: './tools' })).toEqual({ + contract: 'direct-native-v1', + executablePath: runtimeExecutablePath, + nativePath: runtimeExecutablePath, + nativeSha256: createHash('sha256').update(RUNTIME_SOURCE).digest('hex'), + sha256: createHash('sha256').update(RUNTIME_SOURCE).digest('hex'), + version: '0.34.0', + }); + expect(mocks.execFileSync).toHaveBeenCalledWith( + runtimeExecutablePath, + ['--version'], + expect.any(Object), + ); + }); + + it('pins the native artifact and managed launcher contract', () => { + const directory = createDirectory(); + const entrypointPath = path.join(directory, 'agent-browser.js'); + const nativePath = path.join(directory, 'agent-browser-linux-x64'); + const entrypointSource = '#!/usr/bin/env node\n'; + const nativeSource = Buffer.concat([ + RUNTIME_SOURCE, + Buffer.from('managed-native'), + ]); + fs.writeFileSync(entrypointPath, entrypointSource); + fs.writeFileSync(nativePath, nativeSource, { mode: 0o755 }); + const entrypointSha256 = createHash('sha256') + .update(entrypointSource) + .digest('hex'); + const nativeSha256 = createHash('sha256') + .update(nativeSource) + .digest('hex'); + mocks.execFileSync.mockImplementation( + (_executablePath: string, args: string[]) => + args[0] === '--managed-preflight' + ? JSON.stringify({ + agentBrowserVersion: '0.34.0', + entrypointPath, + entrypointSha256, + nativePath, + nativeSha256, + nodeVersion: 'v24.19.0', + result: 'ok', + }) + : 'agent-browser 0.34.0\n', + ); + + expect(resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toEqual({ + contract: 'managed-preflight-v1', + entrypointSha256, + executablePath: runtimeExecutablePath, + nativePath, + nativeSha256, + nodeVersion: 'v24.19.0', + sha256: createHash('sha256').update(RUNTIME_SOURCE).digest('hex'), + version: '0.34.0', + }); + }); + + it('fails before version probing when agent-browser is unavailable', () => { + mocks.findExecutablePath.mockReturnValue(null); + + expect(() => resolveAgentBrowserRuntime({ PATH: '/usr/bin' })).toThrow( + 'agent-browser executable was not found', + ); + expect(mocks.execFileSync).not.toHaveBeenCalled(); + }); + + it('rejects same-path executable replacement after a session starts', () => { + mockVersions('agent-browser 0.34.0\n', 'agent-browser 0.34.0\n'); + const runtime = resolveAgentBrowserRuntime({ PATH: '/usr/bin' }); + fs.writeFileSync( + runtimeExecutablePath, + Buffer.concat([RUNTIME_SOURCE, Buffer.from('replaced')]), + ); + + expect(() => + assertAgentBrowserRuntime(runtime, { PATH: '/usr/bin' }), + ).toThrow('executable changed'); + }); + + it('pins the canonical executable when its discovery symlink is retargeted', () => { + const directory = createDirectory(); + const firstTarget = path.join(directory, 'agent-browser-first'); + const secondTarget = path.join(directory, 'agent-browser-second'); + const symlinkPath = path.join(directory, 'agent-browser'); + fs.writeFileSync(firstTarget, RUNTIME_SOURCE, { mode: 0o755 }); + fs.writeFileSync(secondTarget, Buffer.concat([RUNTIME_SOURCE, Buffer.from('second')]), { + mode: 0o755, + }); + fs.symlinkSync(firstTarget, symlinkPath); + mocks.findExecutablePath.mockReturnValue(symlinkPath); + mockVersions('agent-browser 0.34.0\n', 'agent-browser 0.34.0\n'); + const runtime = resolveAgentBrowserRuntime({ PATH: '/usr/bin' }); + + fs.unlinkSync(symlinkPath); + fs.symlinkSync(secondTarget, symlinkPath); + + expect(() => + assertAgentBrowserRuntime(runtime, { PATH: '/usr/bin' }), + ).not.toThrow(); + expect(mocks.execFileSync).toHaveBeenLastCalledWith( + firstTarget, + ['--version'], + expect.any(Object), + ); + }); +}); + +function mockVersions(...versions: string[]): void { + mocks.execFileSync.mockImplementation( + (_executablePath: string, args: string[]) => { + if (args[0] === '--managed-preflight') { + throw new Error('unsupported option'); + } + const version = versions.shift(); + if (!version) { + throw new Error('missing mocked version'); + } + return version; + }, + ); +} + +function createDirectory(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'proofshot-isolation-')); + createdDirectories.push(directory); + return directory; +} diff --git a/src/browser/isolation.ts b/src/browser/isolation.ts new file mode 100644 index 0000000..d4ad972 --- /dev/null +++ b/src/browser/isolation.ts @@ -0,0 +1,404 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { createHash, randomUUID } from 'crypto'; +import { execFileSync } from 'child_process'; +import { findExecutablePath } from '../utils/process.js'; + +const REQUIRED_AGENT_BROWSER_VERSION = '0.34.0'; +const REQUIRED_AGENT_BROWSER_VERSION_OUTPUT = /^(?:agent-browser\s+)?0\.34\.0$/i; +const INCOMPATIBLE_ENVIRONMENT_KEYS = [ + 'AGENT_BROWSER_ARGS', + 'AGENT_BROWSER_AUTO_CONNECT', + 'AGENT_BROWSER_CDP', + 'AGENT_BROWSER_CONFIG', + 'AGENT_BROWSER_ENGINE', + 'AGENT_BROWSER_EXTENSIONS', + 'AGENT_BROWSER_INIT_SCRIPTS', + 'AGENT_BROWSER_JSON', + 'AGENT_BROWSER_PLUGINS', + 'AGENT_BROWSER_PROFILE', + 'AGENT_BROWSER_PROVIDER', + 'AGENT_BROWSER_PROXY', + 'AGENT_BROWSER_PROXY_BYPASS', + 'AGENT_BROWSER_SESSION', + 'AGENT_BROWSER_SESSION_NAME', + 'AGENT_BROWSER_STATE', +] as const; +const INCOMPATIBLE_CONFIG_KEYS = new Set([ + 'allowfileaccess', + 'args', + 'autoconnect', + 'cdp', + 'enable', + 'engine', + 'extensions', + 'headers', + 'headed', + 'initscript', + 'initscripts', + 'json', + 'profile', + 'plugins', + 'provider', + 'proxy', + 'proxybypass', + 'session', + 'sessionname', + 'state', +]); + +export function loadIsolatedAgentBrowserConfig( + sourcePath: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): Record { + assertIsolatedAgentBrowserEnvironment(env); + if (!sourcePath) { + return {}; + } + const value = JSON.parse(fs.readFileSync(sourcePath, 'utf-8')) as unknown; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`agent-browser config must contain a JSON object: ${sourcePath}`); + } + const config = value as Record; + const incompatibleKey = Object.keys(config).find((key) => + INCOMPATIBLE_CONFIG_KEYS.has(normalizeConfigKey(key)), + ); + if (incompatibleKey) { + throw new Error( + `agent-browser config key "${incompatibleKey}" is incompatible with an isolated local ProofShot browser.`, + ); + } + return config; +} + +export function writeIsolatedAgentBrowserConfig( + privateDirectory: string, + config: Record, +): string { + fs.mkdirSync(privateDirectory, { recursive: true, mode: 0o700 }); + fs.chmodSync(privateDirectory, 0o700); + const configPath = path.join(privateDirectory, 'config.json'); + const temporaryPath = `${configPath}.${process.pid}.${randomUUID()}.tmp`; + try { + fs.writeFileSync(temporaryPath, JSON.stringify(config, null, 2) + '\n', { + mode: 0o600, + }); + fs.renameSync(temporaryPath, configPath); + fs.chmodSync(configPath, 0o600); + } finally { + if (fs.existsSync(temporaryPath)) { + fs.unlinkSync(temporaryPath); + } + } + return configPath; +} + +export type AgentBrowserRuntime = { + contract: AgentBrowserRuntimeContract; + executablePath: string; + entrypointSha256?: string; + nativePath: string; + nativeSha256: string; + nodeVersion?: string; + sha256: string; + version: string; +}; + +export type AgentBrowserRuntimeContract = + | 'direct-native-v1' + | 'managed-preflight-v1' + | 'npm-wrapper-v1'; + +export type AgentBrowserRuntimeReceipt = { + contract: AgentBrowserRuntimeContract; + entrypointSha256?: string; + launcherSha256: string; + nativeSha256: string; + nodeVersion?: string; + version: string; +}; + +export function resolveAgentBrowserRuntime( + env: NodeJS.ProcessEnv = process.env, +): AgentBrowserRuntime { + const discoveredExecutablePath = findExecutablePath('agent-browser'); + if (!discoveredExecutablePath) { + throw new Error('agent-browser executable was not found on PATH.'); + } + return resolveAgentBrowserRuntimeAtPath(discoveredExecutablePath, env); +} + +export function assertAgentBrowserRuntime( + expected: AgentBrowserRuntime, + env: NodeJS.ProcessEnv = process.env, +): void { + const actual = resolveAgentBrowserRuntimeAtPath(expected.executablePath, env); + if ( + actual.executablePath !== expected.executablePath || + actual.contract !== expected.contract || + actual.nativePath !== expected.nativePath || + actual.nativeSha256 !== expected.nativeSha256 || + actual.entrypointSha256 !== expected.entrypointSha256 || + actual.nodeVersion !== expected.nodeVersion || + actual.version !== expected.version || + actual.sha256 !== expected.sha256 + ) { + throw new Error( + 'The pinned agent-browser executable changed after this ProofShot session started.', + ); + } +} + +export function resolveAgentBrowserRuntimeAtPath( + discoveredExecutablePath: string, + env: NodeJS.ProcessEnv = process.env, +): AgentBrowserRuntime { + let executablePath: string; + try { + executablePath = fs.realpathSync(path.resolve(discoveredExecutablePath)); + } catch { + throw new Error('The pinned agent-browser executable is no longer available.'); + } + + const environment = getIsolatedAgentBrowserEnvironment(env); + const artifacts = resolveRuntimeArtifacts(executablePath, environment); + const output = execFileSync(executablePath, ['--version'], { + encoding: 'utf-8', + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }).trim(); + const version = parseAgentBrowserVersion(output); + if ( + !version || + version.join('.') !== REQUIRED_AGENT_BROWSER_VERSION || + !REQUIRED_AGENT_BROWSER_VERSION_OUTPUT.test(output) + ) { + throw new Error( + `ProofShot requires exactly agent-browser ${REQUIRED_AGENT_BROWSER_VERSION}; received ${output || 'no version output'}.`, + ); + } + return { + contract: artifacts.contract, + executablePath, + ...(artifacts.entrypointSha256 + ? { entrypointSha256: artifacts.entrypointSha256 } + : {}), + nativePath: artifacts.nativePath, + nativeSha256: artifacts.nativeSha256, + ...(artifacts.nodeVersion ? { nodeVersion: artifacts.nodeVersion } : {}), + sha256: hashFile(executablePath), + version: version.join('.'), + }; +} + +export function toAgentBrowserRuntimeReceipt( + runtime: AgentBrowserRuntime, +): AgentBrowserRuntimeReceipt { + return { + contract: runtime.contract, + ...(runtime.entrypointSha256 + ? { entrypointSha256: runtime.entrypointSha256 } + : {}), + launcherSha256: runtime.sha256, + nativeSha256: runtime.nativeSha256, + ...(runtime.nodeVersion ? { nodeVersion: runtime.nodeVersion } : {}), + version: runtime.version, + }; +} + +export function getIsolatedAgentBrowserEnvironment( + env: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + const isolatedEnvironment = { ...env }; + for (const key of INCOMPATIBLE_ENVIRONMENT_KEYS) { + delete isolatedEnvironment[key]; + } + return isolatedEnvironment; +} + +export function parseAgentBrowserVersion( + output: string, +): readonly [number, number, number] | null { + const match = output.match(/(?:agent-browser\s+)?(\d+)\.(\d+)\.(\d+)/i); + return match + ? [Number(match[1]), Number(match[2]), Number(match[3])] + : null; +} + +function assertIsolatedAgentBrowserEnvironment(env: NodeJS.ProcessEnv): void { + const incompatibleKey = INCOMPATIBLE_ENVIRONMENT_KEYS.find( + (key) => env[key] !== undefined && env[key] !== '', + ); + if (incompatibleKey) { + throw new Error( + `${incompatibleKey} is incompatible with an isolated local ProofShot browser.`, + ); + } +} + +function normalizeConfigKey(key: string): string { + return key.toLowerCase().replace(/[-_]/g, ''); +} + +type RuntimeArtifacts = Pick< + AgentBrowserRuntime, + | 'contract' + | 'entrypointSha256' + | 'nativePath' + | 'nativeSha256' + | 'nodeVersion' +>; + +type ManagedPreflightReport = { + agentBrowserVersion: string; + entrypointPath?: string; + entrypointSha256?: string; + nativePath: string; + nativeSha256: string; + nodeVersion?: string; + result: string; +}; + +function resolveRuntimeArtifacts( + executablePath: string, + env: NodeJS.ProcessEnv, +): RuntimeArtifacts { + const managed = readManagedPreflight(executablePath, env); + if (managed) { + return managed; + } + if (isNativeExecutable(executablePath)) { + return { + contract: 'direct-native-v1', + nativePath: executablePath, + nativeSha256: hashFile(executablePath), + }; + } + if (path.basename(executablePath) === 'agent-browser.js') { + const nativePath = path.join( + path.dirname(executablePath), + resolveNpmNativeBinaryName(), + ); + const canonicalNativePath = fs.realpathSync(nativePath); + return { + contract: 'npm-wrapper-v1', + entrypointSha256: hashFile(executablePath), + nativePath: canonicalNativePath, + nativeSha256: hashFile(canonicalNativePath), + nodeVersion: process.version, + }; + } + throw new Error( + 'agent-browser launcher does not expose a verifiable native runtime contract.', + ); +} + +function readManagedPreflight( + executablePath: string, + env: NodeJS.ProcessEnv, +): RuntimeArtifacts | null { + let output: string; + try { + output = execFileSync(executablePath, ['--managed-preflight'], { + encoding: 'utf-8', + env, + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }).trim(); + } catch { + return null; + } + + let report: ManagedPreflightReport; + try { + report = JSON.parse(output) as ManagedPreflightReport; + } catch { + throw new Error('agent-browser managed preflight returned invalid JSON.'); + } + if ( + report.result !== 'ok' || + report.agentBrowserVersion !== REQUIRED_AGENT_BROWSER_VERSION || + typeof report.nativePath !== 'string' || + typeof report.nativeSha256 !== 'string' + ) { + throw new Error('agent-browser managed preflight contract is invalid.'); + } + const nativePath = fs.realpathSync(report.nativePath); + const nativeSha256 = hashFile(nativePath); + if (nativeSha256 !== report.nativeSha256) { + throw new Error('agent-browser native digest does not match its launcher contract.'); + } + let entrypointSha256: string | undefined; + if (report.entrypointPath || report.entrypointSha256) { + if ( + typeof report.entrypointPath !== 'string' || + typeof report.entrypointSha256 !== 'string' + ) { + throw new Error('agent-browser entrypoint contract is incomplete.'); + } + entrypointSha256 = hashFile(fs.realpathSync(report.entrypointPath)); + if (entrypointSha256 !== report.entrypointSha256) { + throw new Error( + 'agent-browser entrypoint digest does not match its launcher contract.', + ); + } + } + return { + contract: 'managed-preflight-v1', + entrypointSha256, + nativePath, + nativeSha256, + nodeVersion: report.nodeVersion, + }; +} + +function resolveNpmNativeBinaryName(): string { + const architecture = + process.arch === 'x64' + ? 'x64' + : process.arch === 'arm64' + ? 'arm64' + : null; + if (!architecture) { + throw new Error(`Unsupported agent-browser architecture: ${process.arch}.`); + } + if (process.platform === 'darwin') { + return `agent-browser-darwin-${architecture}`; + } + if (process.platform === 'win32') { + return `agent-browser-win32-${architecture}.exe`; + } + if (process.platform === 'linux') { + const muslLoader = `/lib/ld-musl-${architecture === 'x64' ? 'x86_64' : 'aarch64'}.so.1`; + const platform = fs.existsSync(muslLoader) ? 'linux-musl' : 'linux'; + return `agent-browser-${platform}-${architecture}`; + } + throw new Error(`Unsupported agent-browser platform: ${process.platform}.`); +} + +function isNativeExecutable(filePath: string): boolean { + const header = fs.readFileSync(filePath).subarray(0, 4); + if (header.length < 2) { + return false; + } + if (header[0] === 0x7f && header.subarray(1, 4).toString('ascii') === 'ELF') { + return true; + } + if (header[0] === 0x4d && header[1] === 0x5a) { + return true; + } + const magic = header.readUInt32BE(0); + return [ + 0xcafebabe, + 0xbebafeca, + 0xfeedface, + 0xcefaedfe, + 0xfeedfacf, + 0xcffaedfe, + ].includes(magic); +} + +function hashFile(filePath: string): string { + return createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} diff --git a/src/browser/navigate.ts b/src/browser/navigate.ts index 5528843..0687cfa 100644 --- a/src/browser/navigate.ts +++ b/src/browser/navigate.ts @@ -1,15 +1,18 @@ -import { ab, quoteShellArgument } from '../utils/exec.js'; +import { ab } from '../utils/exec.js'; /** * Navigate to a URL and wait for the page to load. */ export function navigateTo(url: string, sessionName?: string): void { - ab(`open ${quoteShellArgument(url)}`, { + ab(['open', url], { timeoutMs: 60000, session: sessionName, }); try { - ab('wait --load networkidle', { timeoutMs: 30000, session: sessionName }); + ab(['wait', '--load', 'networkidle'], { + timeoutMs: 30000, + session: sessionName, + }); } catch { // networkidle may timeout on some pages — that's okay, // we still have the page in whatever state it loaded to @@ -22,7 +25,7 @@ export function navigateTo(url: string, sessionName?: string): void { */ export function getSnapshot(sessionName?: string): string { try { - return ab('snapshot -i', { timeoutMs: 15000, session: sessionName }); + return ab(['snapshot', '-i'], { timeoutMs: 15000, session: sessionName }); } catch { return ''; } diff --git a/src/browser/provenance.test.ts b/src/browser/provenance.test.ts new file mode 100644 index 0000000..8d9fd9a --- /dev/null +++ b/src/browser/provenance.test.ts @@ -0,0 +1,493 @@ +import { describe, expect, it } from 'vitest'; +import { + buildSanitizedCommandIntent, + classifyInteraction, + sanitizeDiagnosticMessage, + sanitizePageUrl, +} from './provenance.js'; +import { + CONVENTIONAL_FUSED_SECRET_FIELDS, + PUBLIC_FUSED_FIELDS, +} from './redaction-policy.test-data.js'; + +describe('browser interaction provenance', () => { + it('classifies user-equivalent, hybrid, diagnostic, and observation commands honestly', () => { + expect(classifyInteraction(['click', '@e1'])).toBe('pointer-keyboard'); + expect(classifyInteraction(['fill', '@e2', 'value'])).toBe('hybrid'); + expect(classifyInteraction(['eval', 'document.body.click()'])).toBe( + 'synthetic-dom', + ); + expect(classifyInteraction(['snapshot', '-i'])).toBe('observation'); + expect(classifyInteraction(['open', 'https://example.com'])).toBe('setup'); + }); + + it('classifies nested and keyboard commands by their effective action', () => { + expect( + classifyInteraction(['find', 'role', 'button', 'click', '--name', 'Submit']), + ).toBe('pointer-keyboard'); + expect( + classifyInteraction(['find', 'label', 'Password', 'fill', 'private']), + ).toBe('hybrid'); + expect(classifyInteraction(['find', 'text', 'Welcome', 'text'])).toBe( + 'observation', + ); + expect(classifyInteraction(['keyboard', 'type', 'private'])).toBe( + 'pointer-keyboard', + ); + expect( + classifyInteraction(['find', 'nth', '2', 'a', 'click']), + ).toBe('pointer-keyboard'); + expect(classifyInteraction(['find', 'text', 'Submit'])).toBe( + 'pointer-keyboard', + ); + expect(classifyInteraction(['find', 'nth', '2', '.card'])).toBe( + 'pointer-keyboard', + ); + expect(classifyInteraction(['keyboard', 'inserttext', 'private'])).toBe( + 'hybrid', + ); + expect(classifyInteraction(['keydown', 'Shift'])).toBe('pointer-keyboard'); + expect(classifyInteraction(['storage', 'get', 'theme'])).toBe('observation'); + expect(classifyInteraction(['storage', 'set', 'theme', 'dark'])).toBe('setup'); + expect(classifyInteraction(['network', 'requests'])).toBe('observation'); + expect(classifyInteraction(['network', 'har', 'start'])).toBe('setup'); + }); + + it('rejects synthetic DOM mutation hidden in nested batch commands', () => { + const syntheticCommand = JSON.stringify([ + 'eval', + 'document.querySelector("button").click()', + ]); + const nestedBatch = JSON.stringify([ + 'batch', + syntheticCommand, + ]); + + expect( + classifyInteraction([ + 'batch', + '--bail', + '["open","https://example.com"]', + syntheticCommand, + ]), + ).toBe('synthetic-dom'); + expect( + classifyInteraction(['batch', nestedBatch]), + ).toBe('synthetic-dom'); + expect( + classifyInteraction([ + 'batch', + 'open https://example.com', + 'eval "document.body.click()"', + ]), + ).toBe('synthetic-dom'); + expect(classifyInteraction(['batch', '["snapshot","-i"]'])).toBe( + 'observation', + ); + }); + + it('redacts typed values, credentials, scripts, and sensitive URL parameters', () => { + expect( + buildSanitizedCommandIntent(['fill', '@e2', 'private-value']).summary, + ).toBe('fill @e2 [REDACTED]'); + expect( + buildSanitizedCommandIntent([ + 'set', + 'credentials', + 'private-user', + 'private-password', + ]).summary, + ).toBe('set credentials [REDACTED]'); + expect( + buildSanitizedCommandIntent([ + 'eval', + "document.querySelector('#secret').value", + ]).summary, + ).toBe('eval [REDACTED_SCRIPT]'); + expect( + buildSanitizedCommandIntent([ + 'find', + 'nth', + '2', + '.card', + 'fill', + 'private-nth-value', + ]).summary, + ).toBe('find nth 2 .card fill [REDACTED]'); + expect( + buildSanitizedCommandIntent([ + 'open', + 'https://api.example.com', + '--headers', + '{"Authorization":"Bearer private-header-token"}', + ]).summary, + ).toBe('open https://api.example.com/ --headers [REDACTED]'); + expect( + buildSanitizedCommandIntent([ + 'open', + 'https://api.example.com', + '--headers={"Authorization":"Bearer private-inline-token"}', + ]).summary, + ).toBe('open https://api.example.com/ --headers=[REDACTED]'); + expect( + buildSanitizedCommandIntent([ + 'keyboard', + 'inserttext', + 'private-keyboard-value', + ]).summary, + ).toBe('keyboard inserttext [REDACTED]'); + + const nestedSecrets = [ + buildSanitizedCommandIntent([ + 'find', + 'label', + 'Password', + 'fill', + 'private-find-value', + ]).summary, + buildSanitizedCommandIntent([ + 'keyboard', + 'type', + 'private-keyboard-value', + ]).summary, + buildSanitizedCommandIntent([ + 'storage', + 'local', + 'set', + 'password', + 'private-storage-value', + ]).summary, + buildSanitizedCommandIntent([ + 'batch', + '[{"command":"fill","value":"private-batch-value"}]', + ]).summary, + buildSanitizedCommandIntent([ + 'future-command', + 'private-unknown-value', + ]).summary, + ].join('\n'); + expect(nestedSecrets).not.toContain('private-find-value'); + expect(nestedSecrets).not.toContain('private-keyboard-value'); + expect(nestedSecrets).not.toContain('private-storage-value'); + expect(nestedSecrets).not.toContain('private-batch-value'); + expect(nestedSecrets).not.toContain('private-unknown-value'); + + const url = sanitizePageUrl( + 'https://user:pass@example.com/callback?token=private&tab=profile', + ); + expect(url).toBe( + 'https://example.com/callback?token=%5BREDACTED%5D&tab=profile', + ); + }); + + it.each([ + ['--auth', 'private-auth'], + ['--cookies', 'private-cookie'], + ['--session', 'private-session'], + ['--signature', 'private-signature'], + ])('redacts shared secret flag %s', (flag, secret) => { + const summary = buildSanitizedCommandIntent([ + 'open', + 'https://example.test', + flag, + secret, + ]).summary; + + expect(summary).not.toContain(secret); + expect(summary).toContain(`${flag} [REDACTED]`); + }); + + it('still sanitizes secret-shaped URL arguments as URLs', () => { + expect( + buildSanitizedCommandIntent([ + 'open', + 'https://example.test/session/private-session', + ]).summary, + ).toBe('open https://example.test/session/%5BREDACTED%5D'); + }); + + it('sanitizes fragments, opaque URLs, and malformed URLs without recursion', () => { + expect( + sanitizePageUrl( + 'https://example.com/callback#access_token=private-fragment-token', + ), + ).toBe('https://example.com/callback'); + expect(sanitizePageUrl('data:text/html,private-content')).toBe( + '[REDACTED_URL:data]', + ); + expect(sanitizePageUrl('http://[')).toBe('[REDACTED_URL:http]'); + expect(sanitizeDiagnosticMessage('request failed at http://[')).toBe( + 'request failed at [REDACTED_URL:http]', + ); + }); + + it.each([ + [ + 'request failed at HTTPS://user:pass@example.test/auth/verify-email/private-token?code=private-code&safe=yes', + 'request failed at https://example.test/auth/%5BREDACTED%5D/%5BREDACTED%5D?code=%5BREDACTED%5D&safe=yes', + ], + [ + 'request failed at hTtP://user:pass@example.test/authorization/unknown/private-token?key=private-key&safe=yes', + 'request failed at http://example.test/authorization/%5BREDACTED%5D/%5BREDACTED%5D?key=%5BREDACTED%5D&safe=yes', + ], + ])('sanitizes mixed-case HTTP URLs in diagnostics', (message, expected) => { + expect(sanitizeDiagnosticMessage(message)).toBe(expected); + }); + + it('redacts secret-shaped diagnostics before persistence', () => { + const message = sanitizeDiagnosticMessage( + [ + 'Authorization: Bearer private-bearer-token', + 'authorization=Basic cHJpdmF0ZTpzZWNyZXQ=', + 'token=another-private-value', + 'https://downloads.example.test/accounts/token/private-path/report?X-Amz-Signature=private-signature&safe=yes', + ].join('\n'), + ); + expect(message).not.toContain('private-bearer-token'); + expect(message).not.toContain('cHJpdmF0ZTpzZWNyZXQ='); + expect(message).not.toContain('another-private-value'); + expect(message).not.toContain('private-signature'); + expect(message).not.toContain('private-path'); + expect(message).toContain('safe=yes'); + expect(message).toContain('[REDACTED]'); + }); + + it.each([ + ['cookies', 'private-cookie'], + ['auth', 'private-auth'], + ['session', 'private-session'], + ['signature', 'private-signature'], + ['tokens', 'private-tokens'], + ['accessTokens', 'private-access-tokens'], + ['sessions', 'private-sessions'], + ['signatures', 'private-signatures'], + ['passwords', 'private-passwords'], + ['secrets', 'private-secrets'], + ])('redacts shared diagnostic field %s', (field, secret) => { + const message = sanitizeDiagnosticMessage(`${field}=${secret}`); + + expect(message).toBe(`${field}=[REDACTED]`); + }); + + it.each([...CONVENTIONAL_FUSED_SECRET_FIELDS, 'AwsAccessKeyId'])( + 'redacts fused secret field %s in diagnostics and URLs', + (field) => { + const secret = `private-${field.toLowerCase()}`; + + expect(sanitizeDiagnosticMessage(`${field}=${secret}`)).toBe( + `${field}=[REDACTED]`, + ); + expect( + sanitizePageUrl( + `https://example.test/callback?${field}=${secret}&safe=yes`, + ), + ).toBe( + `https://example.test/callback?${field}=%5BREDACTED%5D&safe=yes`, + ); + }, + ); + + it.each(PUBLIC_FUSED_FIELDS)( + 'keeps unrelated fused field %s visible in diagnostics and URLs', + (field) => { + expect(sanitizeDiagnosticMessage(`${field}=visible`)).toBe( + `${field}=visible`, + ); + expect( + sanitizePageUrl(`https://example.test/callback?${field}=visible`), + ).toBe(`https://example.test/callback?${field}=visible`); + }, + ); + + it.each(['code', 'codes', 'key', 'keys'])( + 'treats ambiguous field %s as URL-only', + (field) => { + expect(sanitizeDiagnosticMessage(`${field}=visible`)).toBe( + `${field}=visible`, + ); + expect( + sanitizePageUrl( + `https://example.test/callback?${field}=private&safe=yes`, + ), + ).toBe( + `https://example.test/callback?${field}=%5BREDACTED%5D&safe=yes`, + ); + }, + ); + + it('redacts URL-valued diagnostic secrets as one value', () => { + expect( + sanitizeDiagnosticMessage( + 'token=https://example.test/callback?safe=visible', + ), + ).toBe('token=[REDACTED]'); + }); + + it('redacts complete unquoted authorization values', () => { + const message = sanitizeDiagnosticMessage( + 'authorization=Token private-secret', + ); + + expect(message).not.toContain('private-secret'); + expect(message).toBe('authorization=[REDACTED]'); + }); + + it('fails closed for compound, free-form, and multiline sensitive fields', () => { + const message = [ + 'HTTP_AUTHORIZATION=Token private-env-auth', + 'Cookie: theme=light; session=private-cookie', + 'body=private payload-contents', + 'password:\n private-next-line', + 'token="private-first-line\nprivate-second-line"', + ] + .map((diagnostic) => sanitizeDiagnosticMessage(diagnostic)) + .join('\n'); + + expect(message).not.toContain('private-env-auth'); + expect(message).not.toContain('private-cookie'); + expect(message).not.toContain('payload-contents'); + expect(message).not.toContain('private-next-line'); + expect(message).not.toContain('private-first-line'); + expect(message).not.toContain('private-second-line'); + }); + + it('redacts indented continuations without swallowing later diagnostics', () => { + const body = sanitizeDiagnosticMessage( + 'body=private-first\n private-second\nsafe=visible', + ); + const authorization = sanitizeDiagnosticMessage( + 'Authorization: Digest private-first\n\tresponse=private-second\nsafe=visible', + ); + + expect(body).not.toContain('private-first'); + expect(body).not.toContain('private-second'); + expect(body).toContain('safe=visible'); + expect(authorization).not.toContain('private-first'); + expect(authorization).not.toContain('private-second'); + expect(authorization).toContain('safe=visible'); + }); + + it('redacts continuations separated by blank lines', () => { + for (const newline of ['\n', '\r\n']) { + const message = sanitizeDiagnosticMessage( + `body=private-first${newline}${newline} private-second${newline}safe=visible`, + ); + + expect(message).not.toContain('private-first'); + expect(message).not.toContain('private-second'); + expect(message).toContain('safe=visible'); + } + }); + + it('redacts compound secret keys and complete quoted values', () => { + const message = sanitizeDiagnosticMessage( + [ + 'client_secret=private-client-secret', + 'access_token: private-access-token', + 'password="correct horse battery staple"', + "refresh_token='private refresh token'", + ].join('\n'), + ); + + expect(message).not.toContain('private-client-secret'); + expect(message).not.toContain('private-access-token'); + expect(message).not.toContain('correct horse battery staple'); + expect(message).not.toContain('private refresh token'); + expect(message).not.toContain('horse battery staple'); + }); + + it('redacts structured and escaped diagnostic secret values', () => { + const message = sanitizeDiagnosticMessage( + [ + '{"clientSecret":"private-client","safe":"visible"}', + '{"auth":{"accessToken":"private-access"},"status":"healthy"}', + '{"token":["private-first","private-second"],"safeList":["one","two"]}', + String.raw`password="private-prefix \"quoted\" private-suffix"`, + ].join('\n'), + ); + + expect(message).not.toContain('private-client'); + expect(message).not.toContain('private-access'); + expect(message).not.toContain('private-first'); + expect(message).not.toContain('private-second'); + expect(message).not.toContain('private-prefix'); + expect(message).not.toContain('private-suffix'); + expect(message).toContain('"safe":"visible"'); + expect(message).toContain('"status":"healthy"'); + expect(message).toContain('"safeList":["one","two"]'); + }); + + it('redacts secret keys inside escaped JSON diagnostics', () => { + const message = sanitizeDiagnosticMessage( + String.raw`message="{\"accessToken\":\"private-access\",\"safe\":\"visible\"}"`, + ); + + expect(message).not.toContain('private-access'); + expect(message).toContain(String.raw`\"safe\":\"visible\"`); + }); + + it('redacts signed URL keys and sensitive path values', () => { + expect( + sanitizePageUrl( + 'https://example.test/api/credentials/private-credential/file?sig=private-sig&signature=private-signature&view=full', + ), + ).toBe( + 'https://example.test/api/credentials/%5BREDACTED%5D/file?sig=%5BREDACTED%5D&signature=%5BREDACTED%5D&view=full', + ); + }); + + it('redacts bearer tokens from authentication route paths', () => { + expect( + sanitizePageUrl( + 'https://example.test/reset-password/private-reset-token', + ), + ).toBe('https://example.test/reset-password/%5BREDACTED%5D'); + expect( + sanitizePageUrl('https://example.test/invite/private-invite-token'), + ).toBe('https://example.test/invite/%5BREDACTED%5D'); + expect( + sanitizePageUrl('https://example.test/magic-link/private-magic-token'), + ).toBe('https://example.test/magic-link/%5BREDACTED%5D'); + expect( + sanitizePageUrl( + 'https://example.test/invite/accept/private-invite-token', + ), + ).toBe( + 'https://example.test/invite/%5BREDACTED%5D/%5BREDACTED%5D', + ); + expect( + sanitizePageUrl( + 'https://example.test/reset-password/confirm/private-reset-token', + ), + ).toBe( + 'https://example.test/reset-password/%5BREDACTED%5D/%5BREDACTED%5D', + ); + expect( + sanitizePageUrl( + 'https://example.test/auth/invite/accept/private-token', + ), + ).toBe( + 'https://example.test/auth/%5BREDACTED%5D/%5BREDACTED%5D/%5BREDACTED%5D', + ); + expect( + sanitizePageUrl( + 'https://example.test/api/auth/reset-password/confirm/private-token', + ), + ).toBe( + 'https://example.test/api/auth/%5BREDACTED%5D/%5BREDACTED%5D/%5BREDACTED%5D', + ); + }); + + it.each([ + [ + 'https://example.test/auth/verify-email/private-token', + 'https://example.test/auth/%5BREDACTED%5D/%5BREDACTED%5D', + ], + [ + 'https://example.test/authorization/unknown/private-token', + 'https://example.test/authorization/%5BREDACTED%5D/%5BREDACTED%5D', + ], + ])('redacts every path segment after a generic auth route', (url, expected) => { + expect(sanitizePageUrl(url)).toBe(expected); + }); +}); diff --git a/src/browser/provenance.ts b/src/browser/provenance.ts new file mode 100644 index 0000000..46af405 --- /dev/null +++ b/src/browser/provenance.ts @@ -0,0 +1,442 @@ +import { redactDiagnosticAssignments } from './diagnostic-redaction.js'; +import { + isAuthenticationPathSegment, + isSecretBearingCommandArgument, + isSensitiveUrlField, +} from './redaction-policy.js'; + +const REDACTED = '[REDACTED]'; +const REDACTED_ARGUMENTS = '[REDACTED_ARGUMENTS]'; +const REDACTED_BATCH = '[REDACTED_BATCH]'; +const MAX_BATCH_DEPTH = 8; +const AUTH_FLOW_PATH_KEY = /^(?:invite|invitation|magic[-_]?link|password[-_]?reset|reset[-_]?password)$/i; +const URL_SCHEME = /^([a-z][a-z0-9+.-]*):/i; +const HTTP_URL = /https?:\/\/[^\s"'<>]+/gi; + +const HYBRID_ACTIONS = new Set(['check', 'fill', 'select', 'type', 'uncheck']); +const OBSERVATION_ACTIONS = new Set([ + 'assert-visible', + 'console', + 'errors', + 'get', + 'is', + 'read', + 'screenshot', + 'snapshot', + 'text', +]); +const POINTER_KEYBOARD_ACTIONS = new Set([ + 'click', + 'dblclick', + 'drag', + 'hover', + 'key', + 'keydown', + 'keyup', + 'mouse', + 'press', +]); +const SETUP_ACTIONS = new Set([ + 'auth', + 'close', + 'navigate', + 'open', + 'record', + 'set', + 'tab', + 'upload', + 'wait', +]); + +export type InteractionCategory = + | 'observation' + | 'pointer-keyboard' + | 'hybrid' + | 'setup' + | 'synthetic-dom' + | 'unknown'; + +export type SanitizedCommandIntent = { + command: string; + summary: string; +}; + +export function buildSanitizedCommandIntent( + args: string[], +): SanitizedCommandIntent { + const command = args[0]?.toLowerCase() || 'unknown'; + const sanitizedArgs = sanitizeArguments(command, args); + return { + command, + summary: sanitizedArgs.join(' '), + }; +} + +export function classifyInteraction(args: string[]): InteractionCategory { + return classifyInteractionAtDepth(args, 0); +} + +function classifyInteractionAtDepth( + args: string[], + batchDepth: number, +): InteractionCategory { + const command = args[0]?.toLowerCase(); + if (!command) { + return 'unknown'; + } + if (command === 'batch') { + return classifyBatchInteraction(args, batchDepth); + } + if (command === 'find') { + const nestedAction = args[findActionIndex(args)]?.toLowerCase(); + return nestedAction + ? classifyInteractionAtDepth([nestedAction], batchDepth) + : 'pointer-keyboard'; + } + if (command === 'keyboard') { + const keyboardAction = args[1]?.toLowerCase(); + if (keyboardAction === 'inserttext') { + return 'hybrid'; + } + return ['press', 'type'].includes(keyboardAction) + ? 'pointer-keyboard' + : 'unknown'; + } + if (command === 'cookies' || command === 'storage') { + return !args[1] || ['get', 'list'].includes(args[1].toLowerCase()) + ? 'observation' + : 'setup'; + } + if (command === 'network') { + return !args[1] || ['get', 'requests'].includes(args[1].toLowerCase()) + ? 'observation' + : 'setup'; + } + if (command === 'console') { + return args[1]?.toLowerCase() === 'clear' ? 'setup' : 'observation'; + } + if (OBSERVATION_ACTIONS.has(command)) { + return 'observation'; + } + if (POINTER_KEYBOARD_ACTIONS.has(command)) { + return 'pointer-keyboard'; + } + if (HYBRID_ACTIONS.has(command)) { + return 'hybrid'; + } + if (command === 'eval') { + return 'synthetic-dom'; + } + if (SETUP_ACTIONS.has(command)) { + return 'setup'; + } + return 'unknown'; +} + +function classifyBatchInteraction( + args: string[], + batchDepth: number, +): InteractionCategory { + if (batchDepth >= MAX_BATCH_DEPTH) { + return 'synthetic-dom'; + } + + const nestedCommands = parseAgentBrowserBatchCommands(args); + if (nestedCommands.length === 0) { + return 'unknown'; + } + + const categories = nestedCommands.map((nestedArgs) => + classifyInteractionAtDepth(nestedArgs, batchDepth + 1), + ); + if (categories.includes('synthetic-dom')) { + return 'synthetic-dom'; + } + + const uniqueCategories = new Set(categories); + return uniqueCategories.size === 1 ? categories[0] : 'unknown'; +} + +export function parseAgentBrowserBatchCommands(args: string[]): string[][] { + return args + .slice(1) + .filter((argument) => !['--bail', '--json'].includes(argument)) + .flatMap(parseBatchArgument); +} + +function parseBatchArgument(value: string): string[][] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + const command = tokenizeCommand(value); + return command.length > 0 ? [command] : []; + } + + if (typeof parsed === 'string') { + const command = tokenizeCommand(parsed); + return command.length > 0 ? [command] : []; + } + if (!Array.isArray(parsed)) { + return []; + } + if (parsed.every((argument): argument is string => typeof argument === 'string')) { + return [parsed]; + } + return parsed.filter( + (command): command is string[] => + Array.isArray(command) && command.every((argument) => typeof argument === 'string'), + ); +} + +function tokenizeCommand(value: string): string[] { + const args: string[] = []; + let current = ''; + let quote: 'single' | 'double' | null = null; + let escaped = false; + + for (const character of value) { + if (escaped) { + current += character; + escaped = false; + continue; + } + if (character === '\\' && quote !== 'single') { + escaped = true; + continue; + } + if (character === "'" && quote !== 'double') { + quote = quote === 'single' ? null : 'single'; + continue; + } + if (character === '"' && quote !== 'single') { + quote = quote === 'double' ? null : 'double'; + continue; + } + if (/\s/.test(character) && quote === null) { + if (current.length > 0) { + args.push(current); + current = ''; + } + continue; + } + current += character; + } + + if (escaped) { + current += '\\'; + } + if (current.length > 0) { + args.push(current); + } + return args; +} + +export function sanitizePageUrl(value: string | undefined): string | undefined { + if (!value) { + return value; + } + return sanitizeUrlValue(value); +} + +function sanitizeUrlValue(value: string): string { + const scheme = value.match(URL_SCHEME)?.[1]?.toLowerCase(); + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol)) { + return url.protocol === 'about:' && url.pathname === 'blank' + ? 'about:blank' + : `[REDACTED_URL:${url.protocol.slice(0, -1)}]`; + } + url.username = ''; + url.password = ''; + url.pathname = sanitizeUrlPath(url.pathname); + for (const key of url.searchParams.keys()) { + if (isSensitiveUrlField(key)) { + url.searchParams.set(key, REDACTED); + } + } + url.hash = ''; + return url.toString(); + } catch { + return scheme ? `[REDACTED_URL:${scheme}]` : '[REDACTED_URL]'; + } +} + +export function sanitizeDiagnosticMessage(value: string | undefined): string | undefined { + if (!value) { + return value; + } + const sanitizedTokens = value + .replace(/\b(?:Basic|Bearer)\s+[^\s"',;]+/gi, REDACTED) + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, REDACTED); + return redactDiagnosticAssignmentsOutsideUrls(sanitizedTokens); +} + +function redactDiagnosticAssignmentsOutsideUrls(value: string): string { + const sanitizedUrls: string[] = []; + let markerPrefix = '\0PROOFSHOT_SANITIZED_URL_'; + while (value.includes(markerPrefix)) { + markerPrefix += '_'; + } + const masked = value.replace(HTTP_URL, (url) => { + const marker = `${markerPrefix}${sanitizedUrls.length}\0`; + sanitizedUrls.push(sanitizeUrlValue(url)); + return marker; + }); + let sanitized = redactDiagnosticAssignments(masked); + sanitizedUrls.forEach((url, index) => { + sanitized = sanitized.replace(`${markerPrefix}${index}\0`, () => url); + }); + return sanitized; +} + +function sanitizeUrlPath(pathname: string): string { + const segments = pathname.split('/'); + const decodedSegments = segments.map((segment) => { + try { + return decodeURIComponent(segment); + } catch { + return segment; + } + }); + let redactNext = false; + let redactRemainder = false; + return segments + .map((segment, index) => { + const decoded = decodedSegments[index]; + if (redactRemainder) { + return segment ? encodeURIComponent(REDACTED) : segment; + } + if (redactNext) { + redactNext = false; + return encodeURIComponent(REDACTED); + } + const equalsIndex = decoded.indexOf('='); + if ( + equalsIndex > 0 && + isSensitiveUrlField(decoded.slice(0, equalsIndex)) + ) { + return `${encodeURIComponent(decoded.slice(0, equalsIndex))}=${encodeURIComponent(REDACTED)}`; + } + if (isAuthenticationPathSegment(decoded)) { + redactRemainder = true; + return segment; + } + if (AUTH_FLOW_PATH_KEY.test(decoded)) { + redactRemainder = true; + return segment; + } + if (isSensitiveUrlField(decoded)) { + redactNext = true; + } + return segment; + }) + .join('/'); +} + +function sanitizeArguments(command: string, args: string[]): string[] { + if (command === 'eval') { + return [command, '[REDACTED_SCRIPT]']; + } + if (command === 'batch') { + return [command, REDACTED_BATCH]; + } + if (command === 'auth') { + return [command, REDACTED_ARGUMENTS]; + } + if (command === 'find') { + return sanitizeFindArguments(args); + } + if ( + command === 'keyboard' && + ['inserttext', 'type'].includes(args[1]?.toLowerCase()) + ) { + return [command, args[1].toLowerCase(), REDACTED]; + } + if (['fill', 'select', 'type'].includes(command)) { + return [command, sanitizeArgument(args[1] || ''), REDACTED]; + } + if (command === 'set' && ['credentials', 'headers'].includes(args[1]?.toLowerCase())) { + return [command, args[1].toLowerCase(), REDACTED]; + } + if (command === 'cookies' && args[1]?.toLowerCase() === 'set') { + return [command, 'set', sanitizeArgument(args[2] || ''), REDACTED]; + } + if (command === 'storage') { + const setIndex = args.findIndex( + (argument, index) => index > 0 && argument.toLowerCase() === 'set', + ); + if (setIndex >= 0) { + return [ + ...args.slice(0, setIndex + 2).map(sanitizeArgument), + REDACTED, + ]; + } + } + if (command === 'upload') { + return [command, sanitizeArgument(args[1] || ''), '[LOCAL_FILE]']; + } + if (!isKnownCommand(command)) { + return args.length > 1 ? [command, REDACTED_ARGUMENTS] : [command]; + } + + const sanitized: string[] = []; + let redactNext = false; + for (const [index, argument] of args.entries()) { + if (redactNext) { + sanitized.push(REDACTED); + redactNext = false; + continue; + } + const equalsIndex = argument.indexOf('='); + if ( + index > 0 && + equalsIndex > 0 && + isSecretBearingCommandArgument(argument) + ) { + sanitized.push(`${argument.slice(0, equalsIndex)}=${REDACTED}`); + continue; + } + if (index > 0 && isSecretBearingCommandArgument(argument)) { + sanitized.push(argument); + redactNext = true; + continue; + } + sanitized.push(sanitizeArgument(argument)); + } + return sanitized; +} + +function sanitizeFindArguments(args: string[]): string[] { + const actionIndex = findActionIndex(args); + const nestedAction = args[actionIndex]?.toLowerCase(); + if (!nestedAction) { + return args.map(sanitizeArgument); + } + if (['fill', 'select', 'type'].includes(nestedAction)) { + return [...args.slice(0, actionIndex + 1).map(sanitizeArgument), REDACTED]; + } + return args.map(sanitizeArgument); +} + +function findActionIndex(args: string[]): number { + return args[1]?.toLowerCase() === 'nth' ? 4 : 3; +} + +function isKnownCommand(command: string): boolean { + return ( + HYBRID_ACTIONS.has(command) || + OBSERVATION_ACTIONS.has(command) || + POINTER_KEYBOARD_ACTIONS.has(command) || + SETUP_ACTIONS.has(command) || + ['cookies', 'find', 'keyboard', 'network', 'storage'].includes(command) + ); +} + +function sanitizeArgument(value: string): string { + if (URL_SCHEME.test(value)) { + return sanitizeUrlValue(value); + } + return sanitizeDiagnosticMessage(value) || ''; +} diff --git a/src/browser/redaction-policy.test-data.ts b/src/browser/redaction-policy.test-data.ts new file mode 100644 index 0000000..480b444 --- /dev/null +++ b/src/browser/redaction-policy.test-data.ts @@ -0,0 +1,59 @@ +export const CONVENTIONAL_FUSED_SECRET_FIELDS = [ + 'accesstoken', + 'accesstokens', + 'authtoken', + 'authtokens', + 'bearertoken', + 'bearertokens', + 'csrftoken', + 'csrftokens', + 'idtoken', + 'idtokens', + 'refreshtoken', + 'refreshtokens', + 'sessiontoken', + 'sessiontokens', + 'authcode', + 'authcodes', + 'authkey', + 'authkeys', + 'authorizationcode', + 'authorizationcodes', + 'authorizationkey', + 'authorizationkeys', + 'sessionid', + 'sessionids', + 'clientsecret', + 'clientsecrets', + 'passwordhash', + 'passwordhashes', + 'secretaccesskey', + 'secretaccesskeys', + 'secretkey', + 'secretkeys', + 'requestbody', + 'requestbodies', + 'responsebody', + 'responsebodies', + 'requestheader', + 'requestheaders', + 'responseheader', + 'responseheaders', + 'authorizationheader', + 'authorizationheaders', + 'cookieheader', + 'cookieheaders', + 'tokenvalue', + 'tokenvalues', + 'secretvalue', + 'secretvalues', + 'awsaccesskeyid', + 'awsaccesskeyids', +] as const; + +export const PUBLIC_FUSED_FIELDS = [ + 'monkey', + 'decode', + 'bodyguard', + 'tokenizer', +] as const; diff --git a/src/browser/redaction-policy.test.ts b/src/browser/redaction-policy.test.ts new file mode 100644 index 0000000..0e20b95 --- /dev/null +++ b/src/browser/redaction-policy.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest'; +import { + isAuthenticationPathSegment, + isHighConfidenceSecretField, + isSecretBearingCommandArgument, + isSensitiveUrlField, +} from './redaction-policy.js'; +import { + CONVENTIONAL_FUSED_SECRET_FIELDS, + PUBLIC_FUSED_FIELDS, +} from './redaction-policy.test-data.js'; + +describe('browser redaction policy', () => { + it.each([ + 'auth', + 'authorization', + 'HTTP_AUTHORIZATION', + 'cookie', + 'cookies', + 'credential', + 'credentials', + 'password', + 'passwords', + 'clientSecret', + 'clientSecrets', + 'session', + 'sessionId', + 'sessions', + 'sig', + 'sigs', + 'X-Amz-Signature', + 'signatures', + 'accessToken', + 'accessTokens', + 'headers', + 'requestBody', + 'responseBodies', + 'api-key', + 'apiKey', + 'apiKeys', + 'AwsAccessKeyId', + ...CONVENTIONAL_FUSED_SECRET_FIELDS, + ])('recognizes normalized high-confidence field %s', (field) => { + expect(isHighConfidenceSecretField(field)).toBe(true); + }); + + it.each([ + 'code', + 'codes', + 'key', + 'keys', + 'status', + ...PUBLIC_FUSED_FIELDS, + ])( + 'does not treat ambiguous or unrelated field %s as high-confidence', + (field) => { + expect(isHighConfidenceSecretField(field)).toBe(false); + }, + ); + + it.each(['code', 'codes', 'inviteCode', 'key', 'keys', 'product_key'])( + 'treats ambiguous field %s as sensitive only in URL context', + (field) => { + expect(isSensitiveUrlField(field)).toBe(true); + }, + ); + + it('recognizes secret-bearing command flags without inspecting their values', () => { + expect(isSecretBearingCommandArgument('--cookies')).toBe(true); + expect(isSecretBearingCommandArgument('--api-key=private')).toBe(true); + expect(isSecretBearingCommandArgument('--access-token=private')).toBe(true); + expect(isSecretBearingCommandArgument('--access-tokens=private')).toBe(true); + expect(isSecretBearingCommandArgument('--code=visible')).toBe(false); + expect(isSecretBearingCommandArgument('--key')).toBe(false); + expect( + isSecretBearingCommandArgument( + 'https://example.test/session/private-session', + ), + ).toBe(false); + expect(isSecretBearingCommandArgument('private-session')).toBe(false); + }); + + it('limits fail-closed route handling to auth path segments', () => { + expect(isAuthenticationPathSegment('auth')).toBe(true); + expect(isAuthenticationPathSegment('AUTHORIZATION')).toBe(true); + expect(isAuthenticationPathSegment('oauth')).toBe(false); + expect(isAuthenticationPathSegment('auth-token')).toBe(false); + }); +}); diff --git a/src/browser/redaction-policy.ts b/src/browser/redaction-policy.ts new file mode 100644 index 0000000..9996774 --- /dev/null +++ b/src/browser/redaction-policy.ts @@ -0,0 +1,163 @@ +const HIGH_CONFIDENCE_SECRET_TERMS = new Set([ + 'auth', + 'authorization', + 'body', + 'cookie', + 'credential', + 'header', + 'password', + 'secret', + 'session', + 'sig', + 'signature', + 'token', +]); +const URL_ONLY_SECRET_TERMS = new Set(['code', 'key']); + +type FusedSecretFieldRule = { + prefixes: readonly string[]; + suffixes: readonly string[]; +}; + +// Separator-free names cannot be tokenized safely. Match only conventional +// credential compounds, including the singular and plural terminal nouns. +// This keeps unrelated words such as bodyguard and tokenizer public without +// relying on an open-ended credential-prefix or credential-suffix heuristic. +const CONVENTIONAL_FUSED_SECRET_FIELD_GRAMMAR: readonly FusedSecretFieldRule[] = [ + { + prefixes: ['access', 'auth', 'bearer', 'csrf', 'id', 'refresh', 'session'], + suffixes: ['token', 'tokens'], + }, + { + prefixes: ['auth', 'authorization'], + suffixes: ['code', 'codes', 'key', 'keys'], + }, + { + prefixes: ['session'], + suffixes: ['id', 'ids'], + }, + { + prefixes: ['client'], + suffixes: ['secret', 'secrets'], + }, + { + prefixes: ['password'], + suffixes: ['hash', 'hashes'], + }, + { + prefixes: ['secretaccess'], + suffixes: ['key', 'keys'], + }, + { + prefixes: ['secret'], + suffixes: ['key', 'keys'], + }, + { + prefixes: ['request', 'response'], + suffixes: ['body', 'bodies', 'header', 'headers'], + }, + { + prefixes: ['authorization', 'cookie'], + suffixes: ['header', 'headers'], + }, + { + prefixes: ['token', 'secret'], + suffixes: ['value', 'values'], + }, + { + prefixes: ['aws'], + suffixes: ['accesskeyid', 'accesskeyids'], + }, +]; +const CONVENTIONAL_FUSED_SECRET_FIELDS = new Set( + CONVENTIONAL_FUSED_SECRET_FIELD_GRAMMAR.flatMap(({ prefixes, suffixes }) => + prefixes.flatMap((prefix) => + suffixes.map((suffix) => `${prefix}${suffix}`), + ), + ), +); +const PLURAL_FIELD_TERMS = new Map([ + ['apikeys', 'apikey'], + ['auths', 'auth'], + ['authorizations', 'authorization'], + ['bodies', 'body'], + ['codes', 'code'], + ['cookies', 'cookie'], + ['credentials', 'credential'], + ['headers', 'header'], + ['keys', 'key'], + ['passwords', 'password'], + ['secrets', 'secret'], + ['sessions', 'session'], + ['sigs', 'sig'], + ['signatures', 'signature'], + ['tokens', 'token'], +]); + +export function isHighConfidenceSecretField(value: string): boolean { + const terms = normalizeFieldName(value); + return containsHighConfidenceSecretTerm(terms); +} + +export function isSensitiveUrlField(value: string): boolean { + const terms = normalizeFieldName(value); + return ( + containsHighConfidenceSecretTerm(terms) || + terms.some((term) => URL_ONLY_SECRET_TERMS.has(term)) + ); +} + +export function isSecretBearingCommandArgument(value: string): boolean { + const equalsIndex = value.indexOf('='); + const field = equalsIndex >= 0 ? value.slice(0, equalsIndex) : value; + if (!/^(?:--)?[A-Za-z][A-Za-z0-9_-]*$/.test(field)) { + return false; + } + const terms = normalizeFieldName(field); + if (field.startsWith('--')) { + return containsHighConfidenceSecretTerm(terms); + } + return ( + (terms.length === 1 && containsHighConfidenceSecretTerm(terms)) || + (terms.length === 2 && isApiKey(terms)) + ); +} + +export function isAuthenticationPathSegment(value: string): boolean { + const terms = normalizeFieldName(value); + return ( + terms.length === 1 && + (terms[0] === 'auth' || terms[0] === 'authorization') + ); +} + +function normalizeFieldName(value: string): string[] { + return value + .replace(/^-+/, '') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1-$2') + .replace(/([a-z0-9])([A-Z])/g, '$1-$2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((term) => term.length > 0) + .map((term) => PLURAL_FIELD_TERMS.get(term) || term); +} + +function containsHighConfidenceSecretTerm(terms: string[]): boolean { + return ( + terms.some((term) => HIGH_CONFIDENCE_SECRET_TERMS.has(term)) || + terms.some(hasConventionalFusedSecretShape) || + hasConventionalFusedSecretShape(terms.join('')) || + terms.includes('apikey') || + isApiKey(terms) + ); +} + +function hasConventionalFusedSecretShape(term: string): boolean { + return CONVENTIONAL_FUSED_SECRET_FIELDS.has(term); +} + +function isApiKey(terms: string[]): boolean { + return terms.some( + (term, index) => term === 'api' && terms[index + 1] === 'key', + ); +} diff --git a/src/browser/runtime.ts b/src/browser/runtime.ts index ef99442..4248426 100644 --- a/src/browser/runtime.ts +++ b/src/browser/runtime.ts @@ -34,6 +34,7 @@ export function prepareAgentBrowserSocketDir( sessionName: string, env: NodeJS.ProcessEnv = process.env, accountHome = os.userInfo().homedir, + namespace?: string, ): string { const uid = process.getuid?.() ?? process.pid; const explicit = env.AGENT_BROWSER_SOCKET_DIR; @@ -58,7 +59,10 @@ export function prepareAgentBrowserSocketDir( fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); assertOwnedDirectory(directory); - const socketPath = path.join(directory, `${sessionName}.sock`); + const runtimeDirectory = resolveAgentBrowserRuntimeDir(directory, namespace); + fs.mkdirSync(runtimeDirectory, { recursive: true, mode: 0o700 }); + assertOwnedDirectory(runtimeDirectory); + const socketPath = path.join(runtimeDirectory, `${sessionName}.sock`); const byteLength = Buffer.byteLength(socketPath); if (byteLength > UNIX_SOCKET_PATH_MAX_BYTES) { throw new Error( @@ -70,6 +74,23 @@ export function prepareAgentBrowserSocketDir( return directory; } +export function resolveAgentBrowserRuntimeDir( + socketRoot: string, + namespace?: string, +): string { + if (!namespace) { + return socketRoot; + } + const normalizedNamespace = namespace + .toLowerCase() + .replace(/[^a-z0-9_-]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (!normalizedNamespace) { + throw new Error(`Unsafe agent-browser namespace: ${namespace}`); + } + return path.join(socketRoot, 'namespaces', normalizedNamespace, 'run'); +} + /** Read the exact daemon PID written for this isolated agent-browser session. */ export function captureAgentBrowserProcessIdentity( socketDir: string, diff --git a/src/browser/session.test.ts b/src/browser/session.test.ts index ad5b2f6..ff1753b 100644 --- a/src/browser/session.test.ts +++ b/src/browser/session.test.ts @@ -24,15 +24,18 @@ afterEach(() => { describe('buildOpenBrowserCommand', () => { it('builds a default open command without extra flags', () => { - expect(buildOpenBrowserCommand('http://localhost:3000')).toBe( - "open 'http://localhost:3000'", - ); + expect(buildOpenBrowserCommand('http://localhost:3000')).toEqual([ + 'open', + 'http://localhost:3000', + ]); }); it('includes headed mode when headless is disabled', () => { - expect(buildOpenBrowserCommand('http://localhost:3000', false)).toBe( - "open 'http://localhost:3000' --headed", - ); + expect(buildOpenBrowserCommand('http://localhost:3000', false)).toEqual([ + 'open', + 'http://localhost:3000', + '--headed', + ]); }); it('includes configurable browser flags from ProofShot config', () => { @@ -41,15 +44,22 @@ describe('buildOpenBrowserCommand', () => { ignoreHttpsErrors: true, executablePath: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', }), - ).toBe( - 'open \'https://localhost:3000\' --ignore-https-errors --executable-path "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"', - ); + ).toEqual([ + 'open', + 'https://localhost:3000', + '--ignore-https-errors', + '--executable-path', + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + ]); }); - it('quotes data URLs containing shell metacharacters', () => { + it('keeps data URLs containing shell metacharacters in one argument', () => { expect( buildOpenBrowserCommand("data:text/html,

Ready

"), - ).toBe("open 'data:text/html,

Ready

'"); + ).toEqual([ + 'open', + "data:text/html,

Ready

", + ]); }); }); @@ -70,10 +80,10 @@ describe('openBrowser', () => { 'slow-page', ); - expect(mocks.ab).toHaveBeenNthCalledWith(2, 'get url', { + expect(mocks.ab).toHaveBeenNthCalledWith(2, ['get', 'url'], { session: 'slow-page', }); - expect(mocks.ab).toHaveBeenNthCalledWith(3, 'set viewport 1280 720', { + expect(mocks.ab).toHaveBeenNthCalledWith(3, ['set', 'viewport', '1280', '720'], { session: 'slow-page', }); expect(warn).toHaveBeenCalledOnce(); diff --git a/src/browser/session.ts b/src/browser/session.ts index 885f243..f4fbbbd 100644 --- a/src/browser/session.ts +++ b/src/browser/session.ts @@ -1,7 +1,6 @@ import { ab, ProofShotError, - quoteShellArgument, } from '../utils/exec.js'; import type { BrowserConfig, ViewportConfig } from '../utils/config.js'; @@ -9,15 +8,16 @@ export function buildOpenBrowserCommand( url: string, headless = true, browserConfig?: BrowserConfig, -): string { - const flags: string[] = []; +): string[] { + const args = ['open', url]; - if (!headless) flags.push('--headed'); - if (browserConfig?.ignoreHttpsErrors) flags.push('--ignore-https-errors'); - if (browserConfig?.executablePath) flags.push(`--executable-path "${browserConfig.executablePath.replace(/"/g, '\\"')}"`); + if (!headless) args.push('--headed'); + if (browserConfig?.ignoreHttpsErrors) args.push('--ignore-https-errors'); + if (browserConfig?.executablePath) { + args.push('--executable-path', browserConfig.executablePath); + } - const suffix = flags.length > 0 ? ` ${flags.join(' ')}` : ''; - return `open ${quoteShellArgument(url)}${suffix}`; + return args; } /** @@ -45,7 +45,9 @@ export function openBrowser( 'Browser reached the target URL before its load event timed out; continuing with the active page.', ); } - ab(`set viewport ${viewport.width} ${viewport.height}`, { session: sessionName }); + ab(['set', 'viewport', String(viewport.width), String(viewport.height)], { + session: sessionName, + }); } function isNavigationTimeout(error: unknown): boolean { @@ -67,7 +69,7 @@ function urlsMatch(actual: string, expected: string): boolean { * Close the browser session. */ export function closeBrowser(sessionName?: string): void { - ab('close', { session: sessionName }); + ab(['close'], { session: sessionName }); } /** @@ -75,7 +77,7 @@ export function closeBrowser(sessionName?: string): void { */ export function checkAgentBrowser(): boolean { try { - ab('--version', 5000); + ab(['--version'], 5000); return true; } catch { return false; @@ -86,14 +88,14 @@ export function checkAgentBrowser(): boolean { * Get any console errors from the current page. */ export function getConsoleErrors(sessionName?: string): string { - return ab('errors', { session: sessionName }); + return ab(['errors'], { session: sessionName }); } /** * Get console output from the current page. */ export function getConsoleOutput(sessionName?: string): string { - return ab('console', { session: sessionName }); + return ab(['console'], { session: sessionName }); } export interface ConsoleMessage { @@ -106,7 +108,7 @@ export interface ConsoleMessage { * Get console output as structured JSON with per-message timestamps. */ export function getConsoleOutputJson(sessionName?: string): ConsoleMessage[] { - const raw = ab('console --json', { session: sessionName }); + const raw = ab(['console', '--json'], { session: sessionName }); const parsed = JSON.parse(raw); // agent-browser wraps JSON output: {success, data: {messages: [...]}, error} const messages = parsed?.data?.messages ?? parsed; @@ -121,7 +123,7 @@ export function getConsoleOutputJson(sessionName?: string): ConsoleMessage[] { */ export function getPageTitle(sessionName?: string): string { try { - return ab('get title', { session: sessionName }); + return ab(['get', 'title'], { session: sessionName }); } catch { return ''; } @@ -132,7 +134,7 @@ export function getPageTitle(sessionName?: string): string { */ export function getPageUrl(sessionName?: string): string { try { - return ab('get url', { session: sessionName }); + return ab(['get', 'url'], { session: sessionName }); } catch { return ''; } diff --git a/src/cli.ts b/src/cli.ts index 4540cb2..c3f3259 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -46,9 +46,13 @@ export function createCLI(): Command { program .command('stop') .description('Stop session: stop recording, collect errors, bundle proof artifacts') + .option('--session ', 'Stop one exact registered session') .option('--no-close', 'Don\'t close the browser (keep it open for further use)') .action(async (options) => { - await stopCommand({ noClose: options.close === false }); + await stopCommand({ + noClose: options.close === false, + session: options.session, + }); }); program @@ -111,10 +115,11 @@ export function createCLI(): Command { program .command('exec') .description('Run an agent-browser command with logging (use instead of agent-browser directly)') + .option('--session ', 'Address one exact registered session') .argument('', 'agent-browser command and arguments') .allowUnknownOption() - .action(async (args) => { - await execCommand(args); + .action(async (args, options) => { + await execCommand(args, { session: options.session }); }); const session = program diff --git a/src/commands/clean.test.ts b/src/commands/clean.test.ts index 9355030..b48b83d 100644 --- a/src/commands/clean.test.ts +++ b/src/commands/clean.test.ts @@ -3,8 +3,14 @@ import * as os from 'os'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -const mocks = vi.hoisted(() => ({ loadConfig: vi.fn() })); +const mocks = vi.hoisted(() => ({ + loadConfig: vi.fn(), + listSessionsForControlDir: vi.fn(), +})); vi.mock('../utils/config.js', () => ({ loadConfig: mocks.loadConfig })); +vi.mock('../session/selection.js', () => ({ + listSessionsForControlDir: mocks.listSessionsForControlDir, +})); import { cleanCommand } from './clean.js'; @@ -15,6 +21,7 @@ beforeEach(() => { fs.mkdirSync(cache, { recursive: true }); root = fs.mkdtempSync(path.join(cache, 'proofshot-clean-test-')); mocks.loadConfig.mockReturnValue({ output: root }); + vi.spyOn(console, 'log').mockImplementation(() => {}); vi.spyOn(console, 'error').mockImplementation(() => {}); vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { throw new Error(`process.exit:${code ?? 0}`); @@ -28,18 +35,30 @@ afterEach(() => { }); describe('cleanCommand', () => { - it('refuses to discard active exact-process ownership metadata', async () => { - const controlPath = path.join(root, '.session.json'); - const evidencePath = path.join(root, 'evidence.txt'); - fs.writeFileSync(controlPath, JSON.stringify({ browserRetained: true })); - fs.writeFileSync(evidencePath, 'keep'); - - await expect(cleanCommand()).rejects.toThrow('process.exit:1'); - - expect(fs.readFileSync(controlPath, 'utf-8')).toContain('browserRetained'); - expect(fs.readFileSync(evidencePath, 'utf-8')).toBe('keep'); - expect(console.error).toHaveBeenCalledWith( - expect.stringContaining('Run "proofshot stop" first'), - ); + it.each(['starting', 'active', 'stopping', 'recovery', undefined])( + 'refuses to discard evidence while %s registry state remains', + async (lifecycleStatus) => { + mocks.listSessionsForControlDir.mockReturnValue([ + { lifecycleStatus, sessionName: 'ps-registered' }, + ]); + const evidencePath = path.join(root, 'evidence.txt'); + fs.writeFileSync(evidencePath, 'keep'); + + await expect(cleanCommand()).rejects.toThrow('process.exit:1'); + + expect(fs.readFileSync(evidencePath, 'utf-8')).toBe('keep'); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining('Run "proofshot stop" or "proofshot session clean"'), + ); + }, + ); + + it('removes evidence after all registry state is cleared', async () => { + mocks.listSessionsForControlDir.mockReturnValue([]); + fs.writeFileSync(path.join(root, 'evidence.txt'), 'remove'); + + await cleanCommand(); + + expect(fs.existsSync(root)).toBe(false); }); }); diff --git a/src/commands/clean.ts b/src/commands/clean.ts index 72f1676..2e0446b 100644 --- a/src/commands/clean.ts +++ b/src/commands/clean.ts @@ -2,18 +2,21 @@ import * as fs from 'fs'; import * as path from 'path'; import chalk from 'chalk'; import { loadConfig } from '../utils/config.js'; -import { hasActiveSession, resolveSessionControlDir } from '../session/state.js'; +import { resolveSessionControlDir } from '../session/state.js'; +import { listSessionsForControlDir } from '../session/selection.js'; export async function cleanCommand(): Promise { const config = loadConfig(); const controlDir = resolveSessionControlDir(config.output); const outputDir = path.resolve(config.output); - if (hasActiveSession(controlDir)) { + if (listSessionsForControlDir(controlDir).length > 0) { console.error( chalk.red('✗') + - ' Cannot clean while a ProofShot session owns browser or server processes.\n' + - chalk.dim('Run "proofshot stop" first so exact cleanup metadata is preserved.'), + ' Cannot clean while ProofShot has registered session state for this output.\n' + + chalk.dim( + 'Run "proofshot stop" or "proofshot session clean" first so evidence and exact cleanup metadata are preserved.', + ), ); process.exit(1); return; diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index a5eeefd..4e351bf 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { findConfigPathMock, loadConfigMock, loadSessionMock, listRegisteredSessionsMock, resolveSessionControlDirMock, findExecutablePathMock, readCommandVersionMock } = +const { findConfigPathMock, loadConfigMock, loadSessionMock, listRegisteredSessionsMock, resolveSessionControlDirMock, findExecutablePathMock, getTcpListenerInspectorStatusMock, readCommandVersionMock } = vi.hoisted(() => ({ findConfigPathMock: vi.fn(), loadConfigMock: vi.fn(), @@ -8,6 +8,7 @@ const { findConfigPathMock, loadConfigMock, loadSessionMock, listRegisteredSessi listRegisteredSessionsMock: vi.fn(), resolveSessionControlDirMock: vi.fn(), findExecutablePathMock: vi.fn(), + getTcpListenerInspectorStatusMock: vi.fn(), readCommandVersionMock: vi.fn(), })); @@ -26,6 +27,7 @@ vi.mock('../session/registry.js', () => ({ vi.mock('../utils/process.js', () => ({ findExecutablePath: findExecutablePathMock, + getTcpListenerInspectorStatus: getTcpListenerInspectorStatusMock, readCommandVersion: readCommandVersionMock, })); @@ -45,9 +47,18 @@ describe('doctorCommand', () => { loadSessionMock.mockReturnValue(null); listRegisteredSessionsMock.mockReturnValue([]); resolveSessionControlDirMock.mockReturnValue('/workspace/proofshot-artifacts'); - findExecutablePathMock.mockImplementation((name: string) => - name === 'agent-browser' ? '/usr/local/bin/agent-browser' : '/opt/homebrew/bin/ffmpeg', - ); + findExecutablePathMock.mockImplementation((name: string) => { + if (name === 'agent-browser') return '/usr/local/bin/agent-browser'; + if (name === 'ffmpeg') return '/opt/homebrew/bin/ffmpeg'; + if (name === 'lsof') return '/usr/sbin/lsof'; + return null; + }); + getTcpListenerInspectorStatusMock.mockReturnValue({ + available: true, + command: 'lsof', + label: 'lsof', + error: null, + }); readCommandVersionMock.mockImplementation((name: string) => name === 'agent-browser' ? 'agent-browser 0.25.3' : 'ffmpeg version 7.0', ); @@ -62,7 +73,32 @@ describe('doctorCommand', () => { expect(output).toContain('ProofShot Doctor'); expect(output).toContain('agent-browser'); expect(output).toContain('ffmpeg'); + expect(output).toContain('lsof (TCP listener inspection)'); + expect(output).toContain('/usr/sbin/lsof'); expect(output).toContain('1280x720'); expect(output).toContain('proofshot-artifacts'); }); + + it('prints an actionable listener-inspection failure', async () => { + getTcpListenerInspectorStatusMock.mockReturnValue({ + available: false, + command: 'lsof', + label: 'lsof', + error: 'lsof is required to verify listener ownership. Install lsof and retry.', + }); + findExecutablePathMock.mockImplementation((name: string) => { + if (name === 'agent-browser') return '/usr/local/bin/agent-browser'; + if (name === 'ffmpeg') return '/opt/homebrew/bin/ffmpeg'; + return null; + }); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + await doctorCommand(); + + const output = logSpy.mock.calls.map(([line]) => String(line)).join('\n'); + expect(output).toContain('lsof (TCP listener inspection)'); + expect(output).toContain('not found'); + expect(output).toContain('lsof is required'); + expect(output).toContain('Install lsof and retry'); + }); }); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 96c6734..f08c383 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -1,9 +1,17 @@ import chalk from 'chalk'; import { PROOFSHOT_VERSION } from '../version.js'; import { findConfigPath, loadConfig } from '../utils/config.js'; -import { findExecutablePath, readCommandVersion } from '../utils/process.js'; -import { loadSession, resolveSessionControlDir } from '../session/state.js'; +import { + findExecutablePath, + getTcpListenerInspectorStatus, + readCommandVersion, +} from '../utils/process.js'; +import { resolveSessionControlDir } from '../session/state.js'; import { listRegisteredSessions } from '../session/registry.js'; +import { + listSessionsForControlDir, + sessionHasVerifiedLiveOwnership, +} from '../session/selection.js'; function statusLabel(ok: boolean, text: string): string { return ok ? `${chalk.green('✓')} ${text}` : `${chalk.yellow('⚠')} ${text}`; @@ -17,11 +25,18 @@ export async function doctorCommand(): Promise { const configPath = findConfigPath(); const config = loadConfig(); const controlDir = resolveSessionControlDir(config.output); - const session = loadSession(controlDir); + const activeSessions = listSessionsForControlDir(controlDir).filter( + sessionHasVerifiedLiveOwnership, + ); + const session = activeSessions.length === 1 ? activeSessions[0] : null; const registeredSessions = listRegisteredSessions(); const agentBrowserPath = findExecutablePath('agent-browser'); const ffmpegPath = findExecutablePath('ffmpeg'); + const listenerInspector = getTcpListenerInspectorStatus(); + const listenerInspectorPath = listenerInspector.command + ? findExecutablePath(listenerInspector.command) + : null; const agentBrowserVersion = readCommandVersion('agent-browser'); const ffmpegVersion = readCommandVersion('ffmpeg'); @@ -46,7 +61,20 @@ export async function doctorCommand(): Promise { printLine('Version', ffmpegVersion || chalk.dim('not available')); console.log(''); - console.log(statusLabel(Boolean(session), 'active session')); + console.log( + statusLabel( + listenerInspector.available, + `${listenerInspector.label} (TCP listener inspection)`, + ), + ); + printLine('Path', listenerInspectorPath || chalk.dim('not found')); + if (listenerInspector.error) { + printLine('Issue', listenerInspector.error); + } + console.log(''); + + console.log(statusLabel(activeSessions.length > 0, 'active session')); + printLine('Active here', String(activeSessions.length)); printLine('Sessions', String(registeredSessions.length)); if (session) { printLine('Session dir', session.sessionDir); @@ -54,6 +82,9 @@ export async function doctorCommand(): Promise { printLine('Port', String(session.port)); if (session.targetUrl) printLine('Target', session.targetUrl); } else { - printLine('Session dir', chalk.dim('none')); + printLine( + 'Session dir', + activeSessions.length > 1 ? chalk.dim('multiple; use session list') : chalk.dim('none'), + ); } } diff --git a/src/commands/exec.test.ts b/src/commands/exec.test.ts index 313eeb9..9995a45 100644 --- a/src/commands/exec.test.ts +++ b/src/commands/exec.test.ts @@ -1,26 +1,57 @@ -import { describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { setAgentBrowserDefaults } from '../utils/exec.js'; import { - buildShellCommand, + assertControlledAgentBrowserCommand, + buildExecInvocation, + prepareControlledAgentBrowserCommand, translateProofShotExecArgs, } from './exec.js'; -describe('buildShellCommand', () => { +describe('buildExecInvocation', () => { + beforeEach(() => { + setAgentBrowserDefaults({ + executablePath: '/opt/node24/bin/agent-browser', + }); + }); + + afterEach(() => { + setAgentBrowserDefaults({}); + }); + it('routes regular commands through the active ProofShot session', () => { - expect(buildShellCommand(['click', '@e2'], 'proofshot-2026-04-07_22-30-00')).toBe( - "agent-browser --session 'proofshot-2026-04-07_22-30-00' click @e2", - ); + expect(buildExecInvocation(['click', '@e2'], 'proofshot-2026-04-07_22-30-00')).toEqual({ + executablePath: '/opt/node24/bin/agent-browser', + args: [ + '--session', + 'proofshot-2026-04-07_22-30-00', + 'click', + '@e2', + ], + }); }); - it('preserves eval shell quoting while adding the session flag', () => { - expect(buildShellCommand(['eval', "console.log('hello')"], 'proofshot-dev')).toBe( - "agent-browser --session 'proofshot-dev' eval 'console.log('\\''hello'\\'')'", - ); + it('preserves eval source as one argument while adding the session flag', () => { + expect(buildExecInvocation(['eval', "console.log('hello')"], 'proofshot-dev')).toEqual({ + executablePath: '/opt/node24/bin/agent-browser', + args: [ + '--session', + 'proofshot-dev', + 'eval', + "console.log('hello')", + ], + }); }); - it('quotes regular arguments that contain shell metacharacters', () => { - expect(buildShellCommand(['screenshot', 'step (1).png'], 'proofshot-dev')).toBe( - "agent-browser --session 'proofshot-dev' screenshot 'step (1).png'", - ); + it('preserves regular arguments that contain shell metacharacters', () => { + expect(buildExecInvocation(['screenshot', 'step $(touch nope); (1).png'], 'proofshot-dev')).toEqual({ + executablePath: '/opt/node24/bin/agent-browser', + args: [ + '--session', + 'proofshot-dev', + 'screenshot', + 'step $(touch nope); (1).png', + ], + }); }); it('translates visible-selector assertions into an evidence-bearing browser query', () => { @@ -31,4 +62,161 @@ describe('buildShellCommand', () => { expectedSelector: '#ready', }); }); + + it.each([ + ['snapshot', '--session=another-session'], + ['open', 'https://example.com', '--allowed-domains=attacker.invalid'], + ['open', 'https://example.com', '--provider', 'browserbase'], + ['open', 'https://example.com', '--cdp=9222'], + ['open', 'https://example.com', '--profile', './shared-profile'], + ['open', 'https://example.com', '--state=./shared-state.json'], + ['open', 'https://example.com', '--config', './untrusted.json'], + ['open', 'https://example.com', '--executable-path=/tmp/browser'], + ['open', 'https://example.com', '--namespace=shared'], + ])('rejects ProofShot-owned global options: %j', (...args) => { + expect(() => assertControlledAgentBrowserCommand(args)).toThrow( + 'owned by ProofShot', + ); + }); + + it('rejects commands that can escape the owned browser lifecycle', () => { + for (const args of [ + ['auth', 'login', 'shared'], + ['chat', 'inspect the page'], + ['clipboard', 'read'], + ['close'], + ['connect', '9222'], + ['dashboard', 'start'], + ['doctor', '--fix'], + ['download', '@e1', '/tmp/report.csv'], + ['install'], + ['pdf', '/tmp/page.pdf'], + ['profiles'], + ['pushstate', 'https://example.com/private'], + ['record', 'stop'], + ['session', 'list'], + ['state', 'load', './shared.json'], + ['stream', 'enable', '--port', '4848'], + ['trace', 'stop', '/tmp/trace.zip'], + ['upgrade'], + ['future-command', 'value'], + ]) { + expect(() => assertControlledAgentBrowserCommand(args)).toThrow( + /not permitted|cannot override ProofShot-owned browser state/, + ); + } + expect(() => + assertControlledAgentBrowserCommand(['network', 'har', 'stop']), + ).toThrow('HAR capture is owned by ProofShot'); + expect(() => + assertControlledAgentBrowserCommand([ + 'batch', + 'snapshot -i', + 'open https://example.com --provider browserbase', + ]), + ).toThrow('--provider is owned by ProofShot'); + expect(() => + assertControlledAgentBrowserCommand([ + 'batch', + 'snapshot -i', + 'dashboard start --port 4848', + ]), + ).toThrow('dashboard is not permitted'); + expect(() => + assertControlledAgentBrowserCommand([ + 'batch', + 'snapshot -i', + 'screenshot /tmp/outside.png', + ]), + ).toThrow('screenshot must run as a separate ProofShot action'); + }); + + it.each([ + ['get', 'cdp-url'], + ['network', 'har', 'stop'], + ['network', 'request', '1234.5'], + ['network', 'requests', '--clear'], + ['network', 'route', '**/api/*', '--abort'], + ['console', '--clear'], + ['console', '--future-option'], + ['cookies', 'import', './shared.json'], + ['errors', '--clear'], + ['keyboard', 'paste', 'private'], + ['mouse', 'teleport', '1', '2'], + ['set', 'profile', 'shared'], + ['storage', 'indexeddb', 'get'], + ['storage', 'local', 'export'], + ])('rejects unsupported or evidence-destructive subcommands: %j', (...args) => { + expect(() => assertControlledAgentBrowserCommand(args)).toThrow( + /not permitted|owned by ProofShot/, + ); + }); + + it('requires screenshots to use one PNG filename in the session root', () => { + expect( + prepareControlledAgentBrowserCommand( + ['screenshot', '--full', 'step (1).png'], + '/evidence/session', + ), + ).toEqual([ + 'screenshot', + '--full', + '/evidence/session/step (1).png', + ]); + + for (const args of [ + ['screenshot'], + ['screenshot', '--annotate'], + ['screenshot', '../outside.png'], + ['screenshot', '/tmp/outside.png'], + ['screenshot', 'step.jpg'], + ['screenshot', '--screenshot-dir', '/tmp'], + ]) { + expect(() => + prepareControlledAgentBrowserCommand(args, '/evidence/session'), + ).toThrow(/PNG filename directly inside/); + } + }); + + it('preserves legitimate command-specific options', () => { + expect(() => + assertControlledAgentBrowserCommand(['snapshot', '-i', '--depth', '3']), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand([ + 'find', + 'role', + 'button', + 'click', + '--name', + 'Submit', + ]), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand([ + 'open', + 'https://example.com', + '--headers', + '{"X-Test":"proof"}', + ]), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand(['network', 'requests']), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand(['keyboard', 'type', 'hello']), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand(['set', 'viewport', '1280', '720']), + ).not.toThrow(); + expect(() => + assertControlledAgentBrowserCommand([ + 'storage', + 'local', + 'set', + 'theme', + 'dark', + ]), + ).not.toThrow(); + }); }); diff --git a/src/commands/exec.ts b/src/commands/exec.ts index 7cb844a..f9735f7 100644 --- a/src/commands/exec.ts +++ b/src/commands/exec.ts @@ -1,99 +1,62 @@ -import * as fs from 'fs'; -import * as path from 'path'; -import { execSync } from 'child_process'; +import { + formatAgentBrowserOutputForDisplay, + sanitizeAgentBrowserError, + writePrivateAgentBrowserResult, + type AgentBrowserResultReceipt, +} from '../browser/evidence.js'; +import { + buildSanitizedCommandIntent, + classifyInteraction, + sanitizeDiagnosticMessage, + sanitizePageUrl, +} from '../browser/provenance.js'; +import { + assertControlledAgentBrowserCommand, + prepareControlledAgentBrowserCommand, +} from '../browser/command-policy.js'; import { loadConfig, normalizeViewport } from '../utils/config.js'; import { ab, - buildAgentBrowserCommand, - getAgentBrowserEnvironment, + buildAgentBrowserInvocation, + executeAgentBrowser, setAgentBrowserDefaults, + type AgentBrowserInvocation, } from '../utils/exec.js'; import { - loadSession, resolveSessionControlDir, - saveSession, - type SessionState, } from '../session/state.js'; +import { backfillSessionAgentBrowserRuntime } from '../session/browser-runtime.js'; import { canAddressOwnedBrowserSession } from '../session/lifecycle.js'; import { getPageUrl } from '../browser/session.js'; -import { registerSession } from '../session/registry.js'; +import { + claimSessionOperation, + registerSession, + releaseSessionOperation, +} from '../session/registry.js'; +import { resolveLiveSession } from '../session/selection.js'; +import { + appendSessionLogEntry, + persistSessionLogEntry, + type SessionLogEntry, +} from '../session/action-log.js'; -const SESSION_LOG_FILENAME = 'session-log.json'; -const SESSION_LOG_LOCK_TIMEOUT_MS = 5000; -const SESSION_LOG_STALE_LOCK_MS = 120000; - -export interface SessionLogEntry { - action: string; - relativeTimeSec: number; - timestamp: string; - outcome?: 'passed' | 'failed'; - expectedSelector?: string; - error?: string; - pageUrl?: string; - element?: { - label: string; - bbox: { x: number; y: number; width: number; height: number }; - viewport: { width: number; height: number }; - }; -} +type ExecOptions = { + session?: string; +}; /** - * Load existing session log entries from disk. + * Build an executable + argv invocation for agent-browser. */ -export function loadSessionLog(sessionDir: string): SessionLogEntry[] { - const logPath = path.join(sessionDir, SESSION_LOG_FILENAME); - if (!fs.existsSync(logPath)) return []; - try { - const parsed: unknown = JSON.parse(fs.readFileSync(logPath, 'utf-8')); - if (!Array.isArray(parsed)) { - throw new Error('session log root must be an array'); - } - return parsed as SessionLogEntry[]; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error(`ProofShot session action log is corrupt: ${logPath}\n${message}`); - } -} - -/** - * For screenshot commands, resolve relative paths into the session directory - * so agents can just say `proofshot exec screenshot step-name.png`. - */ -function resolveScreenshotPath(args: string[], sessionDir: string): string[] { - if (args[0] !== 'screenshot' || args.length < 2) return args; - - const screenshotPath = args[args.length - 1]; - const resolved = path.resolve(sessionDir, screenshotPath); - if (path.dirname(resolved) !== path.resolve(sessionDir)) { - throw new Error( - 'ProofShot screenshots must use a filename directly inside the active session.', - ); - } - return [...args.slice(0, -1), resolved]; -} - -/** - * Build the shell command string for agent-browser. - * - * For `eval` commands, we need to pass the JS code as a single quoted argument - * to prevent the shell from interpreting parentheses, brackets, etc. - * For other commands, simple joining is fine. - */ -export function buildShellCommand(args: string[], sessionName?: string): string { - if (args[0] === 'eval' && args.length > 1) { - const jsCode = args.slice(1).join(' '); - const escaped = jsCode.replace(/'/g, "'\\''"); - return buildAgentBrowserCommand(`eval '${escaped}'`, { session: sessionName }); - } - - const quotedArgs = args.map((arg) => { - if (/[(){}[\]$`!#&|;<>*? "'\\]/.test(arg)) { - const escaped = arg.replace(/'/g, "'\\''"); - return `'${escaped}'`; - } - return arg; +export function buildExecInvocation( + args: string[], + sessionName?: string, + structuredOutput = false, +): AgentBrowserInvocation { + assertControlledAgentBrowserCommand(args); + return buildAgentBrowserInvocation(normalizeExecArgs(args), { + json: structuredOutput, + session: sessionName, }); - return buildAgentBrowserCommand(quotedArgs.join(' '), { session: sessionName }); } export function translateProofShotExecArgs(args: string[]): { @@ -142,18 +105,21 @@ function captureElementData( // Strategy 1: Try id-based selector (works for inputs with id attributes) let elemId = ''; - try { elemId = ab(`get attr ${ref} id`, { session: sessionName }); } catch { /* empty */ } + try { elemId = ab(['get', 'attr', ref, 'id'], { session: sessionName }); } catch { /* empty */ } if (elemId) { try { - const raw = ab(`get box '#${elemId}'`, { session: sessionName }); + const raw = ab(['get', 'box', `#${elemId}`], { session: sessionName }); bbox = JSON.parse(raw); } catch { /* empty */ } // For inputs, get label from associated