From db8fef8075a4b80cb5b5589a84bc94085b800da7 Mon Sep 17 00:00:00 2001 From: CuriosityOS <184718100+CuriosityOS@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:19:59 +0000 Subject: [PATCH 1/3] Add binary-targeted E2E suite and fix exposed runtime regressions --- .github/workflows/ci.yml | 12 +- CHANGELOG.md | 14 + CONTRIBUTING.md | 5 + bin/cli-main.ts | 4 +- docs/binary-e2e.md | 152 +++++ package.json | 1 + src/mcp-server.ts | 25 +- src/navigation-defaults.ts | 28 +- src/worker.ts | 38 +- tests/e2e/browser-cases.ts | 829 ++++++++++++++++++++++++++ tests/e2e/browser-fixtures.ts | 115 ++++ tests/e2e/cli-cases.ts | 395 ++++++++++++ tests/e2e/harness.ts | 373 ++++++++++++ tests/e2e/lifecycle-cases.ts | 190 ++++++ tests/e2e/protocol-cases.ts | 335 +++++++++++ tests/e2e/run.ts | 157 +++++ tests/e2e/security-cases.ts | 741 +++++++++++++++++++++++ tests/e2e/security-fixtures.ts | 165 +++++ tests/e2e/types.ts | 47 ++ tests/node/e2e-harness.test.ts | 198 ++++++ tests/node/fixtures/e2e-target.ts | 38 ++ tests/node/navigation-history.test.ts | 60 ++ tsconfig.harness.json | 1 + 23 files changed, 3901 insertions(+), 22 deletions(-) create mode 100644 docs/binary-e2e.md create mode 100644 tests/e2e/browser-cases.ts create mode 100644 tests/e2e/browser-fixtures.ts create mode 100644 tests/e2e/cli-cases.ts create mode 100644 tests/e2e/harness.ts create mode 100644 tests/e2e/lifecycle-cases.ts create mode 100644 tests/e2e/protocol-cases.ts create mode 100644 tests/e2e/run.ts create mode 100644 tests/e2e/security-cases.ts create mode 100644 tests/e2e/security-fixtures.ts create mode 100644 tests/e2e/types.ts create mode 100644 tests/node/e2e-harness.test.ts create mode 100644 tests/node/fixtures/e2e-target.ts create mode 100644 tests/node/navigation-history.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dbfecfc..cd5df6db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: # Keep this display name stable because main branch protection requires it. name: Node tests runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 @@ -91,6 +91,16 @@ jobs: env: BETTERWRIGHT_REQUIRE_BROWSER: "1" BETTERWRIGHT_REQUIRE_RECORDING: "1" + - name: Test the built binary end to end + run: bun run test:e2e --binary bun --binary-arg "$PWD/dist/bin/betterwright.js" --require-all + - name: Upload binary E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: binary-e2e-report + path: .tmp/e2e-report.json + include-hidden-files: true + if-no-files-found: ignore cross-platform: # A NEW job, deliberately not a matrix over the two jobs above. Branch diff --git a/CHANGELOG.md b/CHANGELOG.md index f89d9513..fb865f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ Releases before 1.1.3 predate this file; their notes live on the ### Changed +- `bun run test:e2e` tests a supplied BetterWright executable or `betterwright` + on PATH through isolated local fixtures, with per-case JSON reporting, + explicit skips, and a strict CI mode. See `docs/binary-e2e.md` for coverage + and external-integration limits. - SECURITY.md, README, and the credential and network-policy docs now state the boundary precisely: redaction removes literal secret values from result envelopes and does not chase transformed read-backs, and an upstream egress @@ -38,6 +42,16 @@ Releases before 1.1.3 predate this file; their notes live on the ### Fixed +- `human.type(..., { clear: false })` positions the caret at the end before + typing, rather than inserting at the point where the focus click landed. +- History navigation waits for document readiness on back/forward-cache + restores without waiting for lifecycle events that have already fired. +- The MCP stdio server closes its browser when stdin ends, including when the + transport does not report EOF through its protocol close callback. +- Starting a live view no longer clears content created with `page.setContent` + on an `about:blank` tab while warming up an otherwise empty page. +- The stealth worker selects `patchright-core` directly, so Bun's handling of + Node loader hooks cannot silently leave the stock driver active. - On Windows, a vault lock publish rename that returns EPERM or EACCES while the destination is already gone is treated as lock contention only after the candidate can still rename to a unique sibling. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9db1ce0a..c86dfdba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,11 @@ managed installation, or point them at a real artifact; `off` is no longer supported. The policy, vault, prompt, and challenge suites run anywhere. +To test a shipped CLI as a black box, run `bun run test:e2e` for `betterwright` +on PATH, or `bun run test:e2e /absolute/path/to/betterwright`. The +[binary E2E guide](docs/binary-e2e.md) covers local-build targets, isolated +fixtures, strict skip handling, reports, and the suite's coverage limits. + One note on running the suite locally: **do not run the tests as root.** Several tests simulate an unwritable directory with `chmod`, which root bypasses; they detect this and skip, so a root run reports green while diff --git a/bin/cli-main.ts b/bin/cli-main.ts index f559adee..ce2984f2 100644 --- a/bin/cli-main.ts +++ b/bin/cli-main.ts @@ -2091,7 +2091,7 @@ async function cmdView(flags) { startedHere = !view.alreadyRunning; // Nudge a blank tab so the canvas is not empty on first open. await browser.run( - "if (page.url() === 'about:blank') await page.goto('about:blank'); 'ready'", + "if (page.url() === 'about:blank' && await page.evaluate(() => !document.title && !document.body?.childNodes.length)) await page.goto('about:blank'); 'ready'", ); printLiveViewBanner(view, { dim, bold, attached: true }); await new Promise((resolve) => { @@ -2131,7 +2131,7 @@ async function cmdView(flags) { console.error(view.error || "The live view failed to start."); return 1; } - await browser.run("if (page.url() === 'about:blank') await page.goto('about:blank'); 'ready'"); + await browser.run("if (page.url() === 'about:blank' && await page.evaluate(() => !document.title && !document.body?.childNodes.length)) await page.goto('about:blank'); 'ready'"); printLiveViewBanner(view, { dim, bold, attached: false }); await new Promise((resolve) => { process.once("SIGINT", resolve); diff --git a/docs/binary-e2e.md b/docs/binary-e2e.md new file mode 100644 index 00000000..7e7ff1ac --- /dev/null +++ b/docs/binary-e2e.md @@ -0,0 +1,152 @@ +# Test a BetterWright binary end to end + +The binary E2E suite launches the executable you specify and checks its public +CLI, browser, security, credential, recording, and MCP behavior. It does not +import `src/`, `dist/`, or the target's installed package. It can therefore test +a local build, a globally installed release, or a separately installed candidate +without accidentally testing the checkout instead. + +This is a broad regression suite, not proof that every input or external +integration works. Keep the repository's unit, type, package, Electron, and +cross-platform checks alongside it. + +## Run it + +Use the supported Bun version from `.bun-version`. Install the target binary's +managed browser with that binary's `setup` command first. The suite does not +download browsers, install models, sign in, or call paid providers. + +```bash +# Test betterwright on PATH. +bun run test:e2e + +# Test a specific executable, including paths containing spaces. +bun run test:e2e /absolute/path/to/betterwright + +# Test this checkout's compiled CLI through Bun. +bun run build +bun run test:e2e --binary bun --binary-arg "$PWD/dist/bin/betterwright.js" + +# Require every selected case to run, including optional capabilities. +bun run test:e2e /absolute/path/to/betterwright --require-all +``` + +`--binary-arg` is repeatable and prepends literal arguments to the executable. +There is no shell interpolation. Use absolute paths for launcher scripts and +preloads: target processes run from an isolated temporary working directory. +To use only the suite outside this checkout, copy the complete `tests/e2e/` +directory and run `bun /path/to/e2e/run.ts [binary]`. + +The runner honors `BETTERWRIGHT_CHROMIUM_PATH` or +`BETTERWRIGHT_CHROMIUM_ROOT`. Otherwise it points at the invoking user's +existing `~/.betterwright/chromium` installation while isolating the target's +home. Recording checks require FFmpeg with `libx264` and `libvpx`; an encoder +can be selected with `BETTERWRIGHT_FFMPEG_PATH`. + +## Select cases and diagnose failures + +```bash +bun run test:e2e --list +bun run test:e2e --group security +bun run test:e2e --group cli --group protocol +bun run test:e2e --filter browser.downloads --keep-work +bun run test:e2e --report /absolute/path/to/report.json +``` + +Groups are `cli`, `browser`, `security`, `protocol`, and `recording`. The filter +matches a case ID or title, case-sensitively. Empty selections and duplicate +case IDs fail instead of producing a successful empty run. + +Each command has a 45-second deadline, and each case has a 180-second deadline. +Override them with `--timeout ` and +`--case-timeout `. A timeout is a failure, not a skip. The runner +continues after failed cases to show more than the first defect. + +The default report is `.tmp/e2e-report.json`. It is updated atomically after +each completed case and includes the target version, platform, selected count, +individual durations, failures, skip reasons, and known coverage gaps. An +unfinished report remains marked incomplete. Exit status is nonzero for failed +cases, failed prerequisites, an all-skipped selection, or interrupted work. +With `--require-all`, any skip also fails. Without it, a run with optional +capability skips is explicitly reported as partial rather than fully passed. + +Use `--keep-work` to retain isolated homes and artifacts; the report identifies +each case's directory. Those directories can contain synthetic test passwords, +encrypted test vaults, browser profiles, and capability tokens. Treat retained +work as private and do not upload entire directories. The normal runner removes +them after closing its daemons and fixtures. Cleanup failures fail the case +and retain its directory for diagnosis. + +## Coverage + +Run `--list` for the executable inventory; the case files are the source of truth. + +| Area | Behaviors exercised | +| --- | --- | +| CLI | Every advertised command's help route; version and invalid input; inline/file/stdin/REPL snippets; JSON formatting; configuration persistence and key masking; session management; read-only auth/skills/doctor checks; noninteractive vault and cookie validation failures. | +| Browser | Redirects/history/reload; delayed subresources; forms and human-shaped input; Unicode and rich text; frames and shadow DOM; tabs/popups/dialogs; session/profile separation; cookies and local storage across restart; optional stealth isolation and local CDP-failure fallback. | +| Observation and actions | Snapshot refs, scoping, diffing, and caps; control/media/overlay inspection; console events and history; UI batch consent, ambiguity, asserted reads, and stop-on-failure behavior. | +| Artifacts | PNG/JPEG bytes, dimensions, and annotations; artifact paths; upload boundaries; per-run download approval and exact downloaded bytes; MP4/WebM recording, decoded frames, restart, and finalization. | +| Site capabilities | Same-origin requests and assets; local WebAgents and WebMCP fixtures; local checkbox/grid CAPTCHA flows and evidence on errors. | +| Security | Restricted host globals, constructors, private Playwright channels and routing; internal navigation; read/write path and symlink boundaries; metadata/private/loopback/blocked-host policy with positive controls; concurrent download approval isolation. | +| Credentials | Encrypted vault storage and metadata-only output; trusted fill/submit; result/console/error redaction; origin/host/never scopes; ambiguous and missing forms/accounts; staged generation, commit/discard, rotation, and cross-session recovery. | +| Protocols | Real MCP stdio initialization, discovery, validation, browser calls, recovery, and download denial; an `exec` agent turn through a local OpenAI-compatible mock, including tool-result replay and HTTP/malformed-response failures. | +| Lifecycle | Synchronous and asynchronous snippet timeouts, worker restart and cookie persistence, cyclic/oversized output handling, and token-gated live-view streaming with watch-only enforcement and safe detach. | + +Assertions check resulting DOM state, fixture requests, file bytes, protocol +messages, and exit statuses rather than treating `ok: true` as sufficient. +An absent browser is a failed prerequisite, not a successful suite with all +browser tests silently skipped. Optional capabilities have explicit skip reasons. + +The existing `Node tests` CI job runs the strict suite against the newly built +CLI and uploads its JSON report as `binary-e2e-report`, including on failures. + +## Isolation and limits + +Every case gets a fresh `HOME`, `USERPROFILE`, XDG directories, +`BETTERWRIGHT_HOME`, temporary directory, profiles, and vault. The environment +is allowlisted: inherited model/provider credentials, preload hooks, and default +profile selections are not passed through. The suite uses local HTTP fixtures +and synthetic credentials. Metadata-policy probes use a local synthetic proxy +that never forwards to the requested destination. + +The runner is not an operating-system sandbox for an untrusted executable. +Only point it at a binary you trust: an arbitrary executable can ignore the +isolated environment. The managed browser installation and encoder are shared +read-only prerequisites from the user's normal environment. + +These integrations remain outside this automatic local suite: + +- Real cloud browsers, provider billing, OAuth/MFA and live model behavior. +- Third-party websites, external CAPTCHA services, and hostile-network/DNS + rebinding environments. +- Installer/update downloads, local inference installations and GPU execution, + and interactive onboarding. +- Electron host attachment, OS clipboard/keystroke injection, native extraction + of personal browser cookies, and master-password entry through a real terminal. +- Exhaustive SDK/export/type contracts, configurable SDK-only resource limits, + and all operating systems, browser versions, and hardware combinations. + +Those gaps are not counted as passing tests. Real-browser fixture tests do not +establish success on live websites, and a mock-model protocol test does not +measure an actual model's ability to complete a task. + +## Extend the suite + +Add a case with a stable ID, a meaningful outcome assertion, and local fixtures +to the appropriate `tests/e2e/*-cases.ts` module. Use the shared context for +commands, browser runs, HTTP fixtures, cleanup, and registered redaction values. +Do not import the target's internals or alter another case's home. Register +synthetic secrets with `context.redact` before using them so failure messages +cannot inadvertently print them. + +`tests/node/e2e-harness.test.ts` separately exercises the runner against a tiny +fake executable, checking argument passing, isolation, deadlines, output bounds, +JSON/exit-code validation, cleanup, redaction, selection, and skip semantics. +Run it after changing harness behavior: + +```bash +bun test --timeout 120000 tests/node/e2e-harness.test.ts +bun run build:harness +bun run lint +``` diff --git a/package.json b/package.json index fa1c3b97..f6a2ad47 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "check:build": "bun run build && bun scripts/check-build.ts", "test": "bun run build:harness && bun scripts/run-unit-tests.ts --all", "test:unit": "bun run build:harness && bun scripts/run-unit-tests.ts", + "test:e2e": "bun tests/e2e/run.ts", "test:types": "tsc --noEmit -p tests/types/tsconfig.json", "test:electron": "bun run build && electron tests/electron/reliability.mjs && electron tests/electron/detach-recovery.mjs && electron tests/electron/cookie-sync-takeover.mjs", "test:electron:packaged": "bun run build && bun tests/electron/package.mjs && electron .tmp/electron-e2e/app.asar && electron .tmp/electron-e2e/detach.asar && electron .tmp/electron-e2e/cookie-sync.asar", diff --git a/src/mcp-server.ts b/src/mcp-server.ts index 52515b92..d3fb3277 100644 --- a/src/mcp-server.ts +++ b/src/mcp-server.ts @@ -711,11 +711,24 @@ export async function runMcpServer(env = process.env, options: any = {}) { server.setRequestHandler(CallToolRequestSchema, handlers.callTool); const transport = new StdioServerTransport(); - await server.connect(transport); - // Serve until the client disconnects (stdin closes), then release the - // browser. server.onclose is the SDK's protocol-level close callback. - await new Promise((resolve) => { - server.onclose = resolve; + let disconnect: () => void; + const disconnected = new Promise((resolve) => { + disconnect = resolve; }); - await browser.close(); + server.onclose = disconnect; + process.stdin.once("end", disconnect); + process.stdin.once("close", disconnect); + try { + await server.connect(transport); + if (process.stdin.readableEnded || process.stdin.destroyed) disconnect(); + await disconnected; + } finally { + process.stdin.off("end", disconnect); + process.stdin.off("close", disconnect); + try { + await server.close(); + } finally { + await browser.close(); + } + } } diff --git a/src/navigation-defaults.ts b/src/navigation-defaults.ts index d1aa8544..12d85531 100644 --- a/src/navigation-defaults.ts +++ b/src/navigation-defaults.ts @@ -1,4 +1,4 @@ -import { isRecord } from "./untrusted-value.js"; +import { isRecord, isString, untrustedField } from "./untrusted-value.js"; // Agent navigation needs a usable document, not every third-party subresource. // Keep explicit Playwright options (including invalid ones) for Playwright to validate. @@ -14,3 +14,29 @@ export function applyNavigationDefaults(kind: string, method: string, args: any[ args[0] = navigationOptions(args[0]); } } + +export async function navigateHistory(page: any, method: "goBack" | "goForward", options: any, defaultTimeoutMs: number) { + const waitUntil = untrustedField(options, "waitUntil"); + if (!isRecord(options) || !isString(waitUntil) || !["domcontentloaded", "load"].includes(waitUntil)) { + return page[method](options); + } + const started = performance.now(); + const response = await page[method]({ ...options, waitUntil: "commit" }); + const timeout = Number(untrustedField(options, "timeout") ?? defaultTimeoutMs); + const remaining = timeout === 0 ? 0 : Math.max(1, timeout - (performance.now() - started)); + const ready = await page.waitForFunction( + (waitUntil) => { + const navigation = performance.getEntriesByType("navigation")[0]; + if (navigation instanceof PerformanceNavigationTiming) { + return waitUntil === "load" + ? navigation.loadEventEnd > 0 + : navigation.domContentLoadedEventEnd > 0; + } + return document.readyState === "complete"; + }, + waitUntil, + { timeout: remaining }, + ); + await ready.dispose(); + return response; +} diff --git a/src/worker.ts b/src/worker.ts index 1aca3719..dcc5d211 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -108,7 +108,7 @@ import { import { buildLaunchIdentityPlan, resolveGeoIdentity } from "./launch-identity.js"; import { createLiveViewServer } from "./live-view.js"; import { liveViewHtml, liveViewLoginHtml } from "./live-view-html.js"; -import { applyNavigationDefaults, navigationOptions } from "./navigation-defaults.js"; +import { applyNavigationDefaults, navigateHistory, navigationOptions } from "./navigation-defaults.js"; import { createSnippetPageEvents, isSnippetPageEventMethod, @@ -694,7 +694,7 @@ async function loadPlaywrightDriver() { const pathMod = await import("node:path"); return import(pathToFileURL(pathMod.join(override, "lib", "index.js")).href); } - return import("playwright-core"); + return stealthActive ? import("patchright-core") : import("playwright-core"); } function fingerprintSeedForProfile(profileDir) { const seedFile = path.join(profileDir, ".betterwright-fingerprint-seed"); @@ -2054,32 +2054,43 @@ async function clearTypedField(page, target) { } } -async function insertTypedText(page, target, text, before) { - const value = String(text); - if (!value) return; - if (target && isCallable(untrustedField(target, "focus"))) { - await target.focus().catch(() => {}); - } +async function moveTypedFieldCaretToEnd(page, target) { if (target && isCallable(untrustedField(target, "evaluate"))) { - await target.evaluate((element) => { + const positioned = await target.evaluate((element) => { if ( element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement ) { + if ( + element instanceof HTMLInputElement && + !["text", "search", "url", "tel", "password"].includes(element.type) + ) return false; const end = element.value.length; element.setSelectionRange(end, end); - return; + return true; } - if (!element.isContentEditable) return; + if (!element.isContentEditable) return false; const selection = element.ownerDocument.defaultView?.getSelection(); - if (!selection) return; + if (!selection) return false; const range = element.ownerDocument.createRange(); range.selectNodeContents(element); range.collapse(false); selection.removeAllRanges(); selection.addRange(range); + return true; }); + if (positioned) return; + } + await page.keyboard.press("End"); +} + +async function insertTypedText(page, target, text, before) { + const value = String(text); + if (!value) return; + if (target && isCallable(untrustedField(target, "focus"))) { + await target.focus().catch(() => {}); } + await moveTypedFieldCaretToEnd(page, target); await page.keyboard.insertText(value); const afterInsert = await readTypedFieldText(target); if (afterInsert != null && typedTextLanded(value, before, afterInsert)) return; @@ -3339,6 +3350,8 @@ function wrap(value, realm) { let result; if (kind === "Page" && property === "close") { result = stopPageRecording(value).then(() => member.apply(value, prepared)); + } else if (kind === "Page" && (property === "goBack" || property === "goForward")) { + result = navigateHistory(value, property, prepared[0], DEFAULT_NAVIGATION_TIMEOUT_MS); } else if ( useSetContentCompatibility && ["Page", "Frame"].includes(kind) && @@ -4152,6 +4165,7 @@ function buildSandbox(session, consoleMessages, execution) { const clickedTarget = await humanClickTarget(page, session, target, options); const clear = options?.clear !== false; if (clear) await clearTypedField(page, clickedTarget); + else await moveTypedFieldCaretToEnd(page, clickedTarget); const expected = String(text); const before = await readTypedFieldText(clickedTarget); await typeText(page.keyboard, expected, options); diff --git a/tests/e2e/browser-cases.ts b/tests/e2e/browser-cases.ts new file mode 100644 index 00000000..a0e06613 --- /dev/null +++ b/tests/e2e/browser-cases.ts @@ -0,0 +1,829 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { + animationHtml, + checkboxHtml, + documentHtml, + formHtml, + gridHtml, + serveBrowserFixture, +} from "./browser-fixtures.js"; +import type { E2ECase, E2EContext } from "./types.js"; + +function browserCase(id: string, title: string, run: E2ECase["run"]): E2ECase { + return { id: `browser.${id}`, group: "browser", title, requiresBrowser: true, run }; +} + +function result(envelope: any): any { + assert.equal(envelope.ok, true, envelope.error); + return envelope.result; +} + +function rejected(envelope: any, pattern: RegExp): void { + assert.equal(envelope.ok, false); + assert.match(envelope.error, pattern); +} + +async function png(file: string): Promise<{ width: number; height: number }> { + const bytes = await readFile(file); + assert.deepEqual([...bytes.subarray(0, 8)], [137, 80, 78, 71, 13, 10, 26, 10]); + assert.equal(bytes.subarray(12, 16).toString(), "IHDR"); + const dimensions = { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) }; + assert.ok(dimensions.width > 0 && dimensions.height > 0); + return dimensions; +} + +function encoder(ctx: E2EContext, codecs: string[]): string { + const executable = process.env.BETTERWRIGHT_FFMPEG_PATH || "ffmpeg"; + const probe = spawnSync(executable, ["-encoders"], { encoding: "utf8", timeout: 5_000 }); + if (probe.error || probe.status !== 0) ctx.skip("FFmpeg is unavailable; recording requires an installed encoder"); + for (const codec of codecs) { + if (!new RegExp(`\\b${codec}\\b`).test(probe.stdout)) ctx.skip(`FFmpeg lacks required ${codec} encoder`); + } + return executable; +} + +function decodeVideo(executable: string, file: string): string[] { + const decoded = spawnSync(executable, ["-v", "error", "-i", file, "-f", "framemd5", "-"], { + encoding: "utf8", timeout: 15_000, maxBuffer: 4 * 1024 * 1024, + }); + assert.equal(decoded.status, 0, decoded.stderr); + const hashes = decoded.stdout.split("\n").filter(line => /^\d+,/.test(line)).map(line => line.split(",").at(-1)?.trim()); + assert.ok(hashes.length > 1, "recording must decode to multiple frames"); + return hashes.filter((hash): hash is string => hash !== undefined); +} + +export const cases: E2ECase[] = [ + browserCase("navigation-history", "Navigation follows redirects, reloads, and traverses history", async ctx => { + let visits = 0; + const server = await serveBrowserFixture(ctx, { + "/": documentHtml('

First page

Continue', "First"), + "/redirect": (_request, response) => { response.writeHead(302, { location: "/second" }).end(); }, + "/second": (_request, response) => { + visits++; + response.setHeader("content-type", "text/html"); + response.end(documentHtml(`

Second page

${visits}

`, "Second")); + }, + }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}, {timeout:5000}); + await page.getByRole('link', {name:'Continue'}).click({timeout:5000}); + await page.waitForURL('**/second', {timeout:5000}); + const second = {title: await page.title(), url: page.url()}; + await page.goBack({timeout:5000}); + const back = await page.title(); + await page.goForward({timeout:5000}); + await page.reload({timeout:5000}); + return {second, back, current: await page.title(), visits: Number(await page.locator('#visits').innerText())}; + `)); + assert.deepEqual(actual.second, { title: "Second", url: `${server.origin}/second` }); + assert.equal(actual.back, "First"); + assert.equal(actual.current, "Second"); + assert.ok(actual.visits >= 2); + }), + + browserCase("dom-ready", "Default navigation does not wait for stalled subresources", async ctx => { + const server = await serveBrowserFixture(ctx, { + "/": documentHtml('

Ready before image load

Pending fixture'), + "/pending.png": (_request, _response) => {}, + }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}, {timeout: 3000}); + return {heading: await page.locator('h1').innerText(), loaded: await page.evaluate(() => document.readyState)}; + `)); + assert.equal(actual.heading, "Ready before image load"); + assert.equal(actual.loaded, "interactive"); + }), + + browserCase("forms", "Form locators update live values and submit once", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": formHtml }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.getByLabel('Display name').fill('Ada'); + await page.getByLabel('Plan', {exact:true}).selectOption('pro'); + await page.getByLabel('Accept terms').check(); + await page.getByLabel('Notes').fill('Updated notes'); + await page.getByRole('button', {name:'Create account'}).click(); + return {status: await page.getByRole('status').innerText(), notes: await page.getByLabel('Notes').inputValue(), submissions: await page.evaluate(() => window.submissions)}; + `)); + assert.deepEqual(actual, { status: "Created Ada on pro; terms=true", notes: "Updated notes", submissions: 1 }); + }), + + browserCase("human-unicode", "Human helpers preserve Unicode, append text, click, and scroll", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` +

+
+ + `) }); + const text = "Café 日本語 🧪 مرحبا"; + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await human.type('#message', ${JSON.stringify(text)}, {minDelay:0, maxDelay:0}); + await human.type(page.getByLabel('Message'), ' + done', {clear:false, minDelay:0, maxDelay:0}); + await human.click(page.getByRole('button', {name:'Save'})); + await human.scroll(600, {steps:4}); + await page.waitForFunction(() => window.scrollY > 0); + return {value: await page.getByLabel('Message').inputValue(), status: await page.getByRole('status').innerText(), scrolled: await page.evaluate(() => window.scrollY > 0)}; + `)); + assert.deepEqual(actual, { value: `${text} + done`, status: `${text} + done`, scrolled: true }); + }), + + browserCase("rich-text", "Human typing recovers when a rich editor swallows key events", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` +
Draft
+ + `) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await human.type('#editor', ' + café', {clear:false, minDelay:0, maxDelay:0}); + return page.getByRole('textbox', {name:'Editor'}).innerText(); + `)); + assert.equal(actual, "Draft + café"); + }), + + browserCase("human-append-fields", "Normal human typing appends at the end of inputs and multiline editors", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` + + + +
First line
Second line
+ `) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await human.type('#single', ' appended', {clear:false,minDelay:10,maxDelay:10}); + await human.type('#multiline', ' appended', {clear:false,minDelay:10,maxDelay:10}); + await human.type('#email', '.test', {clear:false,minDelay:10,maxDelay:10}); + await human.type('#editable', ' appended', {clear:false,minDelay:10,maxDelay:10}); + return {single:await page.locator('#single').inputValue(), multiline:await page.locator('#multiline').inputValue(), email:await page.locator('#email').inputValue(), editable:await page.locator('#editable').innerText()}; + `)); + assert.deepEqual(actual, { + single: "Original text appended", + multiline: "First line\nSecond line appended", + email: "reader@example.test", + editable: "First line\nSecond line appended", + }); + }), + + browserCase("frames", "Frame locators and frame-scoped batches reach cross-origin content", async ctx => { + const server = await serveBrowserFixture(ctx, { + "/": (_request, response) => { + response.setHeader("content-type", "text/html"); + response.end(documentHtml(``)); + }, + "/frame": documentHtml(''), + }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const frame = page.frameLocator('iframe'); + await frame.getByLabel('Region').selectOption({label:'EU'}); + await frame.getByRole('button', {name:'Save region'}).click(); + const batch = await controls.batch([{id:'region', action:'read', target:{frameName:'billing', label:'Region'}}]); + return {batch, tree: await snapshot({interactive:true}), saved: await frame.getByRole('button').innerText()}; + `)); + assert.equal(actual.batch.results.region.value, "EU"); + assert.equal(actual.saved, "Saved"); + assert.match(actual.tree, /Saved/); + assert.match(actual.tree, /ref=f\d+e\d+/); + }), + + browserCase("shadow-dom", "Locators and semantic controls reach an open shadow root", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` +
+ `) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.getByLabel('Shadow name').fill('Root'); + await page.getByRole('button', {name:'Save shadow'}).click(); + return {status: await page.getByRole('status').innerText(), batch: await controls.batch([{id:'name',action:'read',target:{label:'Shadow name'},value:'Root'}])}; + `)); + assert.equal(actual.status, "Saved Root"); + assert.equal(actual.batch.results.name.value, "Root"); + }), + + browserCase("tabs", "Tab handles switch the active page and close only the selected tab", async ctx => { + const server = await serveBrowserFixture(ctx, { + "/": documentHtml("

First

", "First"), + "/second": documentHtml("

Second

", "Second"), + }); + const envelope = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const first = page; + const second = await openPage(${JSON.stringify(`${server.origin}/second`)}); + const count = pages.length; + await usePage(second); + const active = await page.title(); + await usePage(first); + await closePage(second); + return {count, active, remaining: pages.length, title: await page.title()}; + `); + assert.deepEqual(result(envelope), { count: 2, active: "Second", remaining: 1, title: "First" }); + assert.equal(envelope.pages.filter((entry: any) => entry.active).length, 1); + assert.equal(envelope.pages.find((entry: any) => entry.active).title, "First"); + }), + + browserCase("popup", "Target-blank popups are adopted into the session", async ctx => { + const server = await serveBrowserFixture(ctx, { + "/": documentHtml('Open popup', "Parent"), + "/popup": documentHtml("

Popup ready

", "Popup"), + }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const waiting = page.waitForEvent('popup'); + await page.getByRole('link', {name:'Open popup'}).click(); + const popup = await waiting; + await popup.getByRole('heading', {name:'Popup ready'}).waitFor(); + await usePage(popup); + const title = await page.title(); + const count = pages.length; + await closePage(); + return {title, count, remaining: pages.length, parent: await page.title()}; + `)); + assert.deepEqual(actual, { title: "Popup", count: 2, remaining: 1, parent: "Parent" }); + }), + + browserCase("dialogs", "Prepared dialog responses accept prompts and dismiss confirmations", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` + + + + `) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + dialogs.acceptNext('Zoë'); + await page.locator('#prompt').click(); + const prompt = await page.locator('output').innerText(); + dialogs.dismissNext(); + await page.locator('#confirm').click(); + const confirm = await page.locator('output').innerText(); + dialogs.acceptNext(); + await page.locator('#alert').click(); + return {prompt, confirm, alert: await page.locator('output').innerText()}; + `)); + assert.deepEqual(actual, { prompt: "Zoë", confirm: "false", alert: "Alert closed" }); + }), + + browserCase("persistent-session", "State, DOM, and session storage persist across binary invocations", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": formHtml }); + result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + state.counter = 41; + await page.getByLabel('Display name').fill('Unsubmitted draft'); + await page.evaluate(() => sessionStorage.setItem('draft', 'saved')); + return page.title(); + `, { session: "persistent" })); + assert.deepEqual(result(await ctx.run(`return { + counter: ++state.counter, draft: await page.getByLabel('Display name').inputValue(), + storage: await page.evaluate(() => sessionStorage.getItem('draft')), pages: pages.length + }`, { session: "persistent" })), { counter: 42, draft: "Unsubmitted draft", storage: "saved", pages: 1 }); + }), + + browserCase("session-isolation", "Sessions isolate tabs and state while sharing profile cookies", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml("

Shared identity

") }); + result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); state.owner = 'alpha'; + await page.evaluate(() => { document.cookie = 'shared=alpha; Path=/'; sessionStorage.setItem('owner', 'alpha'); }); + return page.url(); + `, { session: "alpha" })); + const beta = result(await ctx.run(` + const initialUrl = page.url(); await page.goto(${JSON.stringify(server.origin)}); + return {initialUrl, state: state.owner ?? null, cookie: await page.evaluate(() => document.cookie), storage: await page.evaluate(() => sessionStorage.getItem('owner'))}; + `, { session: "beta" })); + assert.equal(beta.initialUrl, "about:blank"); + assert.equal(beta.state, null); + assert.equal(beta.storage, null); + assert.match(beta.cookie, /shared=alpha/); + assert.equal(result(await ctx.run("return state.owner", { session: "alpha" })), "alpha"); + }), + + browserCase("profiles", "Named profiles keep separate cookie and local-storage identities", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml("

Profile

") }); + for (const profile of ["alice", "bob"]) { + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const before = await page.evaluate(() => ({cookie:document.cookie, storage:localStorage.getItem('owner')})); + await page.evaluate(owner => {document.cookie='owner='+owner+'; Path=/';localStorage.setItem('owner',owner)}, ${JSON.stringify(profile)}); + return before; + `, { profile })); + assert.deepEqual(actual, { cookie: "", storage: null }); + } + for (const profile of ["alice", "bob"]) { + assert.deepEqual(result(await ctx.run("return page.evaluate(() => ({cookie:document.cookie, storage:localStorage.getItem('owner')}))", { profile })), { + cookie: `owner=${profile}`, storage: profile, + }); + } + }), + + browserCase("cookie-restart", "Persistent cookies and local storage survive a clean daemon restart", async ctx => { + let observedCookie = ""; + const server = await serveBrowserFixture(ctx, { + "/": documentHtml("

Persisted identity

"), + "/echo": (request, response) => { + observedCookie = request.headers.cookie || ""; + response.setHeader("content-type", "text/plain"); + response.end(observedCookie); + }, + }); + result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.evaluate(() => {document.cookie='persistent=survives; Max-Age=3600; Path=/';localStorage.setItem('marker','survives')}); + state.volatile = true; return page.url(); + `, { profile: "restart" })); + const closed = await ctx.command(["close", "--all"]); + assert.equal(closed.code, 0, closed.stderr); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return {volatile: state.volatile ?? null, storage: await page.evaluate(() => localStorage.getItem('marker')), echo: await page.evaluate(() => fetch('/echo').then(response => response.text()))}; + `, { profile: "restart" })); + assert.equal(actual.volatile, null); + assert.equal(actual.storage, "survives"); + assert.match(actual.echo, /persistent=survives/); + assert.match(observedCookie, /persistent=survives/); + }), + + browserCase("snapshot-refs", "Snapshot refs are actionable and selector scopes exclude unrelated content", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('
Details
') }); + const initial = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return snapshot({interactive:true});`)); + const ref = /button "Save record"[^\n]*\[ref=(e\d+)\]/.exec(initial)?.[1]; + assert.ok(ref, initial); + const actual = result(await ctx.run(` + await page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click(); + return {scoped: await snapshot({selector:'#main', urls:true}), button: await page.getByRole('button').innerText()}; + `)); + assert.equal(actual.button, "Saved"); + assert.match(actual.scoped, /Saved/); + assert.match(actual.scoped, /\/url: \/target/); + assert.doesNotMatch(actual.scoped, /Outside sentinel/); + }), + + browserCase("snapshot-diff", "Snapshot diffs distinguish unchanged and updated page state", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('

Pending

') }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const first = await snapshot({diff:true}); + const unchanged = await snapshot({diff:true}); + await page.getByRole('button', {name:'Finish'}).click(); + const changed = await snapshot({diff:true}); + return {first, unchanged, changed}; + `)); + assert.match(actual.first, /Pending/); + assert.match(actual.unchanged, /no changes since previous snapshot/); + assert.match(actual.changed, /\+.*Complete/); + assert.match(actual.changed, /-.*Pending/); + }), + + browserCase("snapshot-limits", "Oversized snapshots provide diagnostics and scoped retries recover", async ctx => { + const items = Array.from({ length: 350 }, (_, index) => ``).join(""); + const server = await serveBrowserFixture(ctx, { "/": documentHtml(`
${items}
`) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return {smallLimit: await snapshot({maxChars:1000}), capped: await snapshot({maxChars:100000}), scoped: await snapshot({selector:'#small', maxChars:1000})}; + `)); + assert.match(actual.smallLimit, /over the 1000 limit/i); + assert.match(actual.smallLimit, /selector|ref/); + assert.match(actual.capped, /20000|20,000/); + assert.match(actual.scoped, /Only scoped control/); + assert.doesNotMatch(actual.scoped, /Record 0/); + }), + + browserCase("controls-inspect", "Control inspectors report checked, disabled, selected, and redacted values", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` + + + + `) }); + const actual = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return {controls:await controls.inspect(), tree:await snapshot({interactive:true})};`)); + const controls = actual.controls.frames.flatMap((frame: any) => frame.controls); + assert.ok(controls.some((control: any) => control.value === "Ada")); + assert.ok(controls.some((control: any) => control.checked === true)); + assert.ok(controls.some((control: any) => control.value === "Pro")); + assert.ok(controls.some((control: any) => control.disabled === true && control.value === "locked")); + assert.ok(controls.some((control: any) => control.type === "password" && control.value === "[redacted]")); + assert.doesNotMatch(JSON.stringify(actual), /synthetic-password/); + }), + + browserCase("overlays", "Overlay dismissal rejects optional cookies and preserves task dialogs", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` +
+

Cookie consent: We use cookies for analytics and advertising

+ + +
+

Delete this record?

+ `) }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); const dismissed = await overlays.dismiss(); + return {dismissed, consent: await page.evaluate(() => window.consent), taskDialog: await page.getByRole('dialog', {name:'Delete record'}).isVisible()}; + `)); + assert.equal(actual.consent, "essential"); + assert.equal(actual.taskDialog, true); + assert.ok(actual.dismissed.dismissed.some((entry: any) => entry.kind === "cookie")); + }), + + browserCase("media-inspect", "Media inspection identifies paused audio and video without external resources", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('

Local media

') }); + const actual = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return media.inspect();`)); + const entries = actual.frames.flatMap((frame: any) => frame.media); + for (const kind of ["video", "audio"]) { + const entry = entries.find((item: any) => item.kind === kind); + assert.ok(entry); + assert.equal(entry.title, `Demo ${kind}`); + assert.equal(entry.paused, true); + assert.equal(entry.currentTime, 0); + } + }), + + browserCase("console-events", "Page console and errors are collected once without leaking listeners", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('') }); + const first = await ctx.run(` + state.events = []; + page.on('console', message => state.events.push({type:message.type(), text:message.text()})); + page.once('pageerror', error => state.events.push({type:'pageerror', text:error.message})); + await page.goto(${JSON.stringify(server.origin)}); + console.log('snippet-only'); + return state.events; + `); + const messages = result(first); + assert.ok(messages.some((entry: any) => entry.type === "warning" && entry.text === "fixture-warning")); + assert.ok(messages.some((entry: any) => entry.type === "pageerror" && entry.text === "fixture-boom")); + assert.match(JSON.stringify(first.console), /snippet-only/); + const next = result(await ctx.run("await page.evaluate(() => console.log('later-page-message')); return state.events;")); + assert.deepEqual(next, messages); + }), + + browserCase("console-history", "Console history is bounded and scoped to the current navigation", async ctx => { + const server = await serveBrowserFixture(ctx, { + "/": documentHtml(''), + "/next": documentHtml(''), + }); + result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); await page.evaluate(() => {for(let i=0;i<220;i++)console.debug('noise-'+i)}); return true;`)); + const history = result(await ctx.run("return {count:(await page.consoleMessages()).length, errors:(await page.pageErrors()).map(error => error.message)};")); + assert.ok(history.count > 0 && history.count <= 200); + assert.ok(history.errors.includes("old-error")); + const fresh = result(await ctx.run(`await page.goto(${JSON.stringify(`${server.origin}/next`)}); return { + messages:(await page.consoleMessages({filter:'since-navigation'})).map(message => message.text()), + errors:(await page.pageErrors({filter:'since-navigation'})).map(error => error.message) + };`)); + assert.ok(fresh.messages.includes("new-navigation")); + assert.ok(!fresh.messages.some((text: string) => /noise-|old-warning/.test(text))); + assert.deepEqual(fresh.errors, []); + }), + + browserCase("screenshots", "PNG and JPEG screenshots contain real image bytes and annotation metadata", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('

Screenshot proof

') }); + const envelope = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return {plain:await screenshot({kind:'proof',name:'proof'}), annotated:await screenshot({name:'annotated.png',annotate:true}), jpeg:await screenshot({name:'full.jpg',type:'jpeg',fullPage:true,quality:70})}; + `); + const actual = result(envelope); + const dimensions = await png(actual.plain.path); + assert.deepEqual(await png(actual.annotated.path), dimensions); + assert.ok(actual.annotated.annotations >= 1); + assert.equal(actual.plain.media, `MEDIA:${actual.plain.path}`); + const jpeg = await readFile(actual.jpeg.path); + assert.deepEqual([...jpeg.subarray(0, 3)], [255, 216, 255]); + assert.deepEqual([...jpeg.subarray(-2)], [255, 217]); + assert.equal(envelope.artifacts.length, 3); + assert.ok(envelope.artifacts.every((artifact: any) => artifact.path.startsWith(`${ctx.home}${path.sep}`))); + }), + + browserCase("artifact-paths", "Artifact names cannot escape the managed artifact directory", async ctx => { + const escaped = path.join(ctx.workDir, "escaped.png"); + const envelope = await ctx.run(`await page.setContent('

Bounded output

'); return screenshot({name:${JSON.stringify(escaped)}});`); + if (envelope.ok) { + const artifact = result(envelope); + assert.notEqual(artifact.path, escaped); + assert.ok(artifact.path.startsWith(path.join(ctx.home, "artifacts") + path.sep)); + await png(artifact.path); + } else { + rejected(envelope, /artifact|filename|path|name/i); + } + await assert.rejects(stat(escaped), { code: "ENOENT" }); + }), + + browserCase("uploads", "Artifact uploads preserve bytes and reject host files outside the artifact root", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('') }); + const allowed = path.join(ctx.home, "artifacts", "upload.txt"); + const denied = path.join(ctx.workDir, "outside.txt"); + await mkdir(path.dirname(allowed), { recursive: true }); + await writeFile(allowed, "uploaded café 日本語"); + await writeFile(denied, "outside fixture"); + rejected(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); await page.getByLabel('Attachment').setInputFiles(${JSON.stringify(denied)});`), /artifact directory/i); + const actual = result(await ctx.run(` + await page.getByLabel('Attachment').setInputFiles(${JSON.stringify(allowed)}); + return page.getByLabel('Attachment').evaluate(async element => ({name:element.files[0].name, text:await element.files[0].text(), count:element.files.length})); + `)); + assert.deepEqual(actual, { name: "upload.txt", text: "uploaded café 日本語", count: 1 }); + }), + + browserCase("downloads", "Downloads require per-run approval and saved artifacts retain exact bytes", async ctx => { + const body = "downloaded café 日本語\n"; + const server = await serveBrowserFixture(ctx, { + "/": documentHtml('Download report'), + "/report.txt": (_request, response) => { + response.writeHead(200, { "content-type": "text/plain; charset=utf-8", "content-disposition": 'attachment; filename="report.txt"' }); + response.end(body); + }, + }); + const code = ` + await page.goto(${JSON.stringify(server.origin)}); + const waiting = page.waitForEvent('download'); + await page.getByRole('link', {name:'Download report'}).click(); + const download = await waiting; + return {name:download.suggestedFilename(), failure:await download.failure()}; + `; + const blocked = await ctx.run(code); + result(blocked); + assert.equal((blocked.artifacts || []).filter((artifact: any) => artifact.kind === "download").length, 0); + const approved = await ctx.run(code, { args: ["--approve-downloads"] }); + assert.deepEqual(result(approved), { name: "report.txt", failure: null }); + const files = (approved.artifacts || []).filter((artifact: any) => artifact.kind === "download"); + assert.equal(files.length, 1); + assert.equal(await readFile(files[0].path, "utf8"), body); + const revoked = await ctx.run(code); + result(revoked); + assert.equal((revoked.artifacts || []).filter((artifact: any) => artifact.kind === "download").length, 0); + }), + + browserCase("ui-batch", "A semantic UI transaction verifies its exact live result", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": formHtml }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return controls.batch({operations:[ + {id:'name',action:'fill',target:{label:'Display name'},value:'Ada'}, + {id:'plan',action:'select',target:{label:'Plan',exact:true},value:'pro'}, + {id:'terms',action:'check',target:{label:'Accept terms'}}, + {id:'submit',action:'click',target:{role:'button',name:'Create account',exact:true}}, + {id:'verify',action:'read',target:{role:'status'},value:'Created Ada on pro; terms=true'} + ],allowWrites:true,minIntervalMs:0}); + `)); + assert.equal(actual.protocol, "ui-batch/1"); + assert.equal(actual.pageUpdated, true); + assert.equal(actual.results.verify.text, "Created Ada on pro; terms=true"); + assert.equal(actual.results.terms.checked, true); + assert.equal(result(await ctx.run("return page.evaluate(() => window.submissions)")), 1); + }), + + browserCase("ui-batch-guards", "UI batches validate write authorization and ambiguity before acting", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml('

Ready

') }); + result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return true;`)); + const operations = "[{id:'save',action:'click',target:{role:'button',name:'Save',exact:true}},{id:'verify',action:'read',target:{role:'status'},value:'Ready'}]"; + rejected(await ctx.run(`return controls.batch(${operations})`), /allowWrites:true/); + rejected(await ctx.run(`return controls.batch(${operations}, {allowWrites:true})`), /matched 2 elements/); + rejected(await ctx.run(`return controls.batch([{id:'save',action:'click',target:{role:'button',name:'Save',nth:0}},{id:'verify',action:'read',target:{role:'status'}}], {allowWrites:true})`), /non-empty expected value/); + assert.equal(result(await ctx.run("return page.evaluate(() => window.clicks)")), 0); + }), + + browserCase("ui-batch-stop", "A failed asserted read does not replay or continue completed writes", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` + +

Waiting

Unrelated success

+ + `) }); + const envelope = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return controls.batch({operations:[ + {id:'submit',action:'click',target:{css:'#submit'}}, + {id:'wrong',action:'read',target:{css:'#status'},value:'Unrelated success'}, + {id:'later',action:'click',target:{css:'#later'}}, + {id:'verify',action:'read',target:{css:'#status'},value:'Order 42 confirmed'} + ],allowWrites:true,minIntervalMs:0}); + `); + rejected(envelope, /wrong|expected|Unrelated success/i); + const actual = result(await ctx.run("return {status:await page.locator('#status').innerText(), counts:await page.evaluate(() => ({submissions:window.submissions,later:window.later}))}")); + assert.deepEqual(actual, { status: "Order 42 confirmed", counts: { submissions: 1, later: 0 } }); + }), + + browserCase("site-tools", "Site tools discover assets and make cookie-bearing same-origin JSON requests", async ctx => { + let requestBody: any; + let requestCookie = ""; + const server = await serveBrowserFixture(ctx, { + "/": documentHtml('

Site tools

'), + "/app.js": (_request, response) => { + response.setHeader("content-type", "application/javascript"); + response.end('window.fixtureMarker = "asset-needle";'); + }, + "/api": (request, response) => { + requestCookie = request.headers.cookie || ""; + let body = ""; + request.setEncoding("utf8"); + request.on("data", chunk => { body += chunk; }); + request.on("end", () => { + requestBody = JSON.parse(body); + response.writeHead(200, { "content-type": "application/json", "set-cookie": "returned=ready; Path=/; SameSite=Lax" }); + response.end(JSON.stringify({ received: requestBody, accepted: true })); + }); + }, + }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.evaluate(() => {document.cookie='client=present; Path=/'}); + return {assets:await site.assets(), requests:await site.requests({urlIncludes:'app.js'}), source:await site.read('/app.js',{find:'asset-needle',contextChars:40,maxMatches:1}), response:await site.request('/api',{method:'POST',json:{count:3},response:'json'}), cookie:await page.evaluate(() => document.cookie)}; + `)); + assert.match(JSON.stringify(actual.assets), /app\.js/); + assert.match(JSON.stringify(actual.requests), /app\.js/); + assert.match(JSON.stringify(actual.source), /asset-needle/); + assert.equal(actual.response.status, 200); + assert.deepEqual(actual.response.json, { received: { count: 3 }, accepted: true }); + assert.deepEqual(requestBody, { count: 3 }); + assert.match(requestCookie, /client=present/); + assert.match(actual.cookie, /returned=ready/); + rejected(await ctx.run(`return site.read(${JSON.stringify(`${server.alternateOrigin}/app.js`)})`), /origin/i); + }), + + browserCase("webagents", "WebAgents discovery enforces write consent and refreshes workflow state", async ctx => { + let status = "open"; + const workflows: any[] = []; + const server = await serveBrowserFixture(ctx, { + "/": (_request, response) => { response.setHeader("content-type", "text/html"); response.end(documentHtml(`

${status}

`)); }, + "/webagents.md": (_request, response) => { + response.setHeader("content-type", "text/markdown"); + response.end('```webagents\n{"version":"0.1","workflow":{"endpoint":"/workflow"},"actions":{"resolve":{"effect":"write"},"status":{"effect":"read"}}}\n```'); + }, + "/workflow": (request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", chunk => { body += chunk; }); + request.on("end", () => { + workflows.push(JSON.parse(body)); + status = "resolved"; + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ status })); + }); + }, + }); + const directory = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return webagents.discover();`)); + assert.equal(directory.protocol, "webagents/0.1"); + assert.equal(directory.trust, "untrusted_external_data"); + assert.deepEqual(directory.actions.map((action: any) => action.name).sort(), ["resolve", "status"]); + rejected(await ctx.run("return webagents.batch([{id:'resolve',action:'resolve',input:{}}])"), /allowWrites/i); + assert.equal(workflows.length, 0); + const actual = result(await ctx.run(` + const batch = await webagents.batch([{id:'resolve',action:'resolve',input:{}},{id:'verify',action:'status',input:{}}], {allowWrites:true}); + return {batch, visible:await page.getByRole('status').innerText()}; + `)); + assert.equal(actual.batch.result.status, "resolved"); + assert.equal(actual.batch.pageUpdated, true); + assert.equal(actual.visible, "resolved"); + assert.equal(workflows.length, 1); + assert.deepEqual(workflows[0].operations[1].dependsOn, ["resolve"]); + }), + + browserCase("webmcp", "WebMCP discovers and executes a real page-published tool", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": documentHtml(` +

Waiting

+ `) }); + const available = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return page.evaluate(() => Boolean(navigator.modelContext || document.modelContext));`)); + if (!available) ctx.skip("The selected browser does not expose the WebMCP registration API"); + const actual = result(await ctx.run(` + const tools = await webmcp.tools({timeout:3000}); + const tool = tools.find(tool => tool.name === 'calculateSum'); + if (!tool) throw new Error('Registered calculateSum tool is missing'); + return {tool, invocation:await webmcp.invoke(tool.name,{a:19,b:23},{frameId:tool.frameId,timeout:5000}), visible:await page.getByRole('status').innerText()}; + `)); + assert.equal(actual.tool.trust, "untrusted_external_data"); + assert.equal(actual.invocation.status, "Completed"); + assert.deepEqual(actual.invocation.output, { sum: 42 }); + assert.equal(actual.invocation.trust, "untrusted_external_data"); + assert.equal(actual.visible, "Invoked"); + }), + + browserCase("captcha-evidence", "Visible CAPTCHA evidence survives a failed snippet", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": checkboxHtml }); + const envelope = await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); throw new Error('fixture failure with challenge');`); + rejected(envelope, /fixture failure with challenge/); + assert.ok(envelope.challenges.length >= 1); + const artifact = envelope.artifacts.find((entry: any) => entry.kind === "captcha"); + assert.ok(artifact); + await png(artifact.path); + const detected = result(await ctx.run("return captcha.detect()")); + assert.equal(detected.present, true); + assert.equal(detected.cleared, false); + }), + + browserCase("captcha-checkbox", "The local CAPTCHA solver clears a checkbox without an external solver", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": checkboxHtml }); + const actual = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + const solved = await captcha.solve({timeoutMs:15000,maxStages:2}); + return {solved, token:await page.locator('[name="bw-captcha-response"]').inputValue(), title:await page.title(), status:await page.getByRole('status').innerText()}; + `)); + assert.equal(actual.solved.status, "ready"); + assert.equal(actual.solved.cleared, true); + assert.equal(actual.solved.local, true); + assert.equal(actual.solved.externalApi, false); + assert.ok(actual.solved.attempts.length >= 1); + assert.equal(actual.token, "bw_local_fixture_token"); + assert.equal(actual.title, "Verified"); + assert.equal(actual.status, "Verification complete"); + }), + + browserCase("captcha-grid", "An image-grid CAPTCHA returns a real crop and accepts fixture tile picks", async ctx => { + const server = await serveBrowserFixture(ctx, { "/": gridHtml }); + const initial = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return captcha.solve({timeoutMs:15000,maxStages:2});`)); + assert.equal(initial.status, "processing"); + assert.equal(initial.stage, "image_grid"); + assert.equal(initial.tiles.length, 9); + assert.deepEqual(initial.grid, { rows: 3, cols: 3 }); + await png(initial.artifact.path); + const picks = initial.tiles.filter((tile: any) => tile.label === "traffic light").map((tile: any) => tile.index); + assert.equal(picks.length, 3); + const actual = result(await ctx.run(` + const solved = await captcha.solve({tiles:${JSON.stringify(picks)},timeoutMs:15000}); + return {solved,token:await page.locator('[name="bw-captcha-response"]').inputValue(),status:await page.getByRole('status').innerText()}; + `)); + assert.equal(actual.solved.cleared, true); + assert.equal(actual.token, "bw_grid_fixture_token"); + assert.equal(actual.status, "Grid complete"); + }), + + { + id: "recording.mp4", group: "recording", title: "MP4 recording preserves draft state and contains changing decoded frames", requiresBrowser: true, + async run(ctx) { + const executable = encoder(ctx, ["libx264"]); + const server = await serveBrowserFixture(ctx, { "/": animationHtml }); + const started = result(await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.getByLabel('Draft').fill('Unsubmitted draft'); state.marker='preserved'; + return recording.start({name:'first.mp4',fps:10,maxWidth:480,maxHeight:320,maxDurationMs:10000}); + `)); + assert.equal(started.state, "recording"); + const envelope = await ctx.run(` + const before = await page.evaluate(() => window.frameNumber); + await page.waitForFunction(before => window.frameNumber >= before + 20, before, {timeout:5000}); + const saved = await recording.stop(); + return {saved,repeated:await recording.stop(),draft:await page.getByLabel('Draft').inputValue(),marker:state.marker}; + `); + const actual = result(envelope); + assert.equal(actual.saved.state, "completed"); + assert.deepEqual(actual.repeated, actual.saved); + assert.equal(actual.draft, "Unsubmitted draft"); + assert.equal(actual.marker, "preserved"); + assert.ok(actual.saved.outputFrames > 1); + assert.equal((await stat(actual.saved.path)).size, actual.saved.bytes); + const bytes = await readFile(actual.saved.path); + assert.equal(bytes.subarray(4, 8).toString(), "ftyp"); + assert.ok(new Set(decodeVideo(executable, actual.saved.path)).size > 1); + assert.ok(envelope.artifacts.some((artifact: any) => artifact.path === actual.saved.path && artifact.mimeType === "video/mp4")); + }, + }, + + { + id: "recording.restart", group: "recording", title: "Recording restart finalizes MP4 and starts an isolated WebM take", requiresBrowser: true, + async run(ctx) { + const executable = encoder(ctx, ["libx264", "libvpx"]); + const server = await serveBrowserFixture(ctx, { "/": animationHtml }); + const first = result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return recording.start({name:'first.mp4',fps:10,maxWidth:480,maxHeight:320});`, { session: "recorded" })); + assert.deepEqual(result(await ctx.run("return recording.status()", { session: "other" })), { state: "idle" }); + rejected(await ctx.run("return recording.start()", { session: "recorded" }), /already active/); + const restarted = await ctx.run("return recording.restart({name:'second.webm',fps:10,maxWidth:480,maxHeight:320})", { session: "recorded" }); + const second = result(restarted); + assert.equal(second.state, "recording"); + assert.notEqual(second.path, first.path); + assert.ok(restarted.artifacts.some((artifact: any) => artifact.path === first.path)); + const stopped = result(await ctx.run(` + const before=await page.evaluate(() => window.frameNumber); + await page.waitForFunction(before => window.frameNumber >= before+20,before,{timeout:5000}); + return recording.stop(); + `, { session: "recorded" })); + assert.equal(stopped.state, "completed"); + assert.deepEqual([...(await readFile(second.path)).subarray(0, 4)], [26, 69, 223, 163]); + decodeVideo(executable, first.path); + decodeVideo(executable, second.path); + }, + }, + + { + id: "recording.cli", group: "recording", title: "CLI record commands share the session and tab closure flushes the video", requiresBrowser: true, + async run(ctx) { + const executable = encoder(ctx, ["libx264"]); + const server = await serveBrowserFixture(ctx, { "/": animationHtml }); + result(await ctx.run(`await page.goto(${JSON.stringify(server.origin)}); return page.title();`, { session: "movie" })); + const started = await ctx.json(["record", "start", "cli.mp4", "--session", "movie", "--fps", "10", "--max-width", "480", "--max-height", "320"]); + const status = result(await ctx.run("return recording.status()", { session: "movie" })); + assert.equal(status.state, "recording", JSON.stringify(started)); + const actual = result(await ctx.run(` + const before=await page.evaluate(() => window.frameNumber); + await page.waitForFunction(before => window.frameNumber >= before+20,before,{timeout:5000}); + await closePage(); return recording.status(); + `, { session: "movie" })); + assert.equal(actual.state, "completed"); + assert.equal(actual.path, status.path); + assert.equal((await stat(actual.path)).size, actual.bytes); + decodeVideo(executable, actual.path); + }, + }, +]; diff --git a/tests/e2e/browser-fixtures.ts b/tests/e2e/browser-fixtures.ts new file mode 100644 index 00000000..29f5b196 --- /dev/null +++ b/tests/e2e/browser-fixtures.ts @@ -0,0 +1,115 @@ +import type { RequestListener } from "node:http"; +import type { E2EContext, FixtureServer } from "./types.js"; + +export function documentHtml(body: string, title = "Browser fixture"): string { + return `${title} + + ${body}`; +} + +function isHandler(route: string | RequestListener): route is RequestListener { + return typeof route === "function"; +} + +export function serveBrowserFixture( + ctx: E2EContext, + routes: Record, +): Promise { + return ctx.serve((request, response) => { + const route = routes[new URL(request.url || "/", "http://fixture.test").pathname]; + if (isHandler(route)) { + route(request, response); + return; + } + response.writeHead(route === undefined ? 404 : 200, { + "content-type": "text/html; charset=utf-8", + "cache-control": "no-store", + }); + response.end(route ?? "Not found"); + }); +} + +export const formHtml = documentHtml(` +
+ + + + + +
+

Waiting

+ +`); + +export const checkboxHtml = documentHtml(` +

Verify you are human

I'm not a robot — complete the captcha to continue.

+
+ + I'm not a robot +
+ +

Waiting

+ +`, "Local checkbox CAPTCHA"); + +export const gridHtml = documentHtml(` +

Select all images with traffic lights

Please select all images containing a traffic light to continue.

+
+
+ +
+

Waiting

+ +`, "Local image-grid CAPTCHA"); + +export const animationHtml = documentHtml(` + + +`, "Recording fixture"); diff --git a/tests/e2e/cli-cases.ts b/tests/e2e/cli-cases.ts new file mode 100644 index 00000000..ae660cac --- /dev/null +++ b/tests/e2e/cli-cases.ts @@ -0,0 +1,395 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { readdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { CommandResult, E2ECase, E2EContext } from "./types.js"; + +function success(result: CommandResult) { + assert.equal(result.code, 0, result.stderr || result.stdout); + return result.stdout; +} + +function failure(result: CommandResult, expected: RegExp) { + assert.equal(result.code, 1, result.stdout || result.stderr); + assert.match(`${result.stdout}\n${result.stderr}`, expected); +} + +async function tree(directory: string): Promise { + if (!existsSync(directory)) return []; + const entries = await readdir(directory, { withFileTypes: true }); + const result: string[] = []; + for (const entry of entries) { + result.push(entry.name); + if (entry.isDirectory()) { + result.push(...(await tree(path.join(directory, entry.name))).map((name) => `${entry.name}/${name}`)); + } + } + return result.sort(); +} + +async function stopSessions(ctx: E2EContext) { + success(await ctx.command(["close", "--all"])); +} + +export const cases: E2ECase[] = [ + { + id: "cli-help-is-read-only", + group: "cli", + title: "Every documented command answers help without creating user state", + async run(ctx) { + const before = await tree(ctx.workDir); + const main = success(await ctx.command(["--help"])); + assert.match(main, /Usage: betterwright/); + const commands = [...main.matchAll(/^ {2}([a-z]+)\s{2,}\S/gm)].map((match) => match[1]); + assert.ok(commands.length >= 20, "The public command inventory was not found in help"); + for (const command of commands) { + const direct = success(await ctx.command([command, "--help"], { timeoutMs: 15_000 })); + assert.match(direct, /Usage: betterwright/, command); + const routed = success(await ctx.command(["help", command], { timeoutMs: 15_000 })); + assert.equal(routed, direct, `help ${command} differs from ${command} --help`); + } + assert.match(success(await ctx.command(["--local", "--help"])), /local setup/); + assert.deepEqual(await tree(ctx.workDir), before); + }, + }, + { + id: "cli-version", + group: "cli", + title: "Version aliases return only a semantic version without user-state writes", + async run(ctx) { + const before = await tree(ctx.workDir); + const version = success(await ctx.command(["--version"])); + assert.match(version, /^\d+\.\d+\.\d+(?:[-+][\w.-]+)?\n$/); + assert.equal(success(await ctx.command(["-v"])), version); + assert.equal(success(await ctx.command(["--version", "--profile", "../invalid"])), version); + assert.deepEqual(await tree(ctx.workDir), before); + }, + }, + { + id: "cli-invalid-command-and-profile", + group: "cli", + title: "Unknown commands and unsafe profile names fail before browser startup", + async run(ctx) { + const before = await tree(ctx.workDir); + failure(await ctx.command(["not-a-betterwright-command"]), /Unknown command/); + failure(await ctx.command(["run", "-c", "return 1", "--profile", "../escape"]), /--profile:/); + failure(await ctx.command(["sessions"], { env: { BETTERWRIGHT_PROFILE: "../escape" } }), /BETTERWRIGHT_PROFILE:/); + assert.match(success(await ctx.command(["run", "--help", "--profile", "../escape"])), /Usage:/); + assert.deepEqual(await tree(ctx.workDir), before); + }, + }, + { + id: "cli-exec-input-validation", + group: "cli", + title: "Exec rejects absent, whitespace-only, ambiguous, and removed inputs", + async run(ctx) { + failure(await ctx.command(["exec"]), /Usage: betterwright exec/); + failure(await ctx.command(["exec", "--stdin"], { stdin: " \n\t " }), /Usage: betterwright exec/); + failure(await ctx.command(["exec", "task", "--stdin"], { stdin: "another task" }), /either a task argument or --stdin/); + failure(await ctx.command(["exec", "task", "--model-id", "old-name"]), /--model-id was merged into --model/); + }, + }, + { + id: "cli-missing-snippet-file", + group: "cli", + title: "A missing snippet file fails rather than reading stdin or launching a session", + async run(ctx) { + failure(await ctx.command(["run", path.join(ctx.workDir, "missing-snippet.js")]), /ENOENT|No such file/); + assert.match(success(await ctx.command(["sessions"])), /No session daemon is running/); + }, + }, + { + id: "cli-snippet-input-modes", + group: "cli", + title: "Inline, file, explicit stdin, and implicit stdin execute through the public binary", + requiresBrowser: true, + async run(ctx) { + const file = path.join(ctx.workDir, "cli-input-fixture.js"); + await writeFile(file, 'return { input: "file", value: 6 * 7 };\n'); + const inputs = [ + { args: ["-c", 'return { input: "inline", value: 6 * 7 };'], input: "inline", stdin: "" }, + { args: [file], input: "file", stdin: "" }, + { args: ["-"], input: "stdin", stdin: 'return { input: "stdin", value: 6 * 7 };' }, + { args: [], input: "implicit", stdin: 'return { input: "implicit", value: 6 * 7 };' }, + ]; + for (const item of inputs) { + const output = await ctx.json(["run", ...item.args, "--session", "input-modes"], { stdin: item.stdin }); + assert.equal(output.ok, true); + assert.deepEqual(output.result, { input: item.input, value: 42 }); + } + await stopSessions(ctx); + }, + }, + { + id: "cli-repl-stdin", + group: "cli", + title: "REPL runs blank-line-delimited snippets and flushes a final snippet at EOF", + requiresBrowser: true, + async run(ctx) { + const output = success(await ctx.command(["repl", "--session", "repl-input", "--close"], { + stdin: 'state.value = 20;\nreturn "repl-first-marker";\n\n\nstate.value += 22;\nreturn { marker: "repl-final-marker", value: state.value };', + })); + assert.match(output, /BetterWright REPL/); + const marker = output.indexOf("\n{"); + assert.ok(marker >= 0, output); + const envelopes = output.slice(marker + 1).trim().split(/\n(?=\{)/).map((value) => JSON.parse(value)); + assert.equal(envelopes.length, 2); + assert.equal(envelopes[0].ok, true); + assert.equal(envelopes[0].result, "repl-first-marker"); + assert.equal(envelopes[1].ok, true); + assert.deepEqual(envelopes[1].result, { marker: "repl-final-marker", value: 42 }); + }, + }, + { + id: "cli-json-and-pretty", + group: "cli", + title: "Piped run output is compact JSON unless pretty output is requested", + requiresBrowser: true, + async run(ctx) { + const args = ["run", "-c", 'return { marker: "format-check", nested: { answer: 42 } };', "--session", "format"]; + const compact = success(await ctx.command(args)); + const pretty = success(await ctx.command([...args, "--pretty"])); + assert.equal(compact.trim().split("\n").length, 1); + assert.match(pretty, /\n {2}"ok": true/); + assert.equal(JSON.parse(compact).ok, true); + assert.deepEqual(JSON.parse(pretty).result, JSON.parse(compact).result); + await stopSessions(ctx); + }, + }, + { + id: "cli-snippet-error-envelope", + group: "cli", + title: "Snippet failures return JSON with a failing exit status and leave the session usable", + requiresBrowser: true, + async run(ctx) { + const result = await ctx.run('throw new Error("cli-runtime-failure-marker")', { session: "failures" }); + assert.equal(result.ok, false); + assert.match(JSON.stringify(result.error), /cli-runtime-failure-marker/); + const syntax = await ctx.run("return {", { session: "failures" }); + assert.equal(syntax.ok, false); + assert.ok(syntax.error); + const recovered = await ctx.run("return 42", { session: "failures" }); + assert.equal(recovered.ok, true); + assert.equal(recovered.result, 42); + await stopSessions(ctx); + }, + }, + { + id: "cli-empty-session-management", + group: "cli", + title: "Session inventory and close are idempotent when no daemon exists", + async run(ctx) { + assert.match(success(await ctx.command(["sessions"])), /No session daemon is running/); + for (const args of [["close"], ["close", "absent"], ["close", "--all"], ["close", "--profile", "absent"]]) { + assert.match(success(await ctx.command(args)), /nothing to close/); + } + assert.match(success(await ctx.command(["sessions"])), /No session daemon is running/); + }, + }, + { + id: "cli-session-persistence-and-close", + group: "cli", + title: "Named sessions retain separate state and closing one leaves its sibling intact", + requiresBrowser: true, + async run(ctx) { + assert.equal((await ctx.run('state.marker = "alpha"; return state.marker', { session: "alpha-session" })).result, "alpha"); + assert.equal((await ctx.run('state.marker = "beta"; return state.marker', { session: "beta-session" })).result, "beta"); + assert.equal((await ctx.run("return state.marker", { session: "alpha-session" })).result, "alpha"); + const listed = success(await ctx.command(["sessions"])); + assert.match(listed, /alpha-session/); + assert.match(listed, /beta-session/); + assert.match(success(await ctx.command(["close", "alpha-session"])), /Closed session "alpha-session"/); + const remaining = success(await ctx.command(["sessions"])); + assert.doesNotMatch(remaining, /alpha-session/); + assert.match(remaining, /beta-session/); + assert.equal((await ctx.run("return state.marker", { session: "beta-session" })).result, "beta"); + await stopSessions(ctx); + assert.match(success(await ctx.command(["sessions"])), /No session daemon is running/); + }, + }, + { + id: "cli-run-close", + group: "cli", + title: "Run --close returns its result but does not preserve the closed session state", + requiresBrowser: true, + async run(ctx) { + const first = await ctx.run('state.marker = "discard"; return 42', { session: "bounded", args: ["--close"] }); + assert.equal(first.ok, true); + assert.equal(first.result, 42); + const next = await ctx.run("return Object.hasOwn(state, 'marker')", { session: "bounded", args: ["--close"] }); + assert.equal(next.ok, true); + assert.equal(next.result, false); + }, + }, + { + id: "cli-configure-persistence", + group: "cli", + title: "Configure saves and resets the browser selection across invocations without connecting", + async run(ctx) { + const initial = await ctx.json(["configure", "--show", "--json"]); + assert.equal(initial.default, null); + assert.equal(path.relative(ctx.home, initial.file).startsWith(".."), false); + success(await ctx.command(["configure", "--browser", "wss://browser.invalid/devtools/browser/e2e", "--no-test"])); + const saved = await ctx.json(["configure", "--show", "--json"]); + assert.equal(saved.default.cdpUrl, "wss://browser.invalid/devtools/browser/e2e"); + const persisted = JSON.parse(await readFile(saved.file, "utf8")); + assert.equal(persisted.browser.default.cdpUrl, saved.default.cdpUrl); + assert.equal((await stat(saved.file)).mode & 0o077, 0); + success(await ctx.command(["configure", "--managed"])); + assert.equal((await ctx.json(["configure", "--show", "--json"])).default, null); + }, + }, + { + id: "cli-configure-key-masking", + group: "cli", + title: "Stored synthetic provider keys persist but never appear in configure output", + async run(ctx) { + const key = "e2e-synthetic-browser-key-do-not-use"; + ctx.redact(key); + const configured = await ctx.command(["configure", "--browser", "browserbase", "--browser-key", key, "--no-test"]); + success(configured); + assert.equal(`${configured.stdout}${configured.stderr}`.includes(key), false); + const shown = await ctx.command(["configure", "--show", "--json"]); + const data = JSON.parse(success(shown)); + assert.equal(data.default.apiKey, "***"); + assert.equal(data.accounts.browserbase.apiKey, "***"); + assert.equal(`${shown.stdout}${shown.stderr}`.includes(key), false); + assert.equal(success(await ctx.command(["configure", "--show"])).includes(key), false); + const stored = JSON.parse(await readFile(data.file, "utf8")); + assert.equal(stored.browser.default.apiKey === key, true); + assert.equal((await stat(data.file)).mode & 0o077, 0); + }, + }, + { + id: "cli-configure-environment-and-fallbacks", + group: "cli", + title: "Environment key references avoid persistence and fallback order survives reload", + async run(ctx) { + const key = "e2e-environment-only-provider-key"; + ctx.redact(key); + const env = { E2E_BROWSER_PROVIDER_KEY: key }; + success(await ctx.command(["configure", "--browser", "steel", "--key-env", "E2E_BROWSER_PROVIDER_KEY", "--browser-fallback", "managed", "--browser-fallback", "wss://fallback.invalid/browser", "--no-test"], { env })); + const shown = await ctx.json(["configure", "--show", "--json"], { env }); + assert.equal(shown.default.keyEnv, "E2E_BROWSER_PROVIDER_KEY"); + assert.equal(shown.default.keyEnvSet, true); + assert.equal(Object.hasOwn(shown.default, "apiKey"), false); + assert.deepEqual(shown.fallbacks, [{ provider: "managed" }, { cdpUrl: "wss://fallback.invalid/browser" }]); + assert.equal((await readFile(shown.file, "utf8")).includes(key), false); + assert.equal((await ctx.json(["configure", "--show", "--json"], { env: { E2E_BROWSER_PROVIDER_KEY: "" } })).default.keyEnvSet, false); + success(await ctx.command(["configure", "--clear-fallbacks"])); + assert.deepEqual((await ctx.json(["configure", "--show", "--json"])).fallbacks, []); + }, + }, + { + id: "cli-configure-invalid-values", + group: "cli", + title: "Invalid provider configuration fails without changing the saved selection", + async run(ctx) { + const before = await ctx.json(["configure", "--show", "--json"]); + failure(await ctx.command(["configure", "--browser", "https://browser.invalid"]), /CDP endpoint must be a ws:\/\/ or wss:\/\//); + failure(await ctx.command(["configure", "--browser", "not-a-provider"]), /[Uu]nknown.*provider/); + failure(await ctx.command(["configure", "--browser", "steel", "--browser-key", "synthetic", "--key-env", "E2E_KEY"]), /--browser-key.*--key-env|--key-env.*--browser-key/); + assert.deepEqual(await ctx.json(["configure", "--show", "--json"]), before); + }, + }, + { + id: "cli-auth-status-isolated", + group: "cli", + title: "Auth status reports signed out from isolated homes without attempting login", + async run(ctx) { + const codex = path.join(ctx.home, "empty-codex"); + const grok = path.join(ctx.home, "empty-grok"); + const result = await ctx.command(["auth", "--status"], { env: { CODEX_HOME: codex, GROK_HOME: grok } }); + assert.equal(result.code, 1); + assert.match(result.stdout, /codex\s+not signed in/); + assert.match(result.stdout, /grok\s+not signed in/); + assert.equal(existsSync(codex), false); + assert.equal(existsSync(grok), false); + }, + }, + { + id: "cli-skills-read-only", + group: "cli", + title: "Skill printing, listing, and safe lookup work without installing or traversing paths", + async run(ctx) { + const before = await tree(ctx.home); + assert.match(success(await ctx.command(["skill"])), /betterwright run/); + assert.match(success(await ctx.command(["skill", "--claude"])), /^---\n/); + const listed = success(await ctx.command(["skills", "list"])); + const name = listed.split("\n").find((line) => /^[a-z0-9-]+\t/.test(line))?.split("\t")[0]; + assert.ok(name, "No packaged skills were listed"); + assert.ok(success(await ctx.command(["skills", "show", name])).trim().length > 50); + const sentinel = path.join(ctx.workDir, "cli-skill-traversal.txt"); + await writeFile(sentinel, "CLI_SKILL_TRAVERSAL_MUST_NOT_BE_READ"); + for (const target of ["../cli-skill-traversal.txt", sentinel, "missing-e2e-skill"]) { + const result = await ctx.command(["skills", "show", target]); + assert.equal(result.code, 1); + assert.doesNotMatch(`${result.stdout}${result.stderr}`, /CLI_SKILL_TRAVERSAL_MUST_NOT_BE_READ/); + } + assert.deepEqual(await tree(ctx.home), before); + }, + }, + { + id: "cli-vault-terminal-required", + group: "cli", + title: "Vault setup and unlock refuse piped master passwords without creating credentials", + async run(ctx) { + for (const action of ["setup", "unlock"]) { + const result = await ctx.command(["vault", action], { stdin: "not-a-real-password\nnot-a-real-password\n" }); + failure(result, /terminal is required for master-password entry/); + assert.doesNotMatch(`${result.stdout}${result.stderr}`, /not-a-real-password/); + } + const status = await ctx.json(["vault", "status", "--json"]); + assert.equal(status.configured, false); + const entries = await tree(ctx.home); + assert.equal(entries.some((entry) => /(?:^|\/)(?:vault\.enc|master-key\.json)$/.test(entry)), false); + }, + }, + { + id: "cli-cookie-validation-only", + group: "cli", + title: "Cookie commands reject invalid arguments before inspecting source cookie stores", + async run(ctx) { + const invalid = [ + ["browsers", "unexpected"], + ["profiles"], + ["sync"], + ["sync", "chrome"], + ["sync", "chrome", "--all", "--domain", "example.invalid"], + ["sync", "chrome", "--domain", "--all"], + ]; + for (const args of invalid) { + const result = await ctx.command(["cookies", ...args, "--json"]); + assert.equal(result.code, 1); + const body = JSON.parse(result.stdout); + assert.equal(body.ok, false); + assert.match(body.error, /Usage:|requires|takes no positional/); + } + assert.match(success(await ctx.command(["sessions"])), /No session daemon is running/); + }, + }, + { + id: "cli-doctor-structured-prerequisites", + group: "cli", + title: "Doctor returns structured prerequisites and exit status tracks reported readiness", + async run(ctx) { + const result = await ctx.command(["doctor", "--json"]); + const report = JSON.parse(result.stdout); + assert.equal(result.code, report.ready ? 0 : 1); + assert.equal(report.worker_ok, true); + assert.equal(report.playwright_version, report.playwright_pinned); + assert.ok([true, false].includes(report.provider_ready)); + assert.ok(report.runtime); + assert.ok(report.runtime_version); + assert.ok(Array.isArray(report.checks) && report.checks.length > 0); + for (const check of report.checks) { + assert.ok(check.group && check.label); + assert.ok(["ok", "warn", "fail"].includes(check.status), JSON.stringify(check)); + assert.equal(Object.hasOwn(check, "detail"), true); + } + const output = await ctx.command(["doctor", "--quiet"]); + assert.ok(output.code === 0 || output.code === 1); + assert.match(output.stdout, /ready|Not ready/); + }, + }, +]; diff --git a/tests/e2e/harness.ts b/tests/e2e/harness.ts new file mode 100644 index 00000000..def8fcfc --- /dev/null +++ b/tests/e2e/harness.ts @@ -0,0 +1,373 @@ +import assert from "node:assert/strict"; +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import type { + CommandOptions, + CommandResult, + E2ECase, + E2EContext, +} from "./types.js"; + +export interface RunnerOptions { + binary: string; + binaryArgs: string[]; + report: string; + groups: string[]; + filter: string; + timeoutMs: number; + caseTimeoutMs: number; + keepWork: boolean; + requireAll: boolean; + list: boolean; + help: boolean; +} + +export interface CaseResult { + id: string; + group: string; + title: string; + status: "passed" | "failed" | "skipped"; + durationMs: number; + error?: string; + stack?: string; + reason?: string; + workDir?: string; +} + +export class SkipCase extends Error {} + +function isTcpAddress(value: string | AddressInfo | null): value is AddressInfo { + return value !== null && typeof value === "object"; +} + +export function parseOptions(args: string[]): RunnerOptions { + const options: RunnerOptions = { + binary: "betterwright", + binaryArgs: [], + report: path.resolve(".tmp/e2e-report.json"), + groups: [], + filter: "", + timeoutMs: 45_000, + caseTimeoutMs: 180_000, + keepWork: false, + requireAll: false, + list: false, + help: false, + }; + let binarySet = false; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (["--keep-work", "--require-all", "--list", "--help", "-h"].includes(arg)) { + if (arg === "--keep-work") options.keepWork = true; + if (arg === "--require-all") options.requireAll = true; + if (arg === "--list") options.list = true; + if (arg === "--help" || arg === "-h") options.help = true; + continue; + } + if (["--binary", "--binary-arg", "--report", "--group", "--filter", "--timeout", "--case-timeout"].includes(arg)) { + const value = args[++index]; + if (!value || (value.startsWith("--") && arg !== "--binary-arg")) { + throw new Error(`${arg} requires a value`); + } + if (arg === "--binary") { + if (binarySet) throw new Error("Specify the binary only once"); + options.binary = value; + binarySet = true; + } + if (arg === "--binary-arg") options.binaryArgs.push(value); + if (arg === "--report") options.report = path.resolve(value); + if (arg === "--group") options.groups.push(value); + if (arg === "--filter") options.filter = value; + if (arg === "--timeout" || arg === "--case-timeout") { + const milliseconds = Number(value); + if (!Number.isSafeInteger(milliseconds) || milliseconds < 100 || milliseconds > 3_600_000) { + throw new Error(`${arg} must be an integer from 100 to 3600000 milliseconds`); + } + if (arg === "--timeout") options.timeoutMs = milliseconds; + else options.caseTimeoutMs = milliseconds; + } + continue; + } + if (arg.startsWith("-") || binarySet) throw new Error(`Unexpected argument: ${arg}`); + options.binary = arg; + binarySet = true; + } + if (options.binary.includes("/") || options.binary.includes("\\")) { + options.binary = path.resolve(options.binary); + } + for (const group of options.groups) { + if (!["cli", "browser", "security", "protocol", "recording"].includes(group)) { + throw new Error(`Unknown group: ${group}`); + } + } + return options; +} + +export function selectCases(cases: E2ECase[], options: RunnerOptions): E2ECase[] { + const ids = new Set(); + for (const entry of cases) { + if (!/^[a-z0-9][a-z0-9.-]+$/.test(entry.id)) throw new Error(`Invalid case id: ${entry.id}`); + if (ids.has(entry.id)) throw new Error(`Duplicate case id: ${entry.id}`); + ids.add(entry.id); + } + const selected = cases.filter((entry) => + (!options.groups.length || options.groups.includes(entry.group)) && + (!options.filter || `${entry.id} ${entry.title}`.includes(options.filter)), + ); + if (!selected.length) throw new Error("No test cases matched; refusing an empty successful run"); + return selected; +} + +export function isolatedEnvironment(workDir: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const key of [ + "PATH", "SystemRoot", "SYSTEMROOT", "WINDIR", "windir", "COMSPEC", "ComSpec", + "PATHEXT", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", "DISPLAY", "WAYLAND_DISPLAY", + "XDG_RUNTIME_DIR", "BETTERWRIGHT_FFMPEG_PATH", + ]) { + if (process.env[key]) env[key] = process.env[key]; + } + const userHome = path.join(workDir, "user"); + const temp = path.join(workDir, "tmp"); + for (const dir of [userHome, temp]) fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + Object.assign(env, { + HOME: userHome, + USERPROFILE: userHome, + XDG_CONFIG_HOME: path.join(userHome, ".config"), + XDG_CACHE_HOME: path.join(userHome, ".cache"), + XDG_DATA_HOME: path.join(userHome, ".local", "share"), + APPDATA: path.join(userHome, "AppData", "Roaming"), + LOCALAPPDATA: path.join(userHome, "AppData", "Local"), + TMPDIR: temp, + TMP: temp, + TEMP: temp, + BETTERWRIGHT_HOME: path.join(workDir, "betterwright"), + BETTERWRIGHT_NO_DAEMON: "0", + BETTERWRIGHT_AD_BLOCK: "0", + BUN_RUNTIME_TRANSPILER_CACHE_PATH: "0", + NO_COLOR: "1", + TERM: "dumb", + LANG: "C.UTF-8", + }); + if (process.env.BETTERWRIGHT_CHROMIUM_PATH) { + env.BETTERWRIGHT_CHROMIUM_PATH = process.env.BETTERWRIGHT_CHROMIUM_PATH; + } else { + env.BETTERWRIGHT_CHROMIUM_ROOT = process.env.BETTERWRIGHT_CHROMIUM_ROOT || + path.join(os.homedir(), ".betterwright", "chromium"); + } + return env; +} + +async function stopChild(child: ChildProcessWithoutNullStreams) { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 1_000); + child.once("close", () => { clearTimeout(timer); resolve(); }); + child.kill("SIGTERM"); + }); +} + +export function createContext(options: RunnerOptions, workDir: string) { + const env = isolatedEnvironment(workDir); + const children = new Set(); + const cleanups: Array<() => Promise | void> = []; + const secrets = new Set(); + let disposed = false; + const sanitize = (text: string) => { + let result = text; + for (const secret of [...secrets].sort((left, right) => right.length - left.length)) { + result = result.replaceAll(secret, "[redacted]"); + } + return result; + }; + const start = (args: string[], commandOptions: CommandOptions = {}, duringCleanup = false) => { + if (disposed && !duringCleanup) throw new Error("Test context has already been disposed"); + const child = spawn(options.binary, [...options.binaryArgs, ...args], { + cwd: workDir, + env: { ...env, ...commandOptions.env }, + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + children.add(child); + child.once("close", () => children.delete(child)); + child.on("error", () => {}); + child.stdin.on("error", () => {}); + return child; + }; + const command = (args: string[], commandOptions: CommandOptions = {}, duringCleanup = false) => new Promise((resolve, reject) => { + const child = start(args, commandOptions, duringCleanup); + let stdout = ""; + let stderr = ""; + let bytes = 0; + let failure: Error | null = null; + const timeoutMs = commandOptions.timeoutMs ?? options.timeoutMs; + const timer = setTimeout(() => { + failure = new Error(`Command ${args[0] || "(default)"} exceeded ${timeoutMs}ms`); + child.kill("SIGKILL"); + }, timeoutMs); + const collect = (channel: "stdout" | "stderr", chunk: string) => { + bytes += Buffer.byteLength(chunk); + if (bytes > 2 * 1024 * 1024) { + failure = new Error(`Command ${args[0] || "(default)"} exceeded the 2 MiB output limit`); + child.kill("SIGKILL"); + return; + } + if (channel === "stdout") stdout += chunk; + else stderr += chunk; + }; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => collect("stdout", chunk)); + child.stderr.on("data", (chunk) => collect("stderr", chunk)); + child.once("error", (error) => { failure = error; }); + child.once("close", (code) => { + clearTimeout(timer); + if (failure) reject(failure); + else resolve({ code, stdout, stderr }); + }); + child.stdin.end(commandOptions.stdin ?? ""); + }); + const context: E2EContext = { + home: env.BETTERWRIGHT_HOME, + workDir, + start, + command, + async json(args, commandOptions) { + const result = await command(args, commandOptions); + assert.equal(result.code, 0, sanitize(`${args[0]} failed: ${result.stderr}\n${result.stdout}`)); + try { + return JSON.parse(result.stdout); + } catch { + throw new Error(sanitize(`${args[0]} did not return one JSON value: ${result.stdout.slice(0, 2_000)}`)); + } + }, + async run(code, runOptions = {}) { + const args = ["run", "--no-ad-block", "--no-auto-ui", "--session", runOptions.session || "e2e"]; + if (runOptions.profile) args.push("--profile", runOptions.profile); + args.push(...(runOptions.args || []), "-c", code); + const result = await command(args, runOptions); + let envelope; + try { + envelope = JSON.parse(result.stdout); + } catch { + throw new Error(sanitize(`run did not return JSON (exit ${result.code}): ${result.stderr}\n${result.stdout.slice(0, 2_000)}`)); + } + assert.ok(envelope && [true, false].includes(envelope.ok), "run must return an envelope with a boolean ok"); + if (envelope.ok) assert.equal(result.code, 0, "A successful envelope must have exit code 0"); + else assert.notEqual(result.code, 0, "A failed envelope must have a nonzero exit code"); + assert.notEqual(result.code, null, "run must exit normally rather than die from a signal"); + return envelope; + }, + async serve(handler) { + const server = http.createServer(handler); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + const address = server.address(); + if (!isTcpAddress(address)) throw new Error("Fixture server has no TCP address"); + const close = async () => { + if (!server.listening) return; + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + server.closeAllConnections(); + }); + }; + cleanups.push(close); + return { origin: `http://127.0.0.1:${address.port}`, alternateOrigin: `http://localhost:${address.port}`, close }; + }, + cleanup(action) { cleanups.push(action); }, + skip(reason) { throw new SkipCase(reason); }, + redact(value) { if (value) secrets.add(value); }, + }; + return { + context, + sanitize, + async dispose() { + disposed = true; + const failures: string[] = []; + await Promise.all([...children].map(stopChild)); + try { + if (fs.existsSync(context.home)) { + const result = await command(["close", "--all"], { timeoutMs: 15_000 }, true); + if (result.code !== 0) failures.push(`close --all exited ${result.code}: ${result.stderr}`); + } + } catch (error) { + failures.push(String(error)); + } + for (const cleanup of cleanups.reverse()) { + try { await cleanup(); } catch (error) { failures.push(String(error)); } + } + if (failures.length) throw new Error(sanitize(`Cleanup failed: ${failures.join("; ")}`)); + }, + }; +} + +export async function runCase(entry: E2ECase, options: RunnerOptions, root: string, signal?: AbortSignal): Promise { + const workDir = fs.mkdtempSync(path.join(root, "t-")); + const harness = createContext(options, workDir); + const started = performance.now(); + const result: CaseResult = { + id: entry.id, + group: entry.group, + title: entry.title, + status: "passed", + durationMs: 0, + }; + let timer: ReturnType; + let interrupted: () => void; + try { + signal?.throwIfAborted(); + await Promise.race([ + entry.run(harness.context), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Case exceeded ${options.caseTimeoutMs}ms`)), options.caseTimeoutMs); + interrupted = () => reject(new Error("Run interrupted")); + signal?.addEventListener("abort", interrupted, { once: true }); + }), + ]); + } catch (error) { + result.status = error instanceof SkipCase ? "skipped" : "failed"; + const detail = harness.sanitize(error instanceof Error ? error.message : String(error)); + if (result.status === "skipped") result.reason = detail; + else { + result.error = detail; + if (error instanceof Error && error.stack) result.stack = harness.sanitize(error.stack); + } + } finally { + clearTimeout(timer); + if (interrupted) signal?.removeEventListener("abort", interrupted); + try { + await harness.dispose(); + } catch (error) { + result.status = "failed"; + result.error = [result.error, harness.sanitize(String(error))].filter(Boolean).join("\n"); + delete result.reason; + } + if (options.keepWork || result.error?.includes("Cleanup failed:")) result.workDir = workDir; + else { + try { + fs.rmSync(workDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } catch (error) { + result.status = "failed"; + result.error = [result.error, harness.sanitize(`Cleanup failed: ${String(error)}`)].filter(Boolean).join("\n"); + result.workDir = workDir; + delete result.reason; + } + } + } + result.durationMs = Math.round(performance.now() - started); + return result; +} + +export function reportExitCode(results: CaseResult[], requireAll: boolean): number { + if (!results.length || results.some((result) => result.status === "failed")) return 1; + if (requireAll && results.some((result) => result.status === "skipped")) return 1; + const cases = results.filter((result) => !result.id.startsWith("preflight.")); + if (!cases.length || cases.every((result) => result.status === "skipped")) return 1; + return 0; +} diff --git a/tests/e2e/lifecycle-cases.ts b/tests/e2e/lifecycle-cases.ts new file mode 100644 index 00000000..95e0611b --- /dev/null +++ b/tests/e2e/lifecycle-cases.ts @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { E2ECase } from "./types.js"; + +function isText(value: any): value is string { + return typeof value === "string"; +} + +export const cases: E2ECase[] = [ + { + id: "browser.stealth-runtime", group: "browser", title: "The optional stealth driver isolates main-world globals while preserving DOM actions", requiresBrowser: true, + async run(ctx) { + const doctor = await ctx.json(["doctor", "--json"]); + if (!doctor.stealth_available) ctx.skip("The target does not have its optional stealth driver installed"); + const server = await ctx.serve((_request, response) => { + response.setHeader("content-type", "text/html"); + response.end('Stealth fixture'); + }); + const output = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.getByLabel('Name').fill('isolated'); + return {title:await page.title(),value:await page.getByLabel('Name').inputValue(), + mainWorld:await page.evaluate(() => typeof window.e2eMainWorld),node:typeof process}; + `, { args: ["--stealth"] }); + assert.equal(output.ok, true, output.error); + assert.deepEqual(output.result, { title: "Stealth fixture", value: "isolated", mainWorld: "undefined", node: "undefined" }); + assert.match(output.warnings.join(" "), /isolated world/); + }, + }, + { + id: "browser.provider-fallback", group: "browser", title: "An unavailable local CDP endpoint falls back to the managed browser with diagnostics", requiresBrowser: true, + async run(ctx) { + let attempts = 0; + const unavailable = await ctx.serve((_request, response) => { + attempts++; + response.writeHead(503, { "content-type": "text/plain" }); + response.end("Synthetic CDP outage"); + }); + const configured = await ctx.command(["configure", "--browser", unavailable.origin.replace("http:", "ws:"), "--browser-fallback", "managed", "--no-test"]); + assert.equal(configured.code, 0, configured.stderr); + assert.equal(attempts, 0); + const output = await ctx.run("await page.setContent('Fallback succeeded'); return page.title();"); + assert.equal(output.ok, true, output.error); + assert.equal(output.result, "Fallback succeeded"); + assert.ok(attempts > 0, "The primary provider must actually be attempted"); + assert.match(output.warnings.join(" "), /failed|fallback/i); + }, + }, + { + id: "browser.result-serialization", group: "browser", title: "Empty values, big integers, cycles, and collection caps serialize safely", requiresBrowser: true, + async run(ctx) { + const output = await ctx.run(` + const circular = {label:'cycle'}; circular.self = circular; + return {empty:[undefined,null,false,0,''],big:123456789012345678901234567890n, + circular,items:Array.from({length:250},(_,index)=>index),locator:page.locator('body')}; + `); + assert.equal(output.ok, true, output.error); + assert.deepEqual(output.result.empty, [null, null, false, 0, ""]); + assert.equal(output.result.big, "123456789012345678901234567890"); + assert.deepEqual(output.result.circular, { label: "cycle", self: "[Circular]" }); + assert.equal(output.result.items.length, 200); + assert.equal(output.result.items[199], 199); + assert.equal(output.result.locator.type, "Locator"); + assert.equal(JSON.stringify(output).includes("_connection"), false); + }, + }, + { + id: "browser.result-spill", group: "browser", title: "Oversized results spill to a bounded preview and an explicitly untrusted artifact", requiresBrowser: true, + async run(ctx) { + const output = await ctx.run("return 'E2E-BEGIN-'+'x'.repeat(16000)+'-E2E-END';"); + assert.equal(output.ok, true, output.error); + assert.equal(output.result.truncated, true); + assert.match(output.result.preview, /^E2E-BEGIN-/); + assert.match(output.result.preview, /-E2E-END$/); + assert.ok(output.result.preview.length < 16_000); + const relative = path.relative(ctx.home, output.result.fullOutputPath); + assert.equal(relative.startsWith("..") || path.isAbsolute(relative), false); + const content = await readFile(output.result.fullOutputPath, "utf8"); + assert.match(content, /$/); + }, + }, + { + id: "browser.sync-timeout", group: "browser", title: "An infinite synchronous snippet is bounded and the next command recovers", requiresBrowser: true, + async run(ctx) { + const output = await ctx.run("while(true) {}", { timeoutMs: 15_000 }); + assert.equal(output.ok, false); + assert.match(output.error, /timed out|timeout/i); + const recovered = await ctx.run("return 42;"); + assert.equal(recovered.ok, true, recovered.error); + assert.equal(recovered.result, 42); + }, + }, + { + id: "browser.async-timeout-restart", group: "browser", title: "An unresolved async snippet restarts its worker without losing persistent cookies", requiresBrowser: true, + async run(ctx) { + const server = await ctx.serve((_request, response) => { + response.setHeader("content-type", "text/html"); + response.end('Restart fixture

Restart fixture

'); + }); + const seeded = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + await page.evaluate(() => {document.cookie='e2e_restart=preserved; Max-Age=3600; Path=/';}); + state.marker = 'old-worker'; return page.evaluate(() => document.cookie); + `); + assert.equal(seeded.ok, true, seeded.error); + assert.match(seeded.result, /e2e_restart=preserved/); + const failed = await ctx.run("await new Promise(() => {});", { timeoutMs: 45_000 }); + assert.equal(failed.ok, false); + assert.match(failed.error, /timed out/i); + const recovered = await ctx.run(` + await page.goto(${JSON.stringify(server.origin)}); + return {cookie:await page.evaluate(() => document.cookie),marker:state.marker ?? null,title:await page.title()}; + `); + assert.equal(recovered.ok, true, recovered.error); + assert.equal(recovered.result.marker, null); + assert.equal(recovered.result.title, "Restart fixture"); + assert.match(recovered.result.cookie, /e2e_restart=preserved/); + }, + }, + { + id: "protocol.live-view-watch-only", group: "protocol", title: "Live view requires its token, streams frames, rejects watch-only navigation, and detaches safely", requiresBrowser: true, + async run(ctx) { + const initial = await ctx.run("await page.setContent('Live E2E

Live E2E

'); state.marker='still here'; return page.title();", { session: "viewer" }); + assert.equal(initial.ok, true, initial.error); + const child = ctx.start(["view", "--expose", "local", "--watch-only", "--session", "viewer", "--no-ad-block"]); + let output = ""; + const address = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Live view did not announce its address")), 20_000); + child.once("error", (error) => { clearTimeout(timer); reject(error); }); + child.once("close", () => { clearTimeout(timer); reject(new Error("Live view exited before announcing its address")); }); + child.stdout.on("data", (chunk) => { + output += chunk.toString(); + const matched = /Live view:\s+(http:\/\/127\.0\.0\.1:\d+\/\?t=[A-Za-z0-9_-]+)/.exec(output); + if (matched) { clearTimeout(timer); resolve(matched[1]); } + }); + }); + ctx.redact(address); + const url = new URL(address); + ctx.redact(url.searchParams.get("t")); + assert.equal((await fetch(url.origin, { signal: AbortSignal.timeout(5_000) })).status, 404); + assert.equal((await fetch(`${url.origin}/?t=incorrect`, { signal: AbortSignal.timeout(5_000) })).status, 404); + const page = await fetch(address, { signal: AbortSignal.timeout(5_000) }); + assert.equal(page.status, 200); + assert.equal(page.headers.get("x-frame-options"), "DENY"); + assert.equal(page.headers.get("referrer-policy"), "no-referrer"); + assert.match(await page.text(), /canvas/); + url.protocol = "ws:"; + url.pathname = "/ws"; + const socket = new WebSocket(url); + socket.binaryType = "arraybuffer"; + ctx.cleanup(() => { socket.close(); }); + const messages: any[] = []; + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Live view did not produce a frame and a watch-only rejection")), 15_000); + let frame = false; + let rejected = false; + socket.addEventListener("error", () => { clearTimeout(timer); reject(new Error("Live-view WebSocket failed")); }); + socket.addEventListener("message", (event) => { + try { + if (isText(event.data)) { + const message = JSON.parse(event.data); + messages.push(message); + if (message.t === "hello") { + assert.equal(message.interactive, false); + socket.send(JSON.stringify({ t: "navigate", url: "about:blank" })); + } + if (message.t === "toast" && /watch-only/.test(message.text)) rejected = true; + } else if (event.data instanceof ArrayBuffer && event.data.byteLength > 0) frame = true; + if (frame && rejected) { clearTimeout(timer); resolve(); } + } catch (error) { clearTimeout(timer); reject(error); } + }); + }); + assert.ok(messages.some((message) => message.t === "hello")); + socket.close(); + const closed = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Live view did not detach after SIGINT")), 10_000); + child.once("close", (code) => { clearTimeout(timer); resolve(code); }); + }); + child.kill("SIGINT"); + assert.equal(await closed, 0); + const preserved = await ctx.run("return {title:await page.title(),marker:state.marker};", { session: "viewer" }); + assert.equal(preserved.ok, true, preserved.error); + assert.deepEqual(preserved.result, { title: "Live E2E", marker: "still here" }); + }, + }, +]; diff --git a/tests/e2e/protocol-cases.ts b/tests/e2e/protocol-cases.ts new file mode 100644 index 00000000..7e74e04a --- /dev/null +++ b/tests/e2e/protocol-cases.ts @@ -0,0 +1,335 @@ +import assert from "node:assert/strict"; +import { createInterface } from "node:readline"; +import type { CommandOptions, E2ECase, E2EContext } from "./types.js"; + +async function mcp(ctx: E2EContext, options: CommandOptions = {}) { + const child = ctx.start(["mcp"], options); + const lines = createInterface({ input: child.stdout }); + const pending = new Map }>(); + let nextId = 1; + let stderr = ""; + let protocolFailure: Error | null = null; + const fail = (error: Error) => { + protocolFailure = error; + for (const entry of pending.values()) { + clearTimeout(entry.timer); + entry.reject(error); + } + pending.clear(); + }; + child.stderr.on("data", (chunk) => { stderr = (stderr + chunk.toString()).slice(-16_000); }); + lines.on("line", (line) => { + let message; + try { + message = JSON.parse(line); + } catch { + fail(new Error(`MCP stdout contains non-JSON output: ${line.slice(0, 500)}`)); + return; + } + if (message.jsonrpc !== "2.0") { + fail(new Error(`Invalid JSON-RPC response: ${line.slice(0, 500)}`)); + return; + } + const entry = pending.get(message.id); + if (!entry) return; + clearTimeout(entry.timer); + pending.delete(message.id); + entry.resolve(message); + }); + child.once("error", fail); + const closed = new Promise((resolve) => { + child.once("close", (code) => { + if (pending.size) fail(new Error(`MCP exited with ${code} before responding: ${stderr}`)); + lines.close(); + resolve(code); + }); + }); + const request = (method: string, params: any = {}) => new Promise((resolve, reject) => { + if (protocolFailure) { reject(protocolFailure); return; } + if (child.exitCode !== null || child.signalCode !== null) { + reject(new Error(`MCP is no longer running: ${stderr}`)); + return; + } + const id = nextId++; + const timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`MCP ${method} timed out: ${stderr}`)); + }, options.timeoutMs ?? 45_000); + pending.set(id, { resolve, reject, timer }); + child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", id, method, params })}\n`); + }); + const notify = (method: string) => { + child.stdin.write(`${JSON.stringify({ jsonrpc: "2.0", method })}\n`); + }; + const close = async () => { + child.stdin.end(); + let timer: ReturnType | undefined; + try { + const code = await Promise.race([ + closed, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`MCP did not exit after stdin EOF: ${stderr}`)), 15_000); + }), + ]); + assert.equal(code, 0, stderr); + if (protocolFailure) throw protocolFailure; + } finally { + clearTimeout(timer); + } + }; + const initialized = await request("initialize", { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { name: "betterwright-binary-e2e", version: "1.0.0" }, + }); + assert.equal(initialized.error, undefined, JSON.stringify(initialized)); + notify("notifications/initialized"); + return { request, close, initialized: initialized.result }; +} + +function toolResult(reply: any) { + assert.equal(reply.error, undefined, JSON.stringify(reply)); + assert.ok(Array.isArray(reply.result?.content), JSON.stringify(reply)); + return reply.result; +} + +function envelope(reply: any) { + const result = toolResult(reply); + assert.notEqual(result.isError, true, JSON.stringify(result)); + const text = result.content.find((item: any) => item.type === "text")?.text; + assert.ok(text, "MCP tool response did not include text content"); + return JSON.parse(text); +} + +export const cases: E2ECase[] = [ + { + id: "protocol-mcp-initialize-and-tools", + group: "protocol", + title: "MCP negotiates capabilities, lists public tools, and shuts down cleanly at EOF", + async run(ctx) { + const client = await mcp(ctx); + assert.equal(client.initialized.serverInfo.name, "betterwright"); + assert.match(client.initialized.serverInfo.version, /^\d+\.\d+\.\d+/); + assert.equal(client.initialized.protocolVersion, "2024-11-05"); + assert.ok(client.initialized.capabilities.tools); + const reply = await client.request("tools/list"); + assert.equal(reply.error, undefined); + const tools = reply.result.tools; + const names = tools.map((tool: any) => tool.name); + assert.equal(new Set(names).size, names.length); + for (const name of ["browser", "browser_batch", "browser_download", "browser_record", "browser_handoff", "browser_doctor", "browser_login"]) { + assert.ok(names.includes(name), `MCP did not advertise ${name}`); + } + for (const tool of tools) { + assert.equal(tool.inputSchema.type, "object"); + assert.ok(tool.description.length > 10); + } + const doctor = envelope(await client.request("tools/call", { name: "browser_doctor", arguments: {} })); + assert.equal(doctor.worker_ok, true); + assert.equal(doctor.playwright_version, doctor.playwright_pinned); + assert.ok([true, false].includes(doctor.ready)); + await client.close(); + }, + }, + { + id: "protocol-mcp-invalid-requests", + group: "protocol", + title: "MCP returns JSON-RPC errors for unknown methods and invalid request parameters", + async run(ctx) { + const client = await mcp(ctx); + const unknown = await client.request("e2e/unknown-method"); + assert.equal(unknown.error.code, -32601); + assert.ok(unknown.error.message); + const malformed = await client.request("tools/call", { name: 42 }); + assert.ok(Number.isInteger(malformed.error.code) && malformed.error.code < 0); + assert.ok(malformed.error.message); + assert.equal(malformed.result, undefined); + const missing = await client.request("tools/call", {}); + assert.ok(Number.isInteger(missing.error.code) && missing.error.code < 0); + assert.ok(missing.error.message); + assert.equal(missing.result, undefined); + const healthy = await client.request("tools/list"); + assert.ok(healthy.result.tools.length > 0); + await client.close(); + }, + }, + { + id: "protocol-mcp-invalid-tools", + group: "protocol", + title: "MCP tool validation rejects unknown tools and contradictory batch or recording arguments", + async run(ctx) { + const client = await mcp(ctx); + const invalid = [ + { name: "missing-e2e-tool", arguments: {}, expected: /Unknown tool/ }, + { name: "browser_batch", arguments: {}, expected: /requires url/ }, + { name: "browser_batch", arguments: { discover: true, operations: [] }, expected: /either discovery or operations/ }, + { name: "browser_batch", arguments: { query: "Save", operations: [] }, expected: /query requires/ }, + { name: "browser_record", arguments: { action: "invalid" }, expected: /action must be/ }, + { name: "browser_record", arguments: { action: "stop", fps: 24 }, expected: /options apply only to start and restart/ }, + ]; + for (const item of invalid) { + const result = toolResult(await client.request("tools/call", { name: item.name, arguments: item.arguments })); + assert.equal(result.isError, true); + assert.match(result.content[0].text, item.expected); + } + await client.close(); + }, + }, + { + id: "protocol-mcp-browser-roundtrip", + group: "protocol", + title: "MCP executes browser code against a local page and retains state across calls", + requiresBrowser: true, + async run(ctx) { + const server = await ctx.serve((_request, response) => { + response.writeHead(200, { "content-type": "text/html" }); + response.end("MCP local fixture

Protocol page

"); + }); + const client = await mcp(ctx); + const opened = envelope(await client.request("tools/call", { + name: "browser", + arguments: { code: `await page.goto(${JSON.stringify(server.origin)}); state.counter = 40; return { title: await page.title(), count: state.counter };`, session: "mcp-roundtrip" }, + })); + assert.equal(opened.ok, true); + assert.deepEqual(opened.result, { title: "MCP local fixture", count: 40 }); + const next = envelope(await client.request("tools/call", { + name: "browser", + arguments: { code: "state.counter += 2; return { count: state.counter, title: await page.title() };", session: "mcp-roundtrip" }, + })); + assert.equal(next.ok, true); + assert.deepEqual(next.result, { count: 42, title: "MCP local fixture" }); + const discovery = envelope(await client.request("tools/call", { + name: "browser_batch", arguments: { discover: true, session: "mcp-roundtrip" }, + })); + assert.equal(discovery.ok, true); + assert.match(JSON.stringify(discovery.result), /Read only/); + await client.close(); + }, + }, + { + id: "protocol-mcp-browser-failure-recovery", + group: "protocol", + title: "MCP carries browser failures in run envelopes and accepts the next tool call", + requiresBrowser: true, + async run(ctx) { + const client = await mcp(ctx); + const failed = envelope(await client.request("tools/call", { + name: "browser", arguments: { code: 'throw new Error("mcp-runtime-failure-marker")', session: "mcp-errors" }, + })); + assert.equal(failed.ok, false); + assert.match(JSON.stringify(failed.error), /mcp-runtime-failure-marker/); + const recovered = envelope(await client.request("tools/call", { + name: "browser", arguments: { code: "return 42", session: "mcp-errors" }, + })); + assert.equal(recovered.ok, true); + assert.equal(recovered.result, 42); + await client.close(); + }, + }, + { + id: "protocol-mcp-download-deny", + group: "protocol", + title: "MCP denies browser_download even when tool arguments claim approval", + async run(ctx) { + const client = await mcp(ctx, { env: { BETTERWRIGHT_DOWNLOAD_POLICY: "deny" } }); + const result = toolResult(await client.request("tools/call", { + name: "browser_download", arguments: { code: 'throw new Error("must-not-execute")', approved: true, approvedDownloads: true }, + })); + assert.equal(result.isError, true); + assert.match(result.content[0].text, /BETTERWRIGHT_DOWNLOAD_POLICY=deny/); + assert.doesNotMatch(result.content[0].text, /must-not-execute/); + await client.close(); + }, + }, + { + id: "protocol-exec-local-model-roundtrip", + group: "protocol", + title: "Exec calls a local OpenAI-compatible mock, executes its browser tool, and returns the answer", + requiresBrowser: true, + async run(ctx) { + const requests: any[] = []; + let pageVisits = 0; + const server = await ctx.serve(async (request, response) => { + if (request.url === "/page") { + pageVisits++; + response.writeHead(200, { "content-type": "text/html" }); + response.end("Agent mock page
Local protocol evidence: 42
"); + return; + } + if (request.url !== "/v1/chat/completions" || request.method !== "POST") { + response.writeHead(404); + response.end("Unexpected fixture route"); + return; + } + let body = ""; + for await (const chunk of request) body += chunk.toString(); + const parsed = JSON.parse(body); + requests.push(parsed); + const message = requests.length === 1 + ? { role: "assistant", content: "Read the local page.", tool_calls: [{ + id: "e2e-browser-call", type: "function", function: { + name: "browser", + arguments: JSON.stringify({ code: `await page.goto(${JSON.stringify(`http://${request.headers.host}/page`)}); return { title: await page.title(), evidence: await page.locator("main").innerText() };` }), + }, + }] } + : { role: "assistant", content: "The local page reports 42." }; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ + id: `e2e-completion-${requests.length}`, object: "chat.completion", model: "e2e-model", + choices: [{ index: 0, message, finish_reason: requests.length === 1 ? "tool_calls" : "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + })); + }); + const result = await ctx.json([ + "exec", "--stdin", "--model", "e2e-model", "--base-url", `${server.origin}/v1`, + "--protocol", "chat", "--no-daemon", "--session", "mock-agent", + ], { stdin: "Read the local fixture and report its numeric evidence.", timeoutMs: 90_000 }); + assert.equal(result.ok, true); + assert.equal(result.answer, "The local page reports 42."); + assert.equal(result.session, "mock-agent"); + assert.equal(result.reason, "answered"); + assert.equal(result.toolCalls, 1); + assert.equal(result.steps, 2); + assert.equal(requests.length, 2); + assert.ok(pageVisits >= 1); + assert.equal(requests[0].model, "e2e-model"); + assert.ok(requests[0].tools.some((tool: any) => tool.function.name === "browser")); + assert.ok(requests[0].messages.some((message: any) => message.role === "user" && message.content.includes("numeric evidence"))); + const replay = requests[1].messages.find((message: any) => message.role === "tool" && message.tool_call_id === "e2e-browser-call"); + assert.ok(replay, "The browser tool result was not sent back to the model"); + assert.match(replay.content, /Agent mock page/); + assert.match(replay.content, /Local protocol evidence: 42/); + }, + }, + { + id: "protocol-exec-local-model-failures", + group: "protocol", + title: "Exec reports HTTP and non-JSON model failures from a local endpoint without claiming success", + async run(ctx) { + const requests: string[] = []; + const server = await ctx.serve((request, response) => { + requests.push(request.url || ""); + request.resume(); + if (request.url === "/http/chat/completions") { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "e2e-unauthorized-marker" })); + } else { + response.writeHead(200, { "content-type": "text/html" }); + response.end("not a model response"); + } + }); + for (const route of ["http", "non-json"]) { + const result = await ctx.command([ + "exec", "Report fixture evidence", "--model", "e2e-model", "--base-url", `${server.origin}/${route}`, + "--protocol", "chat", "--no-daemon", "--session", `model-failure-${route}`, + ]); + assert.equal(result.code, 1, result.stdout || result.stderr); + const output = `${result.stdout}\n${result.stderr}`; + assert.match(output, route === "http" ? /401|e2e-unauthorized-marker/ : /non-JSON chat response/); + if (result.stdout.trim()) assert.equal(JSON.parse(result.stdout).ok, false); + } + assert.ok(requests.includes("/http/chat/completions")); + assert.ok(requests.includes("/non-json/chat/completions")); + }, + }, +]; diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts new file mode 100644 index 00000000..31a7e681 --- /dev/null +++ b/tests/e2e/run.ts @@ -0,0 +1,157 @@ +#!/usr/bin/env bun + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { cases as browserCases } from "./browser-cases.js"; +import { cases as cliCases } from "./cli-cases.js"; +import { type CaseResult, parseOptions, reportExitCode, runCase, selectCases } from "./harness.js"; +import { cases as lifecycleCases } from "./lifecycle-cases.js"; +import { cases as protocolCases } from "./protocol-cases.js"; +import { cases as securityCases } from "./security-cases.js"; +import type { E2ECase } from "./types.js"; + +const usage = `Usage: bun tests/e2e/run.ts [binary] [options] + +Runs isolated black-box E2E checks against betterwright on PATH or a supplied executable. +No imports from the target's source or dist directory; no real profiles or credentials. + + --binary executable (default: betterwright) + --binary-arg prepend an argument; repeat for a Bun/Node launcher + --group cli | browser | security | protocol | recording; repeatable + --filter match a case ID or title (case-sensitive) + --list list selected cases without launching the binary + --report JSON report (default: .tmp/e2e-report.json) + --timeout command deadline (default: 45000) + --case-timeout whole-case deadline (default: 180000) + --require-all fail if any selected case is skipped + --keep-work retain isolated homes and artifacts for diagnosis + --help show this help + +Examples: + bun tests/e2e/run.ts + bun tests/e2e/run.ts /absolute/path/to/betterwright + bun tests/e2e/run.ts --binary bun --binary-arg "$PWD/dist/bin/betterwright.js" + +Install the target's managed browser beforehand. BETTERWRIGHT_CHROMIUM_PATH/ROOT +and BETTERWRIGHT_FFMPEG_PATH are honored. No installers or paid services run. +This suite covers local binary behavior, not every possible input or external integration. +`; + +const untested = [ + "Real cloud-browser providers, paid API billing, provider credentials, and remote network policy boundaries.", + "Real model quality, OAuth/MFA flows, external CAPTCHA services, and third-party website behavior.", + "Local model downloads, GPU inference, installer/update downloads, and interactive onboarding.", + "Electron host attachment, OS clipboard/keystroke injection, and native personal-browser cookie extraction.", + "Every SDK/package-export contract and every OS/browser/version combination; retain the existing unit, type, and platform suites.", +]; + +async function main() { + const options = parseOptions(process.argv.slice(2)); + if (options.help) { console.log(usage); return; } + const selected = selectCases([...cliCases, ...browserCases, ...securityCases, ...protocolCases, ...lifecycleCases], options); + if (options.list) { + for (const entry of selected) console.log(`${entry.id}\t${entry.group}\t${entry.title}`); + console.log(`${selected.length} cases`); + return; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), "bw-e2e-")); + const results: CaseResult[] = []; + const startedAt = new Date().toISOString(); + const abort = new AbortController(); + const interrupt = () => abort.abort(); + process.once("SIGINT", interrupt); + process.once("SIGTERM", interrupt); + let targetVersion = ""; + const saveReport = (complete: boolean) => { + const summary = { + passed: results.filter((entry) => entry.status === "passed").length, + failed: results.filter((entry) => entry.status === "failed").length, + skipped: results.filter((entry) => entry.status === "skipped").length, + }; + const report = { + schemaVersion: 1, + target: { binary: options.binary, version: targetVersion }, + platform: { os: process.platform, arch: process.arch }, + startedAt, + updatedAt: new Date().toISOString(), + complete, + outcome: !complete ? "incomplete" : summary.failed ? "failed" : summary.skipped ? "partial" : "passed", + interrupted: abort.signal.aborted, + selectedCases: selected.length, + requireAll: options.requireAll, + summary, + untested, + results, + }; + fs.mkdirSync(path.dirname(options.report), { recursive: true, mode: 0o700 }); + const temporary = `${options.report}.${process.pid}.tmp`; + fs.writeFileSync(temporary, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + fs.renameSync(temporary, options.report); + return summary; + }; + const execute = async (entry: E2ECase) => { + const result = await runCase(entry, options, root, abort.signal); + results.push(result); + console.log(`${result.status.toUpperCase().padEnd(7)} ${entry.id} (${result.durationMs}ms)`); + if (result.error || result.reason) console.log(` ${result.error || result.reason}`); + saveReport(false); + return result.status === "passed"; + }; + try { + console.log(`Testing ${options.binary}: ${selected.length} selected cases`); + const targetReady = await execute({ + id: "preflight.binary", + group: "cli", + title: "Target binary starts and reports its version", + async run(context) { + const result = await context.command(["--version"]); + assert.equal(result.code, 0, "Target --version must succeed"); + assert.match(result.stdout.trim(), /\d+\.\d+\.\d+/); + targetVersion = result.stdout.trim(); + }, + }); + let browserReady = false; + if (targetReady && selected.some((entry) => entry.requiresBrowser) && !abort.signal.aborted) { + browserReady = await execute({ + id: "preflight.browser", + group: "browser", + title: "Target launches its browser and executes a snippet", + async run(context) { + const result = await context.run("return 6 * 7;", { args: ["--no-daemon", "--close"] }); + assert.equal(result.ok, true, result.error || "Install the target's managed browser before running this suite"); + assert.equal(result.result, 42); + }, + }); + } + for (const entry of selected) { + const reason = abort.signal.aborted ? "Run interrupted" : !targetReady + ? "Target binary preflight failed" : entry.requiresBrowser && !browserReady + ? "Browser preflight failed; see preflight.browser" : null; + if (reason) { + results.push({ id: entry.id, group: entry.group, title: entry.title, status: "skipped", reason, durationMs: 0 }); + console.log(`SKIPPED ${entry.id}: ${reason}`); + } else { + await execute(entry); + } + } + const summary = saveReport(!abort.signal.aborted); + console.log(`\n${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped`); + console.log(`Report: ${options.report}`); + console.log("Scope: deterministic local binary checks. Untested external/platform integrations are listed in the report."); + if (options.keepWork || results.some((entry) => entry.workDir)) console.log(`Retained work: ${root}`); + process.exitCode = abort.signal.aborted ? 130 : reportExitCode(results, options.requireAll); + } finally { + process.off("SIGINT", interrupt); + process.off("SIGTERM", interrupt); + if (!options.keepWork && !results.some((entry) => entry.workDir)) { + fs.rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + } +} + +void main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/tests/e2e/security-cases.ts b/tests/e2e/security-cases.ts new file mode 100644 index 00000000..fc409070 --- /dev/null +++ b/tests/e2e/security-cases.ts @@ -0,0 +1,741 @@ +import assert from "node:assert/strict"; +import { mkdir, readdir, readFile, stat, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + confinedUpstream, + credentialSite, + denied, + excludesSecret, + result, + savedLogin, + syntheticSecret, +} from "./security-fixtures.js"; +import type { E2ECase } from "./types.js"; + +export const cases: E2ECase[] = [ + { + id: "security.vm-node-globals", + group: "security", + title: "Snippet globals cannot access Node, host processes, or module loading", + requiresBrowser: true, + async run(ctx) { + const envelope = await ctx.run(` + await page.setContent('

VM fixture

'); + const globals = [typeof process, typeof require, typeof module, typeof Buffer, typeof fs]; + const failures = []; + for (const operation of [() => process.cwd(), () => require('node:fs'), () => import('node:fs')]) { + try { await operation(); failures.push(false); } catch { failures.push(true); } + } + await page.locator('#marker').evaluate(element => element.textContent = 'Browser still usable'); + return {globals, failures, heading: await page.locator('#marker').textContent()}; + `); + assert.deepEqual(result(envelope), { + globals: Array(5).fill("undefined"), failures: [true, true, true], heading: "Browser still usable", + }); + }, + }, + { + id: "security.vm-private-routing", + group: "security", + title: "CDP, private channels, routing, and context mutation stay inaccessible", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + const envelope = await ctx.run(` + const hidden = [page.context, page._channel, page._connection, page.request, page.route, + page.routeWebSocket, page.unroute, page.exposeFunction, page.exposeBinding, + context.browser, context.newCDPSession, context.newPage, context.cookies, + context.storageState, context.addCookies, context.clearCookies, context.close, + context.tracing, context.route, page.locator('body')._frame]; + let eventDenied = false; + try { page.on('request', () => {}); } catch { eventDenied = true; } + await page.goto(${JSON.stringify(site.origin)}); + return {hidden: hidden.map(value => typeof value), eventDenied, title: await page.title()}; + `); + assert.deepEqual(result(envelope), { + hidden: Array(20).fill("undefined"), eventDenied: true, title: "Security fixture", + }); + assert.equal(site.state.requests.get("/"), 1); + }, + }, + { + id: "security.vm-constructor-prototype", + group: "security", + title: "Constructor chains cannot compile host code or expose wrapped prototypes", + requiresBrowser: true, + async run(ctx) { + const envelope = await ctx.run(` + await page.setContent('

Prototype fixture

'); + const attempts = [() => eval('typeof process'), () => Function('return process')(), + () => ({}).constructor.constructor('return process')(), + () => URL.constructor('return process')(), + () => new URLSearchParams().constructor.constructor('return process')()]; + const blocked = attempts.map(operation => { try { operation(); return false; } catch { return true; } }); + return {blocked, pagePrototype: Object.getPrototypeOf(page), + locatorPrototype: Object.getPrototypeOf(page.locator('h1')), + hidden: [typeof page.constructor, typeof page.__proto__, typeof page.locator('h1').constructor], + heading: await page.locator('h1').textContent()}; + `); + assert.deepEqual(result(envelope), { + blocked: Array(5).fill(true), pagePrototype: null, locatorPrototype: null, + hidden: Array(3).fill("undefined"), heading: "Prototype fixture", + }); + }, + }, + { + id: "security.request-response-headers", + group: "security", + title: "Request and response wrappers hide raw cookie headers and private channels", + requiresBrowser: true, + async run(ctx) { + let cookieRequests = 0; + const cookie = syntheticSecret(ctx); + const site = await ctx.serve((request, response) => { + if (request.headers.cookie === `fixture=${cookie}`) cookieRequests += 1; + response.setHeader("set-cookie", `fixture=${cookie}; HttpOnly; SameSite=Lax; Path=/`); + response.setHeader("x-fixture", "public-metadata"); + response.end('Header fixture'); + }); + const envelope = await ctx.run(` + await page.goto(${JSON.stringify(site.origin)}); + const pendingRequest = page.waitForRequest(${JSON.stringify(`${site.origin}/again`)}); + const response = await page.goto(${JSON.stringify(`${site.origin}/again`)}); + const request = await pendingRequest; + return { + status: response.status(), + requestMethods: ['allHeaders', 'headersArray', 'headerValue', '_channel'].map(key => typeof request[key]), + responseMethods: ['allHeaders', 'headersArray', 'headerValue', 'headerValues', '_channel', 'request'].map(key => typeof response[key]), + requestHeaders: await request.headers(), responseHeaders: await response.headers(), + title: await page.title(), + }; + `); + const value = result(envelope); + assert.equal(value.status, 200); + assert.equal(value.title, "Header fixture"); + assert.deepEqual(value.requestMethods, Array(4).fill("undefined")); + assert.deepEqual(value.responseMethods, Array(6).fill("undefined")); + assert.equal(Object.hasOwn(value.requestHeaders, "cookie"), false); + assert.equal(Object.hasOwn(value.responseHeaders, "set-cookie"), false); + assert.equal(value.responseHeaders["x-fixture"], "public-metadata"); + assert.ok(cookieRequests >= 1); + excludesSecret(envelope, cookie); + }, + }, + { + id: "security.internal-navigation", + group: "security", + title: "Host file and browser-internal navigations fail without disturbing a safe page", + requiresBrowser: true, + async run(ctx) { + const sentinel = syntheticSecret(ctx); + const file = path.join(ctx.workDir, "private-document.html"); + await writeFile(file, `

${sentinel}

`); + for (const url of [pathToFileURL(file).href, "chrome://version", "chrome://settings"]) { + result(await ctx.run("await page.setContent('

Safe page

'); return page.locator('h1').textContent();")); + const blocked = await ctx.run(`await page.goto(${JSON.stringify(url)}); return page.locator('body').innerText();`); + denied(blocked, /blocked|scheme|not allowed|unsupported|denied/i); + excludesSecret(blocked, sentinel); + assert.equal(result(await ctx.run("return page.locator('h1').textContent();")), "Safe page"); + } + assert.equal(result(await ctx.run("await page.goto('data:text/html,

Safe data URL

'); return page.locator('h1').textContent();")), "Safe data URL"); + }, + }, + { + id: "security.upload-paths", + group: "security", + title: "File uploads reject outside paths, traversal, and symlinks but allow artifacts", + requiresBrowser: true, + async run(ctx) { + const outside = path.join(ctx.home, "outside-upload.txt"); + const secret = syntheticSecret(ctx); + await mkdir(ctx.home, { recursive: true }); + await writeFile(outside, secret); + const allowed = result(await ctx.run("await page.setContent(''); return artifactPath('allowed.txt');")); + await writeFile(allowed, "allowed local artifact"); + const linked = path.join(path.dirname(allowed), "outside-link.txt"); + await symlink(outside, linked); + const traversal = `${path.dirname(allowed)}/../../outside-upload.txt`; + for (const candidate of [outside, traversal, linked]) { + const blocked = await ctx.run(`await page.locator('#upload').setInputFiles(${JSON.stringify(candidate)});`); + denied(blocked, /artifact directory/i); + excludesSecret(blocked, secret); + assert.equal(result(await ctx.run("return page.locator('#upload').evaluate(element => element.files.length);")), 0); + } + const uploaded = await ctx.run(` + await page.locator('#upload').setInputFiles(${JSON.stringify(allowed)}); + return page.locator('#upload').evaluate(element => element.files[0].text()); + `); + assert.equal(result(uploaded), "allowed local artifact"); + result(await ctx.run("await page.locator('#upload').setInputFiles([]); return true;")); + const chooser = await ctx.run(` + const pending = page.waitForEvent('filechooser'); + await page.locator('#upload').click(); + await (await pending).setFiles(${JSON.stringify(outside)}); + `); + denied(chooser, /artifact directory/i); + assert.equal(result(await ctx.run("return page.locator('#upload').evaluate(element => element.files.length);")), 0); + }, + }, + { + id: "security.script-read-paths", + group: "security", + title: "Script and stylesheet paths cannot read files outside the artifact directory", + requiresBrowser: true, + async run(ctx) { + const script = path.join(ctx.workDir, "outside.js"); + const style = path.join(ctx.workDir, "outside.css"); + await writeFile(script, "globalThis.outsideScriptRan = true;"); + await writeFile(style, "body { --outside-style: read; }"); + const site = await credentialSite(ctx); + result(await ctx.run(`await page.goto(${JSON.stringify(site.origin)}); return true;`)); + for (const code of [ + `await page.addInitScript({path: ${JSON.stringify(script)}});`, + `await page.addScriptTag({path: ${JSON.stringify(script)}});`, + `await page.addStyleTag({path: ${JSON.stringify(style)}});`, + ]) denied(await ctx.run(code), /artifact directory/i); + const before = result(await ctx.run(` + await page.reload(); + return page.evaluate(() => ({ran: window.outsideScriptRan === true, + style: getComputedStyle(document.body).getPropertyValue('--outside-style')})); + `)); + assert.deepEqual(before, { ran: false, style: "" }); + const allowed = result(await ctx.run("return artifactPath('allowed-init.js');")); + await writeFile(allowed, "globalThis.allowedScriptRan = 'local artifact';"); + assert.equal(result(await ctx.run(` + await page.addInitScript({path: ${JSON.stringify(allowed)}}); + await page.reload(); return page.evaluate(() => window.allowedScriptRan); + `)), "local artifact"); + }, + }, + { + id: "security.artifact-write-paths", + group: "security", + title: "Browser writes reject traversal and keep screenshots and PDFs inside artifacts", + requiresBrowser: true, + async run(ctx) { + const outside = path.join(ctx.home, "outside.pdf"); + await mkdir(ctx.home, { recursive: true }); + await writeFile(outside, "do not overwrite"); + const allowed = result(await ctx.run("await page.setContent('

PDF fixture

'); return artifactPath('allowed.pdf');")); + for (const candidate of [outside, `${path.dirname(allowed)}/../../outside.pdf`]) { + denied(await ctx.run(`await page.pdf({path: ${JSON.stringify(candidate)}});`), /artifactPath|screenshot|artifact/i); + assert.equal(await readFile(outside, "utf8"), "do not overwrite"); + } + result(await ctx.run(`await page.pdf({path: ${JSON.stringify(allowed)}}); return true;`)); + assert.equal((await readFile(allowed)).subarray(0, 5).toString(), "%PDF-"); + const envelope = await ctx.run("return screenshot({name: '../../escaped-proof.png'});"); + const screenshot = result(envelope); + const relative = path.relative(path.join(ctx.home, "artifacts"), screenshot.path); + assert.ok(!relative.startsWith("..") && !path.isAbsolute(relative)); + assert.deepEqual([...(await readFile(screenshot.path)).subarray(0, 8)], [137, 80, 78, 71, 13, 10, 26, 10]); + assert.ok(envelope.artifacts.some((artifact) => artifact.path === screenshot.path)); + }, + }, + { + id: "security.metadata-floor", + group: "security", + title: "Metadata addresses remain blocked despite allowlisting and never reach a local upstream", + requiresBrowser: true, + async run(ctx) { + const proxy = await confinedUpstream(ctx); + const args = ["--no-launch-identity", "--upstream-proxy", proxy.origin, + "--allow-host", "169.254.169.254", "--allow-host", "169.254.170.2", + "--allow-host", "100.100.100.200", "--allow-host", "metadata.google.internal"]; + assert.equal(result(await ctx.run(`await page.goto(${JSON.stringify(proxy.origin)}); return page.title();`, { args })), "Confined upstream"); + const probes = ["169.254.169.254", "169.254.170.2", "100.100.100.200", "metadata.google.internal"]; + for (const host of probes) { + denied(await ctx.run(`await page.goto(${JSON.stringify(`http://${host}/e2e-metadata`)});`, { args }), /metadata|blocked|denied/i); + } + assert.ok(proxy.requests.some((request) => request.startsWith("GET / "))); + for (const host of probes) assert.equal(proxy.authorities.includes(`${host}:80`), false); + assert.equal(result(await ctx.run(`await page.goto(${JSON.stringify(proxy.origin)}); return page.title();`, { args })), "Confined upstream"); + }, + }, + { + id: "security.private-network-policy", + group: "security", + title: "Private-network denial stops transport while a separate default policy reaches the fixture", + requiresBrowser: true, + async run(ctx) { + const proxy = await confinedUpstream(ctx); + const args = ["--no-launch-identity", "--upstream-proxy", proxy.origin]; + const url = "http://10.77.0.123/private-fixture"; + denied(await ctx.run(`await page.goto(${JSON.stringify(url)});`, { + profile: "private-denied", args: [...args, "--block-private-network"], + }), /private|blocked|denied/i); + assert.equal(proxy.authorities.includes("10.77.0.123:80"), false); + const allowed = await ctx.run(`await page.goto(${JSON.stringify(url)}); return page.title();`, { + profile: "private-allowed", args, + }); + assert.equal(allowed.ok, true, JSON.stringify({error: allowed.error, authorities: proxy.authorities, requests: proxy.requests})); + assert.equal(allowed.result, "Confined upstream"); + assert.ok(proxy.authorities.includes("10.77.0.123:80")); + assert.ok(proxy.requests.includes("GET /private-fixture HTTP/1.1")); + }, + }, + { + id: "security.loopback-policy", + group: "security", + title: "Loopback denial covers numeric, localhost, and mapped IPv6 URLs with an allow-host control", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + const port = new URL(site.origin).port; + for (const origin of [site.origin, site.alternateOrigin, `http://[::ffff:127.0.0.1]:${port}`]) { + denied(await ctx.run(`await page.goto(${JSON.stringify(`${origin}/denied`)});`, { + profile: "loopback-denied", args: ["--block-loopback"], + }), /loopback|blocked|denied/i); + } + assert.equal(site.state.requests.get("/denied") || 0, 0); + assert.equal(result(await ctx.run(`await page.goto(${JSON.stringify(`${site.origin}/allowed`)}); return page.title();`, { + profile: "loopback-allowed", args: ["--block-loopback", "--allow-host", new URL(site.origin).host], + })), "Security fixture"); + assert.equal(site.state.requests.get("/allowed"), 1); + }, + }, + { + id: "security.blocked-host-subresources", + group: "security", + title: "A blocked local hostname cannot be fetched or reached through a redirect", + requiresBrowser: true, + async run(ctx) { + let blockedHits = 0; + let redirectHits = 0; + const target = await ctx.serve((_request, response) => { + blockedHits += 1; + response.end("blocked target reached"); + }); + const site = await ctx.serve((request, response) => { + if (request.url === "/redirect") { + redirectHits += 1; + response.writeHead(302, { location: `${target.alternateOrigin}/denied` }); + response.end(); + return; + } + response.setHeader("content-type", "text/html"); + response.end('Allowed local page'); + }); + const args = ["--block-host", "localhost", "--allow-host", "localhost"]; + const fetch = result(await ctx.run(` + await page.goto(${JSON.stringify(site.origin)}); + const failed = await page.evaluate(async (url) => { + try { await fetch(url, {mode: 'no-cors'}); return false; } catch { return true; } + }, ${JSON.stringify(`${target.alternateOrigin}/denied`)}); + return {failed, title: await page.title()}; + `, { args })); + assert.deepEqual(fetch, { failed: true, title: "Allowed local page" }); + assert.equal(blockedHits, 0); + denied(await ctx.run(`await page.goto(${JSON.stringify(`${site.origin}/redirect`)});`, { args }), /blocked|denied|ERR_FAILED|ERR_SOCKS_CONNECTION_FAILED/i); + assert.equal(redirectHits, 1); + assert.equal(blockedHits, 0); + assert.equal(result(await ctx.run(`await page.goto(${JSON.stringify(`${target.origin}/positive`)}); return page.locator('body').textContent();`, { args })), "blocked target reached"); + assert.ok(blockedHits >= 1); + }, + }, + { + id: "security.vault-metadata-storage", + group: "security", + title: "Saved credentials expose metadata only and are encrypted in owner-only vault files", + requiresBrowser: true, + async run(ctx) { + const { secret, record, site } = await savedLogin(ctx); + const accounts = result(await ctx.run("return credentials.list({text: 'E2E account', category: 'login'});")); + assert.equal(accounts.length, 1); + assert.equal(accounts[0].id, record.id); + assert.equal(accounts[0].origin, site.origin); + assert.equal(accounts[0].username, "fixture-user"); + for (const key of ["secret", "password", "fields", "notes"]) assert.equal(Object.hasOwn(accounts[0], key), false); + const listed = await ctx.json(["vault", "list", "--json"]); + const shown = await ctx.json(["vault", "show", record.id, "--json"]); + const audit = await ctx.json(["vault", "audit", "--json"]); + excludesSecret([listed, shown, audit], secret); + assert.equal(listed.credentials.length, 1); + assert.equal(listed.credentials[0].id, record.id); + const root = path.join(ctx.home, "vault"); + const encrypted = await readFile(path.join(root, "vault.enc")); + assert.equal(encrypted.includes(Buffer.from(secret)), false); + assert.equal((await readFile(path.join(root, "audit.jsonl"), "utf8")).includes(secret), false); + assert.equal((await stat(root)).mode & 0o077, 0); + for (const file of ["vault.key", "vault.enc", "audit.jsonl"]) { + assert.equal((await stat(path.join(root, file))).mode & 0o077, 0); + } + const refused = await ctx.command(["vault", "show", record.id, "--reveal"]); + assert.equal(refused.code, 1); + excludesSecret(refused, secret); + assert.match(refused.stderr, /terminal|TTY/i); + }, + }, + { + id: "security.credential-fill-submit", + group: "security", + title: "Inspect and trusted fill omit secrets and submit only on an explicit request", + requiresBrowser: true, + async run(ctx) { + const { secret, record, site } = await savedLogin(ctx); + const inspection = result(await ctx.run("return credentials.inspect();")); + assert.equal(inspection.status, "ready"); + assert.equal(inspection.fields.password.autocomplete, "current-password"); + excludesSecret(inspection, secret); + const filled = await ctx.run(`return credentials.fill({id: ${JSON.stringify(record.id)}});`); + assert.deepEqual(result(filled).filled, ["username", "password"]); + assert.equal(result(filled).submitted, false); + excludesSecret(filled, secret); + assert.deepEqual(result(await ctx.run(`return page.evaluate(() => ({ + username: document.querySelector('#username').value, + length: document.querySelector('#password').value.length, submits: window.submits, + }));`)), { username: "fixture-user", length: secret.length, submits: 0 }); + assert.equal(site.state.submissions, 0); + const submitted = await ctx.run(` + const fill = await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); + return {fill, heading: await page.locator('#status').textContent()}; + `); + assert.equal(result(submitted).fill.submitted, true); + assert.equal(result(submitted).heading, "Signed in"); + assert.equal(site.state.accepted, 1); + assert.equal(site.state.rejected, 0); + excludesSecret(submitted, secret); + }, + }, + { + id: "security.credential-output-redaction", + group: "security", + title: "Handled secrets are scrubbed from result values, keys, console history, and errors", + requiresBrowser: true, + async run(ctx) { + const { secret, record } = await savedLogin(ctx); + const envelope = await ctx.run(` + await credentials.fill({id: ${JSON.stringify(record.id)}}); + const value = await page.locator('#password').inputValue(); + console.log('synthetic secret', value); + const consoleReady = page.waitForEvent('console', { + predicate: message => message.type() === 'warning' && message.text().startsWith('e2e-redaction:'), + timeout: 5000, + }); + await page.evaluate(() => console.warn('e2e-redaction:' + document.querySelector('#password').value)); + const observed = (await consoleReady).text(); + return {value, observed, nested: [{password: value}], [value]: value, + history: (await page.consoleMessages()).map(message => message.text())}; + `); + const value = result(envelope); + excludesSecret(envelope, secret); + assert.match(value.value, /redact/i, "the returned DOM password must be redacted"); + assert.ok((envelope.console || []).length >= 1, "snippet console output must be captured"); + assert.match(JSON.stringify(envelope.console), /redact/i, "snippet console output must redact the password"); + assert.match(value.observed, /e2e-redaction:.*redact/i, "the observed page console event must redact the password"); + assert.ok(value.history.some((line) => /e2e-redaction:.*redact/i.test(line)), "page console history must contain the synchronized redacted warning"); + const error = await ctx.run("throw new Error('credential-error:' + await page.locator('#password').inputValue());"); + denied(error, /credential-error:.*redact/i); + excludesSecret(error, secret); + assert.equal(result(await ctx.run("return page.locator('#password').evaluate(element => element.value.length);")), secret.length); + }, + }, + { + id: "security.credential-exact-origin", + group: "security", + title: "Exact-origin credentials cannot fill another hostname or port and recover at the original site", + requiresBrowser: true, + async run(ctx) { + const { record, site, secret } = await savedLogin(ctx); + const other = await credentialSite(ctx, secret); + for (const origin of [site.alternateOrigin, other.origin]) { + const listed = await ctx.run(`await page.goto(${JSON.stringify(`${origin}/login`)}); return credentials.list();`); + assert.deepEqual(result(listed), []); + denied(await ctx.run(`await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true});`), /match|not found|available|scope/i); + assert.equal(result(await ctx.run("return page.locator('#password').inputValue();")), ""); + } + assert.equal(site.state.submissions, 0); + assert.equal(other.state.submissions, 0); + const recovered = await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/login`)}); + await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); return true; + `); + assert.equal(result(recovered), true); + assert.equal(site.state.accepted, 1); + excludesSecret(recovered, secret); + }, + }, + { + id: "security.credential-host-and-never", + group: "security", + title: "Host matching permits a second port while never-matching records remain metadata-only", + requiresBrowser: true, + async run(ctx) { + const { record, secret } = await savedLogin(ctx, "host"); + const other = await credentialSite(ctx, secret); + const filled = await ctx.run(` + await page.goto(${JSON.stringify(`${other.origin}/login`)}); + await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); + return credentials.save({username: 'never-user', password: ${JSON.stringify(secret)}, matchMode: 'never'}); + `); + const never = result(filled); + assert.equal(other.state.accepted, 1); + const accounts = result(await ctx.run("return credentials.list();")); + assert.deepEqual(accounts.map((account) => account.id), [record.id]); + result(await ctx.run("await page.locator('#password').fill(''); return true;")); + denied(await ctx.run(`await credentials.fill({id: ${JSON.stringify(never.id)}});`), /match|not found|available|scope/i); + assert.equal(result(await ctx.run("return page.locator('#password').inputValue();")), ""); + const owner = await ctx.json(["vault", "list", "--json"]); + assert.equal(owner.credentials.length, 2); + assert.ok(owner.credentials.some((account) => account.id === never.id)); + excludesSecret(owner, secret); + }, + }, + { + id: "security.credential-overwrite-metadata", + group: "security", + title: "Snippet secret overwrites fail while metadata updates and removal still work", + requiresBrowser: true, + async run(ctx) { + const { record, secret, site } = await savedLogin(ctx); + const replacement = syntheticSecret(ctx); + for (const code of [ + `await credentials.save({username: 'fixture-user', password: ${JSON.stringify(replacement)}});`, + `await credentials.update({id: ${JSON.stringify(record.id)}, password: ${JSON.stringify(replacement)}});`, + `await credentials.update({id: ${JSON.stringify(record.id)}, notes: ${JSON.stringify(replacement)}});`, + ]) { + const blocked = await ctx.run(code); + denied(blocked, /overwrite|replace|secret|rotation|generateAndFill/i); + excludesSecret(blocked, replacement); + } + const metadata = result(await ctx.run(`return credentials.update({id: ${JSON.stringify(record.id)}, label: 'Renamed fixture'});`)); + assert.equal(metadata.label, "Renamed fixture"); + const filled = await ctx.run(` + await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); + return credentials.remove({id: ${JSON.stringify(record.id)}}); + `); + result(filled); + excludesSecret(filled, secret); + assert.equal(site.state.accepted, 1); + assert.deepEqual(result(await ctx.run("return credentials.list();")), []); + assert.deepEqual((await ctx.json(["vault", "list", "--json"])).credentials, []); + }, + }, + { + id: "security.credential-errors-recovery", + group: "security", + title: "Missing site, missing record, and ambiguous account errors leave forms unchanged", + requiresBrowser: true, + async run(ctx) { + denied(await ctx.run("return credentials.list();"), /HTTP|origin|site|navigate/i); + const { record, site, secret } = await savedLogin(ctx); + denied(await ctx.run("await credentials.fill({id: 'e2e-missing-record'});"), /match|not found|available/i); + const second = syntheticSecret(ctx); + result(await ctx.run(`return credentials.save({username: 'other-user', password: ${JSON.stringify(second)}});`)); + denied(await ctx.run("await credentials.fill({});"), /multiple|ambiguous|select|specify/i); + assert.deepEqual(result(await ctx.run(`return page.evaluate(() => [ + document.querySelector('#username').value, document.querySelector('#password').value, window.submits, + ]);`)), ["", "", 0]); + const recovered = await ctx.run(` + await credentials.fill({id: ${JSON.stringify(record.id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); return true; + `); + assert.equal(result(recovered), true); + excludesSecret(recovered, secret); + assert.equal(site.state.accepted, 1); + }, + }, + { + id: "security.credential-ambiguous-form", + group: "security", + title: "Ambiguous forms fail closed and explicit selectors fill only the selected form", + requiresBrowser: true, + async run(ctx) { + const { record, site, secret } = await savedLogin(ctx); + result(await ctx.run(`await page.goto(${JSON.stringify(`${site.origin}/ambiguous`)}); return true;`)); + denied(await ctx.run(`await credentials.fill({id: ${JSON.stringify(record.id)}});`), /ambiguous|multiple|selector|form/i); + assert.deepEqual(result(await ctx.run("return page.locator('input[type=password]').evaluateAll(inputs => inputs.map(input => input.value.length));")), [0, 0]); + const filled = await ctx.run(` + await credentials.fill({id: ${JSON.stringify(record.id)}, usernameSelector: '#username-two', passwordSelector: '#password-two'}); + return page.locator('input[type=password]').evaluateAll(inputs => inputs.map(input => input.value.length)); + `); + assert.deepEqual(result(filled), [0, secret.length]); + assert.equal(site.state.submissions, 0); + excludesSecret(filled, secret); + }, + }, + { + id: "security.generated-commit", + group: "security", + title: "Generated credentials remain pending until verified signup success and explicit commit", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + const generated = result(await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/signup`)}); + return credentials.generateAndFill({username: 'generated-user', matchMode: 'exact-origin'}); + `)); + assert.ok(generated.pendingId); + assert.equal(generated.expired, false); + assert.equal(generated.origin, site.origin); + assert.equal(generated.matchMode, "exact-origin"); + assert.equal(Object.hasOwn(generated, "secret"), false); + const state = result(await ctx.run(`return { + accounts: await credentials.list(), pending: await credentials.listPending(), + form: await page.evaluate(() => ({length: document.querySelector('#password').value.length, + matches: document.querySelector('#password').value === document.querySelector('#confirm').value, + submits: window.submits})), + };`)); + assert.deepEqual(state.accounts, []); + assert.equal(state.pending.length, 1); + assert.equal(state.pending[0].pendingId, generated.pendingId); + assert.ok(state.form.length >= 16); + assert.equal(state.form.matches, true); + assert.equal(state.form.submits, 0); + denied(await ctx.run("await credentials.fill({username: 'generated-user'});"), /match|not found|available|form|password/i); + const committed = result(await ctx.run(` + await page.getByRole('button', {name: 'Create account'}).click(); + await page.getByRole('heading', {name: 'Account created'}).waitFor(); + return credentials.commitGenerated({pendingId: ${JSON.stringify(generated.pendingId)}}); + `)); + assert.equal(committed.committed, true); + assert.equal(site.state.accepted, 1); + assert.deepEqual(result(await ctx.run("return credentials.listPending();")), []); + const accounts = result(await ctx.run("return credentials.list();")); + assert.equal(accounts.length, 1); + assert.equal(accounts[0].username, "generated-user"); + assert.equal(accounts[0].matchMode, "exact-origin"); + const login = await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/login`)}); + await credentials.fill({id: ${JSON.stringify(accounts[0].id)}, submit: true}); + await page.getByRole('heading', {name: 'Signed in'}).waitFor(); return true; + `); + assert.equal(result(login), true); + assert.equal(site.state.accepted, 2); + }, + }, + { + id: "security.generated-discard", + group: "security", + title: "Discard removes only the requested pending credential and invalid finalization is recoverable", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + const generated = result(await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/signup`)}); + return credentials.generateAndFill({username: 'discard-user', matchMode: 'exact-origin'}); + `)); + denied(await ctx.run("await credentials.commitGenerated({});"), /pendingId/i); + denied(await ctx.run("await credentials.discardGenerated({pendingId: 'missing-pending-id'});"), /pending|not found/i); + const pending = result(await ctx.run("return credentials.listPending();")); + assert.deepEqual(pending.map((entry) => entry.pendingId), [generated.pendingId]); + const discarded = result(await ctx.run(`return credentials.discardGenerated({pendingId: ${JSON.stringify(generated.pendingId)}});`)); + assert.equal(discarded.discarded, true); + assert.deepEqual(result(await ctx.run("return {accounts: await credentials.list(), pending: await credentials.listPending()};")), { accounts: [], pending: [] }); + denied(await ctx.run(`await credentials.commitGenerated({pendingId: ${JSON.stringify(generated.pendingId)}});`), /pending|not found/i); + assert.equal(site.state.submissions, 0); + const owner = await ctx.json(["vault", "list", "--json"]); + assert.deepEqual(owner.credentials, []); + assert.deepEqual(owner.pendingCredentials, []); + }, + }, + { + id: "security.generated-error-recovery", + group: "security", + title: "A failed post-generation snippet retains recoverable pending metadata across sessions", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + const failed = await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/signup`)}); + await credentials.generateAndFill({username: 'recover-user', matchMode: 'exact-origin'}); + throw new Error('synthetic post-generation failure'); + `); + denied(failed, /synthetic post-generation failure/); + assert.ok(failed.pendingCredential?.pendingId); + const pendingId = failed.pendingCredential.pendingId; + const recovered = result(await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/signup`)}); + return {pending: await credentials.listPending(), active: await credentials.list()}; + `, { session: "pending-recovery" })); + assert.deepEqual(recovered.active, []); + assert.deepEqual(recovered.pending.map((entry) => entry.pendingId), [pendingId]); + assert.equal(Object.hasOwn(recovered.pending[0], "secret"), false); + const discarded = result(await ctx.run(`return credentials.discardGenerated({pendingId: ${JSON.stringify(pendingId)}});`, { session: "pending-recovery" })); + assert.equal(discarded.discarded, true); + assert.deepEqual(result(await ctx.run("return credentials.listPending();")), []); + assert.equal(site.state.submissions, 0); + }, + }, + { + id: "security.generated-rotation", + group: "security", + title: "Generated rotation fills the current secret and preserves the existing record and scope", + requiresBrowser: true, + async run(ctx) { + const { record, site, secret } = await savedLogin(ctx); + const generated = await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/rotate`)}); + return credentials.generateAndFill({id: ${JSON.stringify(record.id)}}); + `); + const pending = result(generated); + excludesSecret(generated, secret); + const state = result(await ctx.run(`return page.evaluate(() => ({ + currentLength: document.querySelector('#current').value.length, + newLength: document.querySelector('#password').value.length, + changed: document.querySelector('#current').value !== document.querySelector('#password').value, + matches: document.querySelector('#password').value === document.querySelector('#confirm').value, + }));`)); + assert.equal(state.currentLength, secret.length); + assert.ok(state.newLength >= 16); + assert.equal(state.changed, true); + assert.equal(state.matches, true); + const committed = result(await ctx.run(` + await page.getByRole('button', {name: 'Change password'}).click(); + await page.getByRole('heading', {name: 'Password changed'}).waitFor(); + return credentials.commitGenerated({pendingId: ${JSON.stringify(pending.pendingId)}}); + `)); + assert.equal(committed.committed, true); + assert.equal(committed.id, record.id); + assert.equal(site.state.accepted, 1); + const accounts = result(await ctx.run("return credentials.list();")); + assert.equal(accounts.length, 1); + assert.equal(accounts[0].id, record.id); + assert.equal(accounts[0].matchMode, "exact-origin"); + assert.equal(accounts[0].origin, site.origin); + assert.deepEqual(result(await ctx.run("return credentials.listPending();")), []); + }, + }, + { + id: "security.concurrent-download-approval", + group: "security", + title: "Concurrent CLI sessions do not share a per-run download approval", + requiresBrowser: true, + async run(ctx) { + const site = await credentialSite(ctx); + await Promise.all(["approved", "unapproved"].map(async (session) => { + const opened = await ctx.run(` + await page.goto(${JSON.stringify(site.origin)}); + await page.evaluate(value => window.sessionMarker = value, ${JSON.stringify(session)}); + return page.title(); + `, { session }); + assert.equal(result(opened), "Security fixture"); + })); + const [approved, unapproved] = await Promise.all([ + ctx.run(`await page.locator('#download').click(); await page.waitForTimeout(1400); + return page.evaluate(() => window.sessionMarker);`, { session: "approved", args: ["--approve-downloads"] }), + ctx.run(`await page.waitForTimeout(200); await page.locator('#download').click(); + await page.waitForTimeout(800); return page.evaluate(() => window.sessionMarker);`, { session: "unapproved" }), + ]); + assert.equal(result(approved), "approved"); + assert.equal(result(unapproved), "unapproved"); + const files = (approved.artifacts || []).filter((artifact) => artifact.kind === "download"); + assert.equal(files.length, 1); + assert.equal((unapproved.artifacts || []).filter((artifact) => artifact.kind === "download").length, 0); + assert.equal(await readFile(files[0].path, "utf8"), "local security download fixture"); + assert.ok((site.state.requests.get("/download") || 0) >= 2); + const artifactRoot = path.join(ctx.home, "artifacts"); + await mkdir(artifactRoot, { recursive: true }); + const entries = await readdir(artifactRoot, { recursive: true, withFileTypes: true }); + const retained = entries.filter((entry) => entry.isFile()); + assert.equal(retained.length, 1, "only the approved session may retain downloaded bytes"); + const later = await ctx.run("await page.locator('#download').click(); await page.waitForTimeout(400); return true;", { session: "approved" }); + assert.equal(result(later), true); + assert.equal((later.artifacts || []).filter((artifact) => artifact.kind === "download").length, 0); + }, + }, +]; diff --git a/tests/e2e/security-fixtures.ts b/tests/e2e/security-fixtures.ts new file mode 100644 index 00000000..b69123a5 --- /dev/null +++ b/tests/e2e/security-fixtures.ts @@ -0,0 +1,165 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import net from "node:net"; +import type { E2EContext } from "./types.js"; + +export function result(envelope: any): any { + assert.equal(envelope.ok, true, envelope.error); + return envelope.result; +} + +export function denied(envelope: any, reason: RegExp): void { + assert.equal(envelope.ok, false, "operation unexpectedly succeeded"); + assert.match(envelope.error, reason); +} + +export function syntheticSecret(ctx: E2EContext): string { + const value = `E2E-only!${randomUUID()}-Aa9`; + ctx.redact(value); + return value; +} + +export function excludesSecret(value: any, secret: string): void { + assert.equal(JSON.stringify(value).includes(secret), false, "handled synthetic secret leaked"); +} + +function form(kind: string, suffix = ""): string { + const signup = kind === "signup"; + const rotation = kind === "rotate"; + const passwordRole = signup || rotation ? "new-password" : "current-password"; + return `
+ + ${rotation ? `` : ""} + + ${signup || rotation ? `` : ""} + +
`; +} + +export async function credentialSite(ctx: E2EContext, expectedPassword?: string) { + const state = { + requests: new Map(), + submissions: 0, + accepted: 0, + rejected: 0, + }; + const server = await ctx.serve((request, response) => { + const pathname = new URL(request.url || "/", "http://fixture.invalid").pathname; + state.requests.set(pathname, (state.requests.get(pathname) || 0) + 1); + response.setHeader("cache-control", "no-store"); + if (request.method === "POST" && pathname.startsWith("/submit/")) { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { body += chunk; }); + request.on("end", () => { + const fields = new URLSearchParams(body); + const password = fields.get("password") || ""; + const kind = pathname.slice("/submit/".length); + const accepted = Boolean(fields.get("username")) && (kind === "login" + ? expectedPassword === undefined ? password.length >= 16 : password === expectedPassword + : password.length >= 16 && password === fields.get("confirm") && + (kind !== "rotate" || fields.get("current") === expectedPassword)); + state.submissions += 1; + state.accepted += Number(accepted); + state.rejected += Number(!accepted); + response.setHeader("content-type", "application/json"); + response.end(JSON.stringify({ accepted })); + }); + return; + } + if (pathname === "/download") { + response.setHeader("content-type", "application/octet-stream"); + response.setHeader("content-disposition", 'attachment; filename="fixture.txt"'); + response.end("local security download fixture"); + return; + } + const kind = pathname === "/signup" ? "signup" : pathname === "/rotate" ? "rotate" : "login"; + response.setHeader("content-type", "text/html; charset=utf-8"); + response.end(`Security fixture +

Security fixture

+ ${pathname === "/ambiguous" ? form("login", "-one") + form("login", "-two") : form(kind)} + Download fixture + `); + }); + return { ...server, state }; +} + +export async function savedLogin(ctx: E2EContext, matchMode = "exact-origin") { + const secret = syntheticSecret(ctx); + const site = await credentialSite(ctx, secret); + const saved = await ctx.run(` + await page.goto(${JSON.stringify(`${site.origin}/login`)}); + return credentials.save({username: "fixture-user", password: ${JSON.stringify(secret)}, + label: "E2E account", matchMode: ${JSON.stringify(matchMode)}}); + `); + excludesSecret(saved, secret); + const record = result(saved); + assert.ok(record.id); + return { site, secret, record }; +} + +export async function confinedUpstream(ctx: E2EContext) { + const authorities: string[] = []; + const requests: string[] = []; + const sockets = new Set(); + const body = 'Confined upstream

Local proxy fixture

'; + const server = net.createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("error", () => socket.destroy()); + socket.setTimeout(2_000, () => socket.destroy()); + let buffer = ""; + let connected = false; + socket.on("data", (chunk) => { + buffer += chunk.toString("latin1"); + let end = buffer.indexOf("\r\n\r\n"); + if (end === -1) return; + if (!connected) { + const [method, authority] = buffer.slice(0, end).split("\r\n")[0].split(" "); + if (method !== "CONNECT") { + socket.end("HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + } + authorities.push(authority); + if (authority.endsWith(":443")) { + socket.end("HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + } + connected = true; + buffer = buffer.slice(end + 4); + socket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + end = buffer.indexOf("\r\n\r\n"); + if (end === -1) return; + } + requests.push(buffer.slice(0, end).split("\r\n")[0]); + socket.end(`HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: ${Buffer.byteLength(body)}\r\nConnection: close\r\n\r\n${body}`); + }); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address); + const port = Object.getOwnPropertyDescriptor(address, "port")?.value; + assert.ok(Number.isInteger(port), "TCP fixture has no port"); + ctx.cleanup(async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + }); + return { origin: `http://127.0.0.1:${port}`, authorities, requests }; +} diff --git a/tests/e2e/types.ts b/tests/e2e/types.ts new file mode 100644 index 00000000..6fb29f6d --- /dev/null +++ b/tests/e2e/types.ts @@ -0,0 +1,47 @@ +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import type { RequestListener } from "node:http"; + +export interface CommandOptions { + stdin?: string; + timeoutMs?: number; + env?: Record; +} + +export interface CommandResult { + code: number | null; + stdout: string; + stderr: string; +} + +export interface RunOptions extends CommandOptions { + session?: string; + profile?: string; + args?: string[]; +} + +export interface FixtureServer { + origin: string; + alternateOrigin: string; + close(): Promise; +} + +export interface E2EContext { + home: string; + workDir: string; + command(args: string[], options?: CommandOptions): Promise; + start(args: string[], options?: CommandOptions): ChildProcessWithoutNullStreams; + json(args: string[], options?: CommandOptions): Promise; + run(code: string, options?: RunOptions): Promise; + serve(handler: RequestListener): Promise; + cleanup(action: () => Promise | void): void; + skip(reason: string): never; + redact(value: string): void; +} + +export interface E2ECase { + id: string; + group: "cli" | "browser" | "security" | "protocol" | "recording"; + title: string; + requiresBrowser?: boolean; + run(context: E2EContext): Promise; +} diff --git a/tests/node/e2e-harness.test.ts b/tests/node/e2e-harness.test.ts new file mode 100644 index 00000000..710c0330 --- /dev/null +++ b/tests/node/e2e-harness.test.ts @@ -0,0 +1,198 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { + type CaseResult, + createContext, + isolatedEnvironment, + parseOptions, + reportExitCode, + runCase, + selectCases, +} from "../e2e/harness.js"; +import type { E2ECase } from "../e2e/types.js"; +import { makeTempDir } from "./helpers/temp-dir.js"; + +const fixture = fileURLToPath(new URL("./fixtures/e2e-target.ts", import.meta.url)); + +function options() { + return parseOptions(["--binary", process.execPath, "--binary-arg", fixture]); +} + +function sampleCase(id = "test.case"): E2ECase { + return { id, group: "cli", title: id, async run() {} }; +} + +test("E2E options default to the PATH binary and support an explicit launcher", () => { + assert.equal(parseOptions([]).binary, "betterwright"); + assert.equal(parseOptions(["./bin/custom"]).binary, path.resolve("./bin/custom")); + const parsed = parseOptions(["--binary", "bun", "--binary-arg", "--preload", "--binary-arg", "guard.ts", "--group", "cli", "--group", "security", "--require-all"]); + assert.deepEqual(parsed.binaryArgs, ["--preload", "guard.ts"]); + assert.deepEqual(parsed.groups, ["cli", "security"]); + assert.equal(parsed.requireAll, true); +}); + +test("E2E options reject typos, missing values, duplicate targets and invalid deadlines", () => { + for (const args of [ + ["--typo"], ["--binary"], ["--report", "--list"], ["first", "second"], + ["first", "--binary", "second"], ["--group", "missing"], ["--timeout", "NaN"], + ["--timeout", "0"], ["--timeout", "1.5"], ["--case-timeout", "3600001"], + ]) assert.throws(() => parseOptions(args), undefined, args.join(" ")); +}); + +test("E2E selection cannot silently pass with zero tests or duplicate IDs", () => { + assert.throws(() => selectCases([], options()), /empty successful run/); + assert.throws(() => selectCases([sampleCase(), sampleCase()], options()), /Duplicate/); + assert.throws(() => selectCases([sampleCase("../escape")], options()), /Invalid/); + const parsed = options(); + parsed.filter = "second"; + assert.deepEqual(selectCases([sampleCase(), sampleCase("test.second")], parsed).map((entry) => entry.id), ["test.second"]); + parsed.groups = ["security"]; + assert.throws(() => selectCases([sampleCase("test.second")], parsed), /No test cases/); +}); + +test("E2E environments isolate user state and do not inherit credentials or preload hooks", () => { + const root = makeTempDir("bw-e2e-env-"); + const env = isolatedEnvironment(root); + assert.equal(env.HOME, path.join(root, "user")); + assert.equal(env.USERPROFILE, env.HOME); + assert.equal(env.BETTERWRIGHT_HOME, path.join(root, "betterwright")); + assert.notEqual(env.HOME, os.homedir()); + for (const name of ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NODE_OPTIONS", "BUN_OPTIONS", "BETTERWRIGHT_MODEL", "BETTERWRIGHT_CDP_URL", "BETTERWRIGHT_PROFILE"]) { + assert.equal(env[name], undefined, `${name} must not be inherited`); + } +}); + +test("E2E commands pass literal argv and stdin without a shell", async () => { + const root = makeTempDir("bw-e2e-command-"); + const harness = createContext(options(), root); + try { + const value = await harness.context.json(["echo", "a b", 'quote"value', "$(touch should-not-exist)", "雪"], { + stdin: "line one\nline two\n", + env: { E2E_EXTRA: "extra" }, + }); + assert.deepEqual(value.args, ["a b", 'quote"value', "$(touch should-not-exist)", "雪"]); + assert.equal(value.stdin, "line one\nline two\n"); + assert.equal(value.cwd, root); + assert.equal(value.extra, "extra"); + assert.equal(value.home, path.join(root, "user")); + assert.equal(fs.existsSync(path.join(root, "should-not-exist")), false); + } finally { await harness.dispose(); } +}); + +test("E2E command status, output bounds, and deadlines fail honestly", async () => { + const harness = createContext(options(), makeTempDir("bw-e2e-errors-")); + try { + const failure = await harness.context.command(["fail"]); + assert.equal(failure.code, 7); + await assert.rejects(harness.context.json(["fail"]), /intentional target failure/); + await assert.rejects(harness.context.command(["hang"], { timeoutMs: 100 }), /exceeded 100ms/); + await assert.rejects(harness.context.command(["flood"]), /output limit/); + assert.equal((await harness.context.command(["--version"])).code, 0); + } finally { await harness.dispose(); } +}); + +test("E2E run validates JSON and the exit-code/envelope contract", async () => { + const harness = createContext(options(), makeTempDir("bw-e2e-envelope-")); + try { + assert.equal((await harness.context.run("success")).result, 42); + assert.equal((await harness.context.run("failure")).ok, false); + await assert.rejects(harness.context.run("wrong-exit"), /nonzero exit/); + await assert.rejects(harness.context.run("malformed"), /did not return JSON/); + } finally { await harness.dispose(); } +}); + +test("E2E cleanup closes only its home and prevents late commands", async () => { + const root = makeTempDir("bw-e2e-cleanup-"); + const harness = createContext(options(), root); + await harness.context.command(["touch-home"]); + const server = await harness.context.serve((_request, response) => response.end("fixture")); + assert.equal(await (await fetch(server.origin)).text(), "fixture"); + await harness.dispose(); + assert.equal(fs.readFileSync(path.join(root, "closed.txt"), "utf8"), "close --all"); + await assert.rejects(fetch(server.origin)); + await assert.rejects(harness.context.command(["--version"]), /disposed/); +}); + +test("E2E cases redact diagnostics, distinguish skips and fail cleanup errors", async () => { + const root = makeTempDir("bw-e2e-cases-"); + const failed = await runCase({ ...sampleCase(), async run(context) { + context.redact("synthetic-secret-for-test"); + throw new Error("failure includes synthetic-secret-for-test"); + } }, options(), root); + assert.equal(failed.status, "failed"); + assert.equal(failed.error, "failure includes [redacted]"); + const skipped = await runCase({ ...sampleCase(), async run(context) { context.skip("encoder unavailable"); } }, options(), root); + assert.equal(skipped.status, "skipped"); + assert.equal(skipped.reason, "encoder unavailable"); + const cleanup = await runCase({ ...sampleCase(), async run(context) { + await context.command(["touch-home"]); + fs.writeFileSync(path.join(context.workDir, "fail-cleanup"), "1"); + } }, options(), root); + assert.equal(cleanup.status, "failed"); + assert.match(cleanup.error, /Cleanup failed/); + assert.ok(cleanup.workDir); +}); + +test("E2E whole-case deadline and interruption cannot report a pass", async () => { + const root = makeTempDir("bw-e2e-deadline-"); + const config = options(); + config.caseTimeoutMs = 100; + const timedOut = await runCase({ ...sampleCase(), async run(context) { await context.command(["hang"]); } }, config, root); + assert.equal(timedOut.status, "failed"); + assert.match(timedOut.error, /Case exceeded/); + const controller = new AbortController(); + controller.abort(); + const interrupted = await runCase(sampleCase(), config, root, controller.signal); + assert.equal(interrupted.status, "failed"); +}); + +test("E2E exit status rejects failures, all-skipped runs, and strict partial coverage", () => { + const result = (status: CaseResult["status"]): CaseResult => ({ id: status, group: "cli", title: status, status, durationMs: 0 }); + assert.equal(reportExitCode([], false), 1); + assert.equal(reportExitCode([result("passed")], false), 0); + assert.equal(reportExitCode([result("failed")], false), 1); + assert.equal(reportExitCode([result("skipped")], false), 1); + assert.equal(reportExitCode([result("passed"), result("skipped")], false), 0); + assert.equal(reportExitCode([result("passed"), result("skipped")], true), 1); + assert.equal(reportExitCode([{ ...result("passed"), id: "preflight.binary" }, result("skipped")], false), 1); +}); + +test("E2E executable writes a complete machine-readable passing report", async () => { + const root = makeTempDir("bw-e2e-report-"); + const report = path.join(root, "report.json"); + const runner = fileURLToPath(new URL("../e2e/run.ts", import.meta.url)); + const result = await promisify(execFile)(process.execPath, [runner, + "--binary", process.execPath, "--binary-arg", fixture, + "--filter", "cli-version", "--require-all", "--report", report, + ], { timeout: 30_000, env: { ...process.env, BUN_RUNTIME_TRANSPILER_CACHE_PATH: "0" } }); + const parsed = JSON.parse(fs.readFileSync(report, "utf8")); + assert.equal(parsed.complete, true); + assert.equal(parsed.outcome, "passed"); + assert.equal(parsed.selectedCases, 1); + assert.equal(parsed.target.version, "0.0.0-e2e-fixture"); + assert.deepEqual(parsed.summary, { passed: 2, failed: 0, skipped: 0 }); + assert.ok(parsed.untested.length > 0); + assert.match(result.stdout, /2 passed, 0 failed, 0 skipped/); +}); + +test("E2E executable reports an unavailable target as failure, not all-skipped success", async () => { + const root = makeTempDir("bw-e2e-unavailable-"); + const report = path.join(root, "report.json"); + const runner = fileURLToPath(new URL("../e2e/run.ts", import.meta.url)); + await assert.rejects(promisify(execFile)(process.execPath, [runner, + "--binary", path.join(root, "missing-betterwright"), + "--filter", "cli-version", "--report", report, + ], { timeout: 30_000 })); + const parsed = JSON.parse(fs.readFileSync(report, "utf8")); + assert.equal(parsed.complete, true); + assert.equal(parsed.outcome, "failed"); + assert.deepEqual(parsed.summary, { passed: 0, failed: 1, skipped: 1 }); + assert.match(parsed.results[0].error, /ENOENT/); + assert.equal(parsed.results[1].reason, "Target binary preflight failed"); +}); diff --git a/tests/node/fixtures/e2e-target.ts b/tests/node/fixtures/e2e-target.ts new file mode 100644 index 00000000..a0647a2c --- /dev/null +++ b/tests/node/fixtures/e2e-target.ts @@ -0,0 +1,38 @@ +import fs from "node:fs"; +import path from "node:path"; + +const args = process.argv.slice(2); +const command = args[0]; +if (command === "--version" || command === "-v") { + console.log("0.0.0-e2e-fixture"); +} else if (command === "echo") { + console.log(JSON.stringify({ + args: args.slice(1), + stdin: fs.readFileSync(0, "utf8"), + cwd: process.cwd(), + home: process.env.HOME, + betterwrightHome: process.env.BETTERWRIGHT_HOME, + extra: process.env.E2E_EXTRA, + })); +} else if (command === "touch-home") { + fs.mkdirSync(process.env.BETTERWRIGHT_HOME, { recursive: true }); +} else if (command === "close") { + fs.writeFileSync(path.join(process.cwd(), "closed.txt"), args.join(" ")); + process.exitCode = fs.existsSync(path.join(process.cwd(), "fail-cleanup")) ? 1 : 0; +} else if (command === "run") { + const code = args[args.indexOf("-c") + 1]; + if (code === "malformed") console.log("not json"); + else console.log(JSON.stringify({ ok: code !== "failure" && code !== "wrong-exit", result: 42 })); + process.exitCode = code === "failure" ? 1 : 0; +} else if (command === "hang") { + if (args[1]) fs.writeFileSync(args[1], String(process.pid)); + setInterval(() => {}, 1_000); +} else if (command === "flood") { + process.stdout.write("x".repeat(3 * 1024 * 1024)); +} else if (command === "fail") { + console.error("intentional target failure"); + process.exitCode = 7; +} else { + console.error("unknown fixture command"); + process.exitCode = 1; +} diff --git a/tests/node/navigation-history.test.ts b/tests/node/navigation-history.test.ts new file mode 100644 index 00000000..488cfbf0 --- /dev/null +++ b/tests/node/navigation-history.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import vm from "node:vm"; +import { navigateHistory } from "../../dist/src/navigation-defaults.js"; + +test("history waits for document readiness after commit rather than repeating BFCache lifecycle events", async () => { + for (const method of ["goBack", "goForward"] as const) { + for (const waitUntil of ["domcontentloaded", "load"]) { + const calls = []; + let disposed = false; + const response = { status: 200 }; + const page = { + async [method](options) { calls.push(options); return response; }, + async waitForFunction(predicate, state, options) { + assert.equal(state, waitUntil); + assert.ok(options.timeout > 0 && options.timeout <= 500); + for (const [domContentLoadedEventEnd, loadEventEnd] of [[0, 0], [1, 0], [1, 2]]) { + class PerformanceNavigationTiming { + domContentLoadedEventEnd = domContentLoadedEventEnd; + loadEventEnd = loadEventEnd; + } + const ready = vm.runInNewContext(`(${predicate.toString()})(state)`, { + state, + PerformanceNavigationTiming, + performance: { getEntriesByType: () => [new PerformanceNavigationTiming()] }, + document: { readyState: "interactive" }, + }); + assert.equal(ready, waitUntil === "load" ? loadEventEnd > 0 : domContentLoadedEventEnd > 0); + } + return { async dispose() { disposed = true; } }; + }, + }; + const options = { waitUntil, timeout: 500 }; + assert.equal(await navigateHistory(page, method, options, 30_000), response); + assert.deepEqual(calls, [{ waitUntil: "commit", timeout: 500 }]); + assert.equal(options.waitUntil, waitUntil); + assert.equal(disposed, true); + } + } +}); + +test("history preserves commit, networkidle, invalid options, and navigation failures", async () => { + for (const options of [undefined, null, false, "invalid", { waitUntil: "commit" }, { waitUntil: "networkidle" }, { waitUntil: "invalid" }]) { + const page = { async goBack(supplied) { assert.equal(supplied, options); return null; } }; + assert.equal(await navigateHistory(page, "goBack", options, 30_000), null); + } + const failure = new Error("navigation failed"); + await assert.rejects(navigateHistory({ async goBack() { throw failure; } }, "goBack", { waitUntil: "load" }, 30_000), failure); +}); + +test("history preserves disabled timeouts and propagates readiness failures", async () => { + const page = { + async goForward() { return null; }, + async waitForFunction(_predicate, _state, options) { + assert.equal(options.timeout, 0); + throw new Error("document readiness failed"); + }, + }; + await assert.rejects(navigateHistory(page, "goForward", { waitUntil: "load", timeout: 0 }, 30_000), /document readiness failed/); +}); diff --git a/tsconfig.harness.json b/tsconfig.harness.json index 1703f329..2eddb2b8 100644 --- a/tsconfig.harness.json +++ b/tsconfig.harness.json @@ -41,6 +41,7 @@ "research/warm-profile.ts", "research/verify-local-ai-catalog.ts", "tests/node/**/*.ts", + "tests/e2e/**/*.ts", "benchmarks/**/*.ts" ], "exclude": ["dist", "node_modules"] From 777760eebc5c35fbeb5770e81af1c76357f36b03 Mon Sep 17 00:00:00 2001 From: CuriosityOS <184718100+CuriosityOS@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:26:34 +0000 Subject: [PATCH 2/3] Keep E2E additions under Unreleased after the 2.8.8 release --- CHANGELOG.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f39f2828..6315ac41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,26 @@ Releases before 1.1.3 predate this file; their notes live on the ## [Unreleased] +### Changed + +- `bun run test:e2e` tests a supplied BetterWright executable or `betterwright` + on PATH through isolated local fixtures, with per-case JSON reporting, + explicit skips, and a strict CI mode. See `docs/binary-e2e.md` for coverage + and external-integration limits. + +### Fixed + +- `human.type(..., { clear: false })` positions the caret at the end before + typing, rather than inserting at the point where the focus click landed. +- History navigation waits for document readiness on back/forward-cache + restores without waiting for lifecycle events that have already fired. +- The MCP stdio server closes its browser when stdin ends, including when the + transport does not report EOF through its protocol close callback. +- Starting a live view no longer clears content created with `page.setContent` + on an `about:blank` tab while warming up an otherwise empty page. +- The stealth worker selects `patchright-core` directly, so Bun's handling of + Node loader hooks cannot silently leave the stock driver active. + ## [2.8.8] - 2026-09-21 ### Security @@ -32,10 +52,6 @@ Releases before 1.1.3 predate this file; their notes live on the ### Changed -- `bun run test:e2e` tests a supplied BetterWright executable or `betterwright` - on PATH through isolated local fixtures, with per-case JSON reporting, - explicit skips, and a strict CI mode. See `docs/binary-e2e.md` for coverage - and external-integration limits. - SECURITY.md, README, and the credential and network-policy docs now state the boundary precisely: redaction removes literal secret values from result envelopes and does not chase transformed read-backs, and an upstream egress @@ -44,16 +60,6 @@ Releases before 1.1.3 predate this file; their notes live on the ### Fixed -- `human.type(..., { clear: false })` positions the caret at the end before - typing, rather than inserting at the point where the focus click landed. -- History navigation waits for document readiness on back/forward-cache - restores without waiting for lifecycle events that have already fired. -- The MCP stdio server closes its browser when stdin ends, including when the - transport does not report EOF through its protocol close callback. -- Starting a live view no longer clears content created with `page.setContent` - on an `about:blank` tab while warming up an otherwise empty page. -- The stealth worker selects `patchright-core` directly, so Bun's handling of - Node loader hooks cannot silently leave the stock driver active. - On Windows, a vault lock publish rename that returns EPERM or EACCES while the destination is already gone is treated as lock contention only after the candidate can still rename to a unique sibling. From 24346599b0f45283750a309baee0498abf86aae9 Mon Sep 17 00:00:00 2001 From: CuriosityOS <184718100+CuriosityOS@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:30:22 +0000 Subject: [PATCH 3/3] Handle canonical paths and Windows spawn errors in E2E harness tests --- tests/node/e2e-harness.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/node/e2e-harness.test.ts b/tests/node/e2e-harness.test.ts index 710c0330..c0932135 100644 --- a/tests/node/e2e-harness.test.ts +++ b/tests/node/e2e-harness.test.ts @@ -78,13 +78,27 @@ test("E2E commands pass literal argv and stdin without a shell", async () => { }); assert.deepEqual(value.args, ["a b", 'quote"value', "$(touch should-not-exist)", "雪"]); assert.equal(value.stdin, "line one\nline two\n"); - assert.equal(value.cwd, root); + assert.equal(fs.realpathSync(value.cwd), fs.realpathSync(root)); assert.equal(value.extra, "extra"); assert.equal(value.home, path.join(root, "user")); assert.equal(fs.existsSync(path.join(root, "should-not-exist")), false); } finally { await harness.dispose(); } }); +test("E2E working-directory isolation survives a symlinked temporary root", async () => { + const parent = makeTempDir("bw-e2e-symlink-"); + const actual = path.join(parent, "actual"); + const alias = path.join(parent, "alias"); + fs.mkdirSync(actual); + fs.symlinkSync(actual, alias, process.platform === "win32" ? "junction" : "dir"); + const harness = createContext(options(), alias); + try { + const value = await harness.context.json(["echo"]); + assert.equal(fs.realpathSync(value.cwd), fs.realpathSync(actual)); + assert.equal(value.betterwrightHome, path.join(alias, "betterwright")); + } finally { await harness.dispose(); } +}); + test("E2E command status, output bounds, and deadlines fail honestly", async () => { const harness = createContext(options(), makeTempDir("bw-e2e-errors-")); try { @@ -193,6 +207,7 @@ test("E2E executable reports an unavailable target as failure, not all-skipped s assert.equal(parsed.complete, true); assert.equal(parsed.outcome, "failed"); assert.deepEqual(parsed.summary, { passed: 0, failed: 1, skipped: 1 }); - assert.match(parsed.results[0].error, /ENOENT/); + assert.match(parsed.results[0].error, /ENOENT|Executable not found/i); + assert.match(parsed.results[0].error, /missing-betterwright/); assert.equal(parsed.results[1].reason, "Target binary preflight failed"); });