From 0d7bc171281429d5b9cc50e17f14eb9663d03da2 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 22:16:38 +0530 Subject: [PATCH] test(onboarding): end-to-end funnel telemetry against a real CLI process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opt-in via ALTIMATE_E2E=1; skipped otherwise, so it cannot flake anyone's commit. Two scenarios, ~27s total. Every other telemetry test in this repo spies inside one process, which cannot see the thing that actually matters: the TUI renders on the main thread and the server runs in a Bun Worker, each with its own Telemetry module instance and its own buffer. This spawns the real CLI in a PTY with its telemetry endpoint pointed at a local sink, drives it with keystrokes, and asserts on the envelopes that arrive over HTTP. It has already earned its place — it caught `launch_id` differing between the two threads, which made the correlation id useless for the exact join it exists to provide. No single-process test could have seen that. Covered: - A full first run through Big Pickle: the eight Part 1 events, their properties, and that every event across both threads and two different session ids shares one launch_id. - Quitting at the picker: `onboarding_abandoned{last_stage: model_picker}` arrives after process exit, which is the only proof the flush-on-exit path delivers. Notes for anyone extending it: - A throwaway HOME is essential. A developer machine has credentials, so the first-run gate never opens and the funnel never starts. - The CLI must run from packages/opencode, as the `dev` script does — the JSX runtime comes from the workspace bunfig.toml, and bun only picks it up from there. The project under test is passed as the positional argument. - Steps are paced with sleeps, not by waiting on events or screen text. Events arrive on the flush interval and lag the UI by seconds; terminal output is per-cell ANSI, so a visible label is split across escape sequences and never matches a substring search. - bun-pty, not @lydell/node-pty: the latter loads a platform-specific native package that is absent in this workspace and fails silently — no output, no error, no exit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- bun.lock | 1 + packages/opencode/package.json | 1 + .../test/e2e/onboarding-funnel.e2e.test.ts | 104 +++++++++++++ packages/opencode/test/e2e/telemetry-sink.ts | 146 ++++++++++++++++++ 4 files changed, 252 insertions(+) create mode 100644 packages/opencode/test/e2e/onboarding-funnel.e2e.test.ts create mode 100644 packages/opencode/test/e2e/telemetry-sink.ts diff --git a/bun.lock b/bun.lock index c7dfd8ffd..8487b9bdb 100644 --- a/bun.lock +++ b/bun.lock @@ -369,6 +369,7 @@ "@types/turndown": "5.0.5", "@types/yargs": "17.0.33", "@typescript/native-preview": "catalog:", + "bun-pty": "0.4.8", "drizzle-orm": "catalog:", "playwright-core": "1.59.1", "prettier": "3.6.2", diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 173bd0e1b..7badd1cb0 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -32,6 +32,7 @@ } }, "devDependencies": { + "bun-pty": "0.4.8", "@babel/core": "7.28.4", "@octokit/webhooks-types": "7.6.1", "@opencode-ai/core": "workspace:*", diff --git a/packages/opencode/test/e2e/onboarding-funnel.e2e.test.ts b/packages/opencode/test/e2e/onboarding-funnel.e2e.test.ts new file mode 100644 index 000000000..50bc52aa6 --- /dev/null +++ b/packages/opencode/test/e2e/onboarding-funnel.e2e.test.ts @@ -0,0 +1,104 @@ +// altimate_change — end-to-end onboarding funnel telemetry. +// +// Opt-in: ALTIMATE_E2E=1. These drive a real CLI process through a PTY, so they are slower and +// more timing-sensitive than the rest of the suite and should not gate anyone's commit. +// +// Assertions are on the event stream that arrives over HTTP, never on screen contents beyond the +// minimum needed to know which screen we are on — terminal rendering changes far more often than +// the telemetry contract does. +import { describe, expect, test } from "bun:test" +import { KEY, startCli, startSink } from "./telemetry-sink" + +const enabled = process.env.ALTIMATE_E2E === "1" + +describe.skipIf(!enabled)("onboarding funnel (e2e)", () => { + test( + "a first run through Big Pickle emits the Part 1 funnel, joined by one launch_id", + async () => { + const sink = startSink() + const cli = await startCli(sink) + try { + // First run opens the curated picker with no stored credentials. + await sink.waitFor("model_picker_shown") + + // Big Pickle is the fifth row and needs no signup, so the whole flow stays local. + // + // Steps are paced with sleeps rather than waiting for an event or for screen text. + // Events arrive on the flush interval, so they lag the UI by seconds and cannot gate the + // next keystroke; and the terminal output is per-cell ANSI, so a rendered label is split + // across escape sequences and never matches a substring search. + cli.press(KEY.down.repeat(4)) + await Bun.sleep(400) + cli.press(KEY.enter) + await Bun.sleep(1200) + + cli.press("y") // accept the Big Pickle interstitial + await Bun.sleep(2500) + + cli.press("n") // decline the scan gate + await Bun.sleep(3000) + + cli.press(KEY.ctrlC) + await Bun.sleep(1200) + cli.press(KEY.ctrlC) + await Promise.race([cli.exited, Bun.sleep(15_000)]) + await Bun.sleep(1500) + + const names = sink.names() + expect(names).toContain("onboarding_started") + expect(names).toContain("model_picker_shown") + expect(names).toContain("provider_selected") + expect(names).toContain("big_pickle_choice") + + const picker = sink.envelopes.find((e) => e.name === "model_picker_shown")! + expect(picker.properties.trigger).toBe("first_run") + + const provider = sink.envelopes.find((e) => e.name === "provider_selected")! + expect(provider.properties.provider).toBe("big_pickle") + + const choice = sink.envelopes.find((e) => e.name === "scan_gate_choice")! + expect(choice.properties.choice).toBe("skip") + + // The point of launch_id: TUI-thread events (started, picker) and worker-thread events + // must be attributable to the same run even though they carry different sessions. + const launchIds = new Set(sink.envelopes.map((e) => e.properties.launch_id)) + expect(launchIds.size).toBe(1) + expect([...launchIds][0]).toBeTruthy() + + // A completed onboarding is not an abandoned one. + expect(names).not.toContain("onboarding_abandoned") + } finally { + await cli.cleanup() + sink.stop() + } + }, + 120_000, + ) + + test( + "quitting at the picker reports abandonment, and it survives process exit", + async () => { + const sink = startSink() + const cli = await startCli(sink) + try { + await sink.waitFor("model_picker_shown") + + // Quit without choosing anything. The event is emitted during shutdown, so this is the + // only test that proves the exit-flush path actually delivers. + cli.press(KEY.ctrlC) + await Bun.sleep(1000) + cli.press(KEY.ctrlC) + await Promise.race([cli.exited, Bun.sleep(20_000)]) + await Bun.sleep(2000) + + const abandoned = await sink.waitFor("onboarding_abandoned", 15_000) + expect(abandoned.properties.last_stage).toBe("model_picker") + expect(sink.names()).not.toContain("onboarding_completed") + } finally { + await cli.cleanup() + sink.stop() + } + }, + 120_000, + ) +}) diff --git a/packages/opencode/test/e2e/telemetry-sink.ts b/packages/opencode/test/e2e/telemetry-sink.ts new file mode 100644 index 000000000..089a8f423 --- /dev/null +++ b/packages/opencode/test/e2e/telemetry-sink.ts @@ -0,0 +1,146 @@ +// altimate_change — end-to-end harness for onboarding telemetry. +// +// Every other test in this suite spies inside one process. That cannot see the real thing: the +// TUI renders on the main thread and the server runs in a Worker, each with its own Telemetry +// instance and its own buffer. Only a real process, driven through a real terminal, exercises +// both — and only a real exit proves the flush-on-exit path works, which is where onboarding +// events are most likely to be silently lost. +// +// So this spawns the CLI in a PTY with its telemetry endpoint pointed at a local sink, sends +// keystrokes, and asserts on the envelopes that actually arrive over HTTP. +// bun-pty rather than @lydell/node-pty: the latter loads a platform-specific native package +// that is not installed in this workspace and fails SILENTLY when missing — no output, no error, +// no exit — which is a miserable thing to debug. bun-pty is a devDependency of this package. +import { spawn as ptySpawn, type IPty } from "bun-pty" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" + +export type Envelope = { + name: string + properties: Record + measurements: Record + sessionId: string +} + +export type Sink = { + url: string + envelopes: Envelope[] + /** Resolves once an event with this name arrives, or throws after `timeout`. */ + waitFor(name: string, timeout?: number): Promise + names(): string[] + stop(): void +} + +export function startSink(): Sink { + const envelopes: Envelope[] = [] + + const server = Bun.serve({ + port: 0, + async fetch(req) { + if (!req.url.endsWith("/v2/track")) return new Response("not found", { status: 404 }) + const raw = (await req.json()) as any[] + for (const item of raw) { + const base = item?.data?.baseData ?? {} + envelopes.push({ + name: String(base.name ?? ""), + properties: base.properties ?? {}, + measurements: base.measurements ?? {}, + sessionId: item?.tags?.["ai.session.id"] ?? "", + }) + } + return new Response("", { status: 200 }) + }, + }) + + return { + url: `http://127.0.0.1:${server.port}`, + envelopes, + names: () => envelopes.map((e) => e.name), + async waitFor(name, timeout = 45_000) { + const start = Date.now() + for (;;) { + const found = envelopes.find((e) => e.name === name) + if (found) return found + if (Date.now() - start > timeout) { + throw new Error(`timed out waiting for "${name}". Received: ${JSON.stringify(envelopes.map((e) => e.name))}`) + } + await Bun.sleep(100) + } + }, + stop: () => server.stop(true), + } +} + +export type Cli = { + pty: IPty + /** + * Raw terminal output, for diagnosing a run that never reached the expected screen. Not usable + * for assertions: opentui emits per-cell escape sequences, so a visible label like "Big Pickle" + * is split across positioning codes and never appears as a contiguous substring. + */ + output(): string + press(keys: string): void + exited: Promise + cleanup(): Promise +} + +const REPO_ROOT = path.resolve(import.meta.dir, "../../../..") + +/** + * Launch the CLI as a real process against `sink`, in a throwaway HOME so it looks like a first + * run — a developer machine has credentials, which would skip the entire funnel — and so the + * machine-id file this run creates cannot collide with the real one. + */ +export async function startCli(sink: Sink): Promise { + const home = await mkdtemp(path.join(tmpdir(), "altimate-e2e-home-")) + const project = await mkdtemp(path.join(tmpdir(), "altimate-e2e-proj-")) + + let buffer = "" + // Run from packages/opencode, exactly as the repo's `dev` script does: the JSX runtime is + // configured in the workspace bunfig.toml, which bun only picks up from that directory. The + // project under test is passed as the positional argument instead of via cwd. + const pty = ptySpawn(process.execPath, ["run", "--conditions=browser", "src/index.ts", project], { + name: "xterm-256color", + cols: 120, + rows: 40, + cwd: path.join(REPO_ROOT, "packages/opencode"), + env: { + ...process.env, + HOME: home, + XDG_DATA_HOME: path.join(home, ".local/share"), + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_STATE_HOME: path.join(home, ".local/state"), + APPLICATIONINSIGHTS_CONNECTION_STRING: `InstrumentationKey=e2e-local;IngestionEndpoint=${sink.url}`, + ALTIMATE_TELEMETRY_DISABLED: "false", + // Keep the run hermetic: no plugin installs, no upgrade check chatter. + OPENCODE_DISABLE_DEFAULT_PLUGINS: "1", + } as Record, + }) + + let resolveExit!: (code: number) => void + const exited = new Promise((resolve) => (resolveExit = resolve)) + pty.onData((d) => (buffer += d)) + pty.onExit(({ exitCode }) => resolveExit(exitCode)) + + return { + pty, + output: () => buffer, + press: (keys) => pty.write(keys), + exited, + async cleanup() { + try { + pty.kill() + } catch {} + await rm(home, { recursive: true, force: true }).catch(() => {}) + await rm(project, { recursive: true, force: true }).catch(() => {}) + }, + } +} + +export const KEY = { + enter: "\r", + ctrlC: "\x03", + down: "\x1b[B", +} as const