From 80168ce885f684b4568a83f57a8864f3f0533c3a Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 14:28:27 +0530 Subject: [PATCH 01/14] feat(onboarding): telemetry taxonomy + first batch of funnel events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the onboarding funnel event taxonomy and wires the first six events, all emitted from the TUI worker thread. - `Telemetry.Event`: 19 onboarding variants under the spec's own event and property names. Doc comments record the two things that would otherwise mislead whoever writes the queries — `gateway_device_code_issued` describes a browser loopback OAuth (this flow has no device code), and the activation events are inferred from proxies rather than observed, so their counts are lower bounds. - `altimate/telemetry/onboarding.ts`: typed emitter that awaits the idempotent `Telemetry.init()` before tracking. Onboarding events can fire before any prompt runs, and in the TUI worker `init()` only happens via `session/prompt.ts` — a user who quits during gateway auth never reaches a prompt, so those events would sit in the pre-init buffer and never ship. Also owns the monotonic funnel-stage state behind `onboarding_abandoned`. - `Telemetry.shutdown()`: serialize concurrent calls. Three paths can now overlap, and each would enter `flush()`, which splices a shared buffer — one caller could post a half-empty batch while the other dropped events. - Close two flush holes. `cmd/tui.ts` ends its handler with `process.exit(0)`, skipping the outer `finally` in `index.ts` that normally flushes; and the TUI worker's `rpc.shutdown()` drained traces but never its own telemetry buffer. The two threads hold separate `Telemetry` module instances, so both need one. Bounded at 2s each: `flush()` can block for `REQUEST_TIMEOUT_MS` (10s), which exceeds the 5s the shutdown RPC is given and would be a visible hang on exit. Events wired: the four gateway auth events, `environment_scan_completed`, and `sample_setup_completed`. `gateway_auth_failed.reason` is classified by tagging the error at each rejection site rather than by matching message text. The `error` query param is attacker-influenced text we must neither parse nor forward, and message matching would drift the moment a string is reworded. Note that an unknown `state` never rejects a pending flow — the handler returns 400 without touching the map — so a CSRF mismatch correctly surfaces as `timeout`, not `denied`. `has_warehouse` and `connections_found` derive from `totalConnections`, not `connections.alreadyConfigured`: warehouses discovered from dbt profiles, docker, or env vars are counted separately, and a user whose only warehouse was auto-discovered would otherwise be recorded as having none. `sample_setup_completed` counts models and seeds by walking the shipped sample tree. A constant would drift the first time someone adds a model, and dbt's `target/manifest.json` is ~17k lines to extract two integers on a path the user is waiting on. No instance or tenant names, filesystem paths, raw error text, or authorize URLs (which carry the CSRF `state`) reach telemetry. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../opencode/src/altimate/plugin/altimate.ts | 52 +++++- .../opencode/src/altimate/telemetry/index.ts | 161 ++++++++++++++++++ .../src/altimate/telemetry/onboarding.ts | 148 ++++++++++++++++ .../src/altimate/tools/project-scan.ts | 25 +++ .../src/altimate/tools/sample-setup.ts | 60 +++++++ packages/opencode/src/cli/cmd/tui.ts | 20 +++ packages/opencode/src/cli/tui/worker.ts | 17 ++ 7 files changed, 481 insertions(+), 2 deletions(-) create mode 100644 packages/opencode/src/altimate/telemetry/onboarding.ts diff --git a/packages/opencode/src/altimate/plugin/altimate.ts b/packages/opencode/src/altimate/plugin/altimate.ts index af4a40072..7ba1be5d2 100644 --- a/packages/opencode/src/altimate/plugin/altimate.ts +++ b/packages/opencode/src/altimate/plugin/altimate.ts @@ -3,6 +3,28 @@ import { createServer } from "http" import { randomBytes } from "crypto" import open from "open" import { AltimateApi } from "../api/client" +// altimate_change — onboarding telemetry for the gateway sign-in funnel +import * as OnboardingTelemetry from "../telemetry/onboarding" + +/** + * Why a failure reason is attached at the rejection site rather than inferred from the message: + * `gateway_auth_failed.reason` is a closed enum, and the callback's catch sees only an Error. + * Matching on message text would silently drift the moment any of these strings is reworded, and + * the `error` query param is attacker-influenced text we must not parse or forward. Tagging the + * error where the cause is known keeps the classification deterministic — and note that an + * unknown/invalid `state` never rejects a pending flow at all (the handler 400s without touching + * the map), so a CSRF mismatch legitimately surfaces later as `timeout`, not `denied`. + */ +type GatewayFailureReason = "timeout" | "denied" | "error" + +function markReason(err: Error, reason: GatewayFailureReason): Error { + return Object.assign(err, { altimateGatewayReason: reason }) +} + +function reasonOf(err: unknown): GatewayFailureReason { + const tagged = (err as { altimateGatewayReason?: GatewayFailureReason } | undefined)?.altimateGatewayReason + return tagged ?? "error" +} // Loopback port the CLI listens on for the browser to deliver the gateway // credential after sign-in. Must match the redirect the web authorize page posts @@ -83,7 +105,8 @@ async function startCallbackServer(): Promise { const error = url.searchParams.get("error") if (error) { - entry.reject(new Error(error)) + // altimate_change — the browser reported an explicit failure: the only true `denied` signal + entry.reject(markReason(new Error(error), "denied")) html(200, HTML_ERROR(error)) return } @@ -135,7 +158,8 @@ function stopCallbackServer() { function registerPending(state: string, timeoutMs = 5 * 60 * 1000): Promise { return new Promise((resolve, reject) => { const timeout = setTimeout(() => { - if (pending.delete(state)) reject(new Error("Timed out waiting for browser sign-in")) + // altimate_change — tag as `timeout` for gateway_auth_failed classification + if (pending.delete(state)) reject(markReason(new Error("Timed out waiting for browser sign-in"), "timeout")) }, timeoutMs) pending.set(state, { resolve: (creds) => { @@ -183,6 +207,16 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { await open(authorizeUrl).catch(() => undefined) + // altimate_change start — onboarding funnel: gateway sign-in started. + // Spec name is `gateway_device_code_issued`; this flow is a browser loopback OAuth + // with no device code, so the event means "authorize URL built, browser open + // attempted". open() failures are swallowed above (the URL is also printed for the + // user to paste), so this fires even when no browser actually launched. + // The URL is never sent — it carries the CSRF `state`. + const startedAt = Date.now() + void OnboardingTelemetry.emit({ type: "gateway_device_code_issued" }) + // altimate_change end + return { url: authorizeUrl, instructions: "Complete sign-in in your browser to connect Altimate LLM Gateway.", @@ -206,11 +240,25 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { altimateInstanceName: creds.instance, altimateApiKey: authToken, }) + // altimate_change start — onboarding funnel: auth succeeded and the instance + // is live. The instance name is the customer's tenant identifier and is never + // sent. time_to_connect_ms is measured from the browser-open above, so it + // belongs to this attempt's closure rather than any shared pending state. + void OnboardingTelemetry.emit({ type: "gateway_auth_completed" }) + void OnboardingTelemetry.emit({ + type: "instance_connected", + time_to_connect_ms: Date.now() - startedAt, + }) + // altimate_change end return { type: "success", key: authToken, provider: "altimate-backend" } } catch (err) { // Log the reason (CSRF / timeout / invalid instance / …). Runs in the // server process, so this goes to the log, not the TUI display. console.error("[altimate] gateway sign-in failed:", err instanceof Error ? err.message : err) + // altimate_change — onboarding funnel: only the classified enum is sent. The + // message can embed the instance name (see the invalid-instance throw above), + // so it never reaches telemetry. + void OnboardingTelemetry.emit({ type: "gateway_auth_failed", reason: reasonOf(err) }) return { type: "failed" } } finally { // Keep the shared server up while another flow is still waiting. diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index d8f6e737e..60784611c 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -731,6 +731,147 @@ export namespace Telemetry { /** output tokens on the stop-without-tools generation — helps distinguish "refused" (low) from "wrote a long text plan" (high) */ tokens_output: number } + // altimate_change end + // altimate_change start — first-run onboarding funnel taxonomy. + // + // Event names and property names follow the product spec verbatim so the taxonomy is + // queryable under the names it was specified with. Two naming notes for whoever writes + // the queries: + // - `gateway_device_code_issued` is a spec name kept for fidelity. The gateway flow is + // actually a browser loopback OAuth (see altimate/plugin/altimate.ts) — there is no + // device code. The event means "authorize URL built and browser open attempted". + // - `environment_scan_completed` overlaps `environment_census` above; census stays the + // richer dbt/warehouse fingerprint, this one is the onboarding-shaped scan result. + // + // Events tagged "derived" are inferred from a proxy signal, not observed directly — the + // activation menu is model-rendered text (src/command/template/onboard-connect.txt), so + // there is no UI event to capture. Treat their counts as lower bounds. + | { + type: "onboarding_started" + timestamp: number + session_id: string + } + | { + type: "model_picker_shown" + timestamp: number + session_id: string + /** the picker mounts from several paths — without this the event over-counts first runs */ + trigger: "first_run" | "connect_command" | "big_pickle_back" | "prompt_gate" + } + | { + type: "provider_selected" + timestamp: number + session_id: string + provider: "altimate_gateway" | "anthropic" | "openai" | "google" | "big_pickle" | "search_all" + } + | { + type: "big_pickle_confirm_shown" + timestamp: number + session_id: string + origin: "welcome" | "model" + } + | { + type: "big_pickle_choice" + timestamp: number + session_id: string + choice: "accept" | "cancel" + } + | { + type: "gateway_device_code_issued" + timestamp: number + session_id: string + } + | { + type: "gateway_auth_completed" + timestamp: number + session_id: string + } + | { + type: "gateway_auth_failed" + timestamp: number + session_id: string + /** `denied` only from an explicit error callback; an unknown/invalid state never rejects + * the pending promise, so CSRF mismatches surface as `timeout`. Never carries error text. */ + reason: "timeout" | "denied" | "error" + } + | { + type: "instance_connected" + timestamp: number + session_id: string + /** measured from the authorize() call that opened the browser */ + time_to_connect_ms: number + } + | { + type: "onboarding_completed" + timestamp: number + session_id: string + } + | { + type: "scan_gate_shown" + timestamp: number + session_id: string + } + | { + type: "scan_gate_choice" + timestamp: number + session_id: string + choice: "scan" | "skip" + } + | { + type: "environment_scan_completed" + timestamp: number + session_id: string + has_dbt: boolean + has_warehouse: boolean + is_repo: boolean + connections_found: number + /** bounded list of short enum reasons — arrays are JSON.stringify'd into customDimensions */ + degraded: string[] + } + | { + type: "activation_menu_shown" + timestamp: number + session_id: string + variant: "warehouse" | "no_data" + } + | { + /** derived — inferred from the first job tool/skill after the menu. `something_else` + * has no tool anchor and is systematically under-counted. */ + type: "activation_job_selected" + timestamp: number + session_id: string + job: "sample_duck_db" | "breaks_downstream" | "sql_review" | "cost" | "something_else" + } + | { + /** derived — see activation_job_selected */ + type: "first_job_completed" + timestamp: number + session_id: string + job: "sample_duck_db" | "breaks_downstream" | "sql_review" | "cost" | "something_else" + } + | { + type: "sample_setup_completed" + timestamp: number + session_id: string + success: boolean + /** counts come from the shipped jaffle-shop manifest; the target path is never sent */ + models: number + tables: number + /** the tool is deliberately re-callable (reuse / reset / install-alongside) */ + reused: boolean + } + | { + type: "first_prompt_sent" + timestamp: number + session_id: string + } + | { + type: "onboarding_abandoned" + timestamp: number + session_id: string + /** last funnel stage observed before exit without a completion */ + last_stage: string + } // altimate_change end /** SHA256 hash a masked error message for anonymous grouping. */ @@ -1437,7 +1578,27 @@ export namespace Telemetry { } // altimate_change end + // altimate_change start — serialize concurrent shutdowns. + // shutdown() is called from several independent paths (session/prompt.ts at the end of each + // session loop, the CLI's outer finally, and — for onboarding telemetry — the TUI exit path + // and the TUI worker's rpc.shutdown). Two overlapping calls would both enter flush(), which + // splices the shared buffer, so one caller can post a half-empty batch while the other drops + // events. The in-flight promise is cleared on settle, so a later init/shutdown cycle (the + // per-session pattern in prompt.ts) still works. + let shutdownPromise: Promise | undefined + export async function shutdown() { + if (shutdownPromise) return shutdownPromise + shutdownPromise = doShutdown().finally(() => { + shutdownPromise = undefined + }) + return shutdownPromise + } + + /** Flush and reset. Bound this with `withTimeout` on exit paths — flush() can block for + * REQUEST_TIMEOUT_MS (10s), which is longer than the TUI's shutdown budget. */ + async function doShutdown() { + // altimate_change end // Wait for init to complete so we know whether telemetry is enabled // and have a valid endpoint to flush to. init() is fire-and-forget // in CLI middleware, so it may still be in-flight when shutdown runs. diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts new file mode 100644 index 000000000..5fe23208c --- /dev/null +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -0,0 +1,148 @@ +// altimate_change start — first-run onboarding funnel emitter. +// +// Thin, typed wrapper over Telemetry.track for the onboarding taxonomy. It exists for two +// reasons that a bare track() call cannot cover: +// +// 1. INIT ORDERING. Telemetry.init() runs in the CLI middleware on the main thread +// (src/index.ts) but, in the TUI worker, only via session/prompt.ts when a prompt runs. +// Onboarding events fire before any prompt exists — a user who quits during gateway auth +// never reaches one — so those events would sit in the pre-init buffer and never ship. +// emit() awaits the idempotent init() first. +// +// 2. ABANDONMENT. `onboarding_abandoned` is defined by what did NOT happen, so something has +// to remember how far the user got. See the thread note below for where that state lives. +// +// THREAD NOTE — this matters and is easy to get wrong. `altimate-code tui` runs the TUI on the +// process main thread and the HTTP server in a Worker (src/cli/cmd/tui.ts), and each thread +// loads its own instance of the Telemetry module with its own buffer. TUI-side events +// (model picker, provider rows, scan gate) are tracked on the MAIN thread; server-side events +// (gateway auth, project scan, sample setup) on the WORKER. Neither can see the other's stage. +// +// Abandonment is therefore owned by the main thread: it is where the process exits, and where +// the user-visible funnel position is known. `last_stage` means "the furthest point the user +// reached in the TUI", which is the question the funnel is actually asking. Server-side stages +// are marked from their TUI-visible trigger (choosing the gateway provider marks `gateway_auth`) +// rather than from the worker, which cannot reach this state. +import { Telemetry } from "./index" + +/** Funnel positions, ordered. `last_stage` on abandonment reports the furthest one reached. */ +export const ONBOARDING_STAGES = [ + "started", + "model_picker", + "big_pickle_confirm", + "gateway_auth", + "connected", + "scan_gate", + "activation", +] as const + +export type OnboardingStage = (typeof ONBOARDING_STAGES)[number] + +/** Every onboarding variant of Telemetry.Event, minus the envelope fields emit() fills in. */ +type OnboardingEventInput = Extract< + Telemetry.Event, + { + type: + | "onboarding_started" + | "model_picker_shown" + | "provider_selected" + | "big_pickle_confirm_shown" + | "big_pickle_choice" + | "gateway_device_code_issued" + | "gateway_auth_completed" + | "gateway_auth_failed" + | "instance_connected" + | "onboarding_completed" + | "scan_gate_shown" + | "scan_gate_choice" + | "environment_scan_completed" + | "activation_menu_shown" + | "activation_job_selected" + | "first_job_completed" + | "sample_setup_completed" + | "first_prompt_sent" + | "onboarding_abandoned" + } +> + +// Distributive: a bare Omit collapses the union to its shared keys, which would drop +// every per-event property (reason, choice, job, …) from the emit() signature. +type DistributiveOmit = T extends unknown ? Omit : never + +type EmitInput = DistributiveOmit + +// Module-global, per thread. Resets on every process launch, which is correct: a fresh launch +// is a fresh onboarding attempt. +let furthestStage: OnboardingStage | undefined +let completed = false +let abandonedEmitted = false + +/** Stage implied by an event, where one is implied. Events not listed leave the stage alone. */ +const STAGE_FOR_EVENT: Partial> = { + onboarding_started: "started", + model_picker_shown: "model_picker", + big_pickle_confirm_shown: "big_pickle_confirm", + gateway_device_code_issued: "gateway_auth", + instance_connected: "connected", + onboarding_completed: "connected", + scan_gate_shown: "scan_gate", + activation_menu_shown: "activation", +} + +function advance(stage: OnboardingStage) { + const next = ONBOARDING_STAGES.indexOf(stage) + const current = furthestStage ? ONBOARDING_STAGES.indexOf(furthestStage) : -1 + // Monotonic: re-opening the picker after reaching the scan gate must not walk the funnel back. + if (next > current) furthestStage = stage +} + +/** + * Emit an onboarding event. Fire-and-forget by design — onboarding must never wait on, or fail + * because of, telemetry. Awaiting the returned promise is only useful on an exit path. + */ +export async function emit(event: EmitInput): Promise { + try { + const stage = STAGE_FOR_EVENT[event.type] + if (stage) advance(stage) + if (event.type === "onboarding_completed") completed = true + + await Telemetry.init() + Telemetry.track({ + ...event, + timestamp: Date.now(), + session_id: Telemetry.getContext().sessionId, + } as Telemetry.Event) + } catch { + // Telemetry must never break onboarding. + } +} + +/** Mark the gateway provider path as entered. The browser flow itself runs in the worker, whose + * telemetry state this thread cannot see — see the thread note at the top of this file. */ +export function markStage(stage: OnboardingStage) { + advance(stage) +} + +/** True once `onboarding_completed` has been emitted on this thread. */ +export function isCompleted() { + return completed +} + +/** + * Emit `onboarding_abandoned` if the user got somewhere in the funnel and never completed. + * Call on the exit path, before the final flush. No-ops when the user never started (a + * returning user with credentials), already completed, or when it has already fired. + */ +export async function emitAbandonedIfIncomplete(): Promise { + if (completed || abandonedEmitted || !furthestStage) return + abandonedEmitted = true + await emit({ type: "onboarding_abandoned", last_stage: furthestStage }) +} + +/** Test seam — module state is per-process by design, so tests need an explicit reset. */ +export function resetForTest() { + furthestStage = undefined + completed = false + abandonedEmitted = false +} +// altimate_change end diff --git a/packages/opencode/src/altimate/tools/project-scan.ts b/packages/opencode/src/altimate/tools/project-scan.ts index 2c80ce779..de051d2f9 100644 --- a/packages/opencode/src/altimate/tools/project-scan.ts +++ b/packages/opencode/src/altimate/tools/project-scan.ts @@ -4,6 +4,8 @@ import { Dispatcher } from "../native" import { existsSync, readFileSync } from "fs" import path from "path" import { Telemetry } from "@/telemetry" +// altimate_change — onboarding funnel: environment_scan_completed +import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" import { Config } from "@/config/config" import { Skill } from "../../skill" @@ -930,6 +932,29 @@ export const ProjectScanTool = Tool.define("project_scan", { } } + // altimate_change start — onboarding funnel: the scan result, in the shape the funnel asks + // for. Deliberately separate from `environment_census` above, which stays the richer + // dbt/warehouse fingerprint; this one answers "did the scan find the user a working setup". + // + // Fires on every project_scan, not just the onboarding one — the tool is also reachable via + // /discover and any model-initiated call. + // + // has_warehouse and connections_found both come from totalConnections, NOT + // connections.alreadyConfigured: connections discovered from dbt profiles, docker, and env + // vars are counted separately, and a user whose only warehouse was auto-discovered would + // otherwise be recorded as having none. + // + // degraded[] is a sorted list of short detection-failure keys — no paths, hosts, or messages. + void OnboardingTelemetry.emit({ + type: "environment_scan_completed", + has_dbt: dbtProject.found, + has_warehouse: totalConnections > 0, + is_repo: git.isRepo, + connections_found: totalConnections, + degraded: degradedList, + }) + // altimate_change end + // Build metadata const toolsFound = dataTools.filter((t) => t.installed).map((t) => t.name) diff --git a/packages/opencode/src/altimate/tools/sample-setup.ts b/packages/opencode/src/altimate/tools/sample-setup.ts index 06398c309..683ae607d 100644 --- a/packages/opencode/src/altimate/tools/sample-setup.ts +++ b/packages/opencode/src/altimate/tools/sample-setup.ts @@ -6,6 +6,8 @@ import { Tool } from "../../tool/tool" import { materializeSample } from "../onboarding/materialize" import { DEFAULT_SAMPLE_NAME, resolveSampleSource, type SampleSourceLocation } from "../onboarding/sample-source-resolver" import { detectDbtRuntime } from "../onboarding/tool-detection" +// altimate_change — onboarding funnel: sample_setup_completed +import * as OnboardingTelemetry from "../telemetry/onboarding" /** * `sample_setup` — LLM-invoked tool that copies the shipped jaffle-shop @@ -148,6 +150,19 @@ export const SampleSetupTool = Tool.define("sample_setup", { `suffix: ${result.suffix}\n` + `${dbtLine}\n` + `note: ${result.note}` + // altimate_change start — onboarding funnel. `reused` is carried because the tool is + // deliberately re-callable (reuse / reset / install-alongside / dbt re-probe), so this + // fires per invocation, not once per sample. targetPath is never sent — it is a filesystem + // path under the user's home. + const sampleContents = countSampleContents(sampleSource.path) + void OnboardingTelemetry.emit({ + type: "sample_setup_completed", + success: true, + models: sampleContents.models, + tables: sampleContents.tables, + reused: result.reused, + }) + // altimate_change end return { title: result.reused ? `Reused starter sample at ${result.targetPath}` : `Materialized starter sample at ${result.targetPath}`, metadata: { @@ -168,6 +183,15 @@ export const SampleSetupTool = Tool.define("sample_setup", { // reliably detect it from `output` alone — metadata never reaches // the model. const message = err instanceof Error ? err.message : String(err) + // altimate_change — onboarding funnel: failed setup. The error message embeds filesystem + // paths (unsafe HOME, unwritable parent), so it is not sent — only the boolean. + void OnboardingTelemetry.emit({ + type: "sample_setup_completed", + success: false, + models: 0, + tables: 0, + reused: false, + }) return { title: "Starter materialization failed", metadata: { success: false, error: message, targetPath: "", reused: false, suffix: 0, note: "" }, @@ -186,6 +210,42 @@ export const SampleSetupTool = Tool.define("sample_setup", { * The version stamps into the on-disk marker so a future run can detect * whether the materialized copy is current or lags a CLI upgrade. */ +// altimate_change start — onboarding funnel: model/seed counts for sample_setup_completed. +// +// Counted from the shipped source tree rather than read from a constant or from dbt's +// target/manifest.json. A constant silently drifts the first time someone adds a model; +// target/manifest.json is ~17k lines and parsing it to count two things is a waste on a path +// the user is actively waiting on. Counting files can't drift and costs one shallow walk. +// +// Best-effort by construction: telemetry must never fail a sample setup, so any fs error +// yields 0 rather than propagating. +function countFilesWithExtension(dir: string, extension: string): number { + let total = 0 + let entries: fs.Dirent[] + try { + entries = fs.readdirSync(dir, { withFileTypes: true }) + } catch { + return 0 + } + for (const entry of entries) { + if (entry.isDirectory()) { + total += countFilesWithExtension(path.join(dir, entry.name), extension) + } else if (entry.isFile() && entry.name.endsWith(extension)) { + total += 1 + } + } + return total +} + +/** dbt models (`models/**\/*.sql`) and seed tables (`seeds/*.csv`) in the shipped sample. */ +function countSampleContents(sampleSourcePath: string): { models: number; tables: number } { + return { + models: countFilesWithExtension(path.join(sampleSourcePath, "models"), ".sql"), + tables: countFilesWithExtension(path.join(sampleSourcePath, "seeds"), ".csv"), + } +} +// altimate_change end + function readSampleVersionAt(sampleSourcePath: string): string { const manifestPath = path.join(sampleSourcePath, "sample-manifest.json") const raw = fs.readFileSync(manifestPath, "utf8") diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index e93aa4f44..6eb89241c 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -16,6 +16,10 @@ import type { EventSource } from "@opencode-ai/tui/context/sdk" import { writeHeapSnapshot } from "v8" import { validateSession } from "../tui/validate-session" import { win32InstallCtrlCGuard } from "@opencode-ai/tui/terminal-win32" +// altimate_change start — onboarding telemetry: main-thread flush on the TUI exit path +import { Telemetry } from "@/altimate/telemetry" +import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" +// altimate_change end declare global { const OPENCODE_WORKER_PATH: string @@ -224,6 +228,22 @@ export const TuiThreadCommand = cmd({ try { unguard?.() } catch {} + // altimate_change start — flush main-thread telemetry before the explicit exit below. + // This handler ends with process.exit(0), which skips the outer `finally` in src/index.ts + // that normally calls Telemetry.shutdown(). Without this, every event tracked on the TUI + // thread since the last 5s interval flush is lost — including onboarding_abandoned, which + // by definition only fires here. Runs after stop() so the worker has already drained. + // + // Bounded: flush() can block for REQUEST_TIMEOUT_MS (10s) on a slow or blackholed network, + // and that would be a visible hang between the user quitting and the shell prompt + // returning. 2s is the same budget the worker gives its trace drain. + try { + await OnboardingTelemetry.emitAbandonedIfIncomplete() + await withTimeout(Telemetry.shutdown(), 2000) + } catch { + // Never let telemetry delay or break exit. + } + // altimate_change end } process.exit(0) }, diff --git a/packages/opencode/src/cli/tui/worker.ts b/packages/opencode/src/cli/tui/worker.ts index 29c7559de..3d8c64a5c 100644 --- a/packages/opencode/src/cli/tui/worker.ts +++ b/packages/opencode/src/cli/tui/worker.ts @@ -24,6 +24,8 @@ import { disposeAllInstancesAndEmitGlobalDisposed } from "@/server/global-lifecy // does NOT receive this worker's in-process (Server.Default().fetch) session events. import { TraceConsumer } from "@/altimate/observability/trace-consumer" import { Instance } from "@/project/instance" +// altimate_change — onboarding telemetry: flush this thread's buffer in rpc.shutdown() +import { Telemetry } from "@/altimate/telemetry" Heap.start() @@ -111,6 +113,21 @@ export const rpc = { await Promise.race([traceTail, new Promise((r) => setTimeout(r, 2000))]).catch(() => {}) traceConsumer.flushSync() // altimate_change end + // altimate_change start — flush this thread's telemetry buffer before the worker dies. + // The worker loads its own instance of the Telemetry module (separate buffer from the main + // thread), and server-side events — gateway auth, project scan, sample setup, session + // events — land here. cli/cmd/tui.ts terminates the worker immediately after this RPC + // returns, so anything still buffered is lost. + // + // Bounded at 2s: the caller allows 5s total for this RPC and the trace drain above already + // claims up to 2s of it, so an unbounded flush (up to REQUEST_TIMEOUT_MS = 10s) would be + // cut off mid-request by worker.terminate() anyway. + try { + await Promise.race([Telemetry.shutdown(), new Promise((r) => setTimeout(r, 2000))]) + } catch { + // Telemetry must never block worker shutdown. + } + // altimate_change end await InstanceRuntime.disposeAllInstances() if (server) await server.stop(true) }, From 89f7e1e6c7eb444076bd5b775acec5a826d38f5a Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 14:37:45 +0530 Subject: [PATCH 02/14] =?UTF-8?q?fix(onboarding):=20telemetry=20review=20f?= =?UTF-8?q?ixes=20=E2=80=94=20shutdown=20races,=20gateway=20gaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the first telemetry batch, from a review pass over it. Shutdown correctness: - `init()` now waits out an in-flight `shutdown()`. The previous commit only serialized shutdown callers, which was half the problem: `session/prompt.ts` init()s at the start of every session loop and shuts down at the end, so a new session could begin while the previous shutdown was still awaiting `flush()`. `init()` returned the stale `initPromise`, the new session's events landed in `buffer`, and the in-flight `doShutdown()` then cleared that buffer — every event tracked in the gap was lost. - The 2s bounds on the exit paths did not bound anything. `withTimeout()` and `Promise.race()` do not cancel the promise they lose to, so a timed-out flush kept running and reset module state after the caller had already resumed, and the worker's raw race additionally left a live timer behind. The deadline is now applied inside — `shutdown({timeoutMs})` → `flush(timeoutMs)` → the existing `AbortController` — so the request is genuinely aborted and no external timer exists. Gateway funnel gaps: - `gateway_auth_failed` was never emitted when `startCallbackServer()` throws, which is what happens when port 7317 is already in use. That is a real and reasonably common failure, and it occurs before any callback object exists, so it was invisible to the funnel. - Repeated `callback()` invocations re-ran the whole body and could emit completion/failure more than once per attempt, re-reporting a connect time measured from the original attempt. One outcome per attempt now. - `time_to_connect_ms` started after the browser open, excluding the callback server startup and the browser launch — both part of the wait the user actually experiences. It now runs from the top of `authorize()`. Coverage: - `sample_setup_completed(success: false)` now also fires when the shipped sample assets cannot be resolved (a CLI installed without its wrapper package assets). Previously only materialization failures were counted, so failure numbers silently excluded the broken-install case. Also brackets the whole moved `shutdown()` body in `altimate_change` markers; the previous commit left most of it outside, which would conflict awkwardly on a future upstream merge. Known and unchanged: `environment_scan_completed` and `sample_setup_completed` fire on every invocation of their tools, including `/discover` and model-initiated calls, so they are not onboarding-scoped. Scoping them needs the session-level onboarding state that lands with the activation events. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../opencode/src/altimate/plugin/altimate.ts | 43 ++++++++++++++----- .../opencode/src/altimate/telemetry/index.ts | 40 +++++++++++------ .../src/altimate/tools/sample-setup.ts | 10 +++++ packages/opencode/src/cli/cmd/tui.ts | 9 ++-- packages/opencode/src/cli/tui/worker.ts | 5 ++- 5 files changed, 78 insertions(+), 29 deletions(-) diff --git a/packages/opencode/src/altimate/plugin/altimate.ts b/packages/opencode/src/altimate/plugin/altimate.ts index 7ba1be5d2..7052e2fe6 100644 --- a/packages/opencode/src/altimate/plugin/altimate.ts +++ b/packages/opencode/src/altimate/plugin/altimate.ts @@ -184,7 +184,18 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { label: "Altimate LLM Gateway", async authorize() { const state = randomBytes(16).toString("hex") - await startCallbackServer() + // altimate_change start — the attempt starts here, before the callback server and the + // browser open. startCallbackServer() throws when port 7317 is taken, which is a real + // and reasonably common gateway-auth failure that happens before any callback object + // exists — without this catch it would never appear in the funnel. + const startedAt = Date.now() + try { + await startCallbackServer() + } catch (err) { + void OnboardingTelemetry.emit({ type: "gateway_auth_failed", reason: "error" }) + throw err + } + // altimate_change end // Register the pending flow BEFORE opening the browser so an instant // redirect can be matched by state rather than dropped as CSRF. const result = registerPending(state) @@ -213,8 +224,12 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { // attempted". open() failures are swallowed above (the URL is also printed for the // user to paste), so this fires even when no browser actually launched. // The URL is never sent — it carries the CSRF `state`. - const startedAt = Date.now() void OnboardingTelemetry.emit({ type: "gateway_device_code_issued" }) + + // One outcome per attempt. callback() closes over `result` and re-runs its whole body + // on every invocation, so a repeated call would otherwise re-emit completion/failure + // (and re-report a connect time measured from the original attempt). + let outcomeReported = false // altimate_change end return { @@ -242,13 +257,18 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { }) // altimate_change start — onboarding funnel: auth succeeded and the instance // is live. The instance name is the customer's tenant identifier and is never - // sent. time_to_connect_ms is measured from the browser-open above, so it - // belongs to this attempt's closure rather than any shared pending state. - void OnboardingTelemetry.emit({ type: "gateway_auth_completed" }) - void OnboardingTelemetry.emit({ - type: "instance_connected", - time_to_connect_ms: Date.now() - startedAt, - }) + // sent. time_to_connect_ms runs from the start of authorize() — before the + // callback server and browser open, both of which are part of the wait the user + // actually experiences — and lives in this attempt's closure, so a concurrent + // attempt cannot overwrite it. + if (!outcomeReported) { + outcomeReported = true + void OnboardingTelemetry.emit({ type: "gateway_auth_completed" }) + void OnboardingTelemetry.emit({ + type: "instance_connected", + time_to_connect_ms: Date.now() - startedAt, + }) + } // altimate_change end return { type: "success", key: authToken, provider: "altimate-backend" } } catch (err) { @@ -258,7 +278,10 @@ export async function AltimateAuthPlugin(_input: PluginInput): Promise { // altimate_change — onboarding funnel: only the classified enum is sent. The // message can embed the instance name (see the invalid-instance throw above), // so it never reaches telemetry. - void OnboardingTelemetry.emit({ type: "gateway_auth_failed", reason: reasonOf(err) }) + if (!outcomeReported) { + outcomeReported = true + void OnboardingTelemetry.emit({ type: "gateway_auth_failed", reason: reasonOf(err) }) + } return { type: "failed" } } finally { // Keep the shared server up while another flow is still waiting. diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 60784611c..b1a593cf5 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1347,6 +1347,9 @@ export namespace Telemetry { let droppedEvents = 0 let initPromise: Promise | undefined let initDone = false + // altimate_change — in-flight shutdown, declared with the rest of the module state so init() + // above can consult it. See shutdown() for why concurrent shutdowns must be serialized. + let shutdownPromise: Promise | undefined function parseConnectionString(cs: string): AppInsightsConfig | undefined { const parts: Record = {} @@ -1430,7 +1433,13 @@ export namespace Telemetry { // won't race with await init() in session prompt. export function init(): Promise { if (!initPromise) { - initPromise = doInit() + // altimate_change start — never re-init across an in-flight shutdown. + // session/prompt.ts init()s at the start of every session loop and shutdown()s at the end, + // so a new session can begin while the previous shutdown is still awaiting flush(). Without + // this, init() returns immediately, the new session's events land in `buffer`, and the + // in-flight doShutdown() then clears that buffer — losing every event tracked in the gap. + initPromise = shutdownPromise ? shutdownPromise.catch(() => {}).then(doInit) : doInit() + // altimate_change end } return initPromise } @@ -1520,7 +1529,11 @@ export namespace Telemetry { } } - export async function flush() { + // altimate_change — `timeoutMs` lets exit paths bound the flush from the INSIDE. Racing + // flush() against an external timer does not cancel the fetch: the losing promise keeps + // running and can reset module state after the caller has moved on. Threading the deadline + // into the existing AbortController actually aborts the request. + export async function flush(timeoutMs: number = REQUEST_TIMEOUT_MS) { if (!enabled || buffer.length === 0 || !appInsights) return const events = buffer.splice(0, buffer.length) @@ -1538,7 +1551,7 @@ export namespace Telemetry { } const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) + const timeout = setTimeout(() => controller.abort(), timeoutMs) try { const response = await fetch(appInsights.endpoint, { method: "POST", @@ -1578,27 +1591,27 @@ export namespace Telemetry { } // altimate_change end - // altimate_change start — serialize concurrent shutdowns. + // altimate_change start — serialize concurrent shutdowns, and let callers bound the flush. + // // shutdown() is called from several independent paths (session/prompt.ts at the end of each // session loop, the CLI's outer finally, and — for onboarding telemetry — the TUI exit path // and the TUI worker's rpc.shutdown). Two overlapping calls would both enter flush(), which // splices the shared buffer, so one caller can post a half-empty batch while the other drops // events. The in-flight promise is cleared on settle, so a later init/shutdown cycle (the // per-session pattern in prompt.ts) still works. - let shutdownPromise: Promise | undefined - - export async function shutdown() { + // + // `timeoutMs` bounds the flush from the inside. Racing shutdown() against an external timer + // does NOT cancel it: the losing promise keeps running and resets module state after the + // caller has already moved on. Exit paths pass a budget so the fetch itself is aborted. + export async function shutdown(opts?: { timeoutMs?: number }) { if (shutdownPromise) return shutdownPromise - shutdownPromise = doShutdown().finally(() => { + shutdownPromise = doShutdown(opts?.timeoutMs).finally(() => { shutdownPromise = undefined }) return shutdownPromise } - /** Flush and reset. Bound this with `withTimeout` on exit paths — flush() can block for - * REQUEST_TIMEOUT_MS (10s), which is longer than the TUI's shutdown budget. */ - async function doShutdown() { - // altimate_change end + async function doShutdown(timeoutMs?: number) { // Wait for init to complete so we know whether telemetry is enabled // and have a valid endpoint to flush to. init() is fire-and-forget // in CLI middleware, so it may still be in-flight when shutdown runs. @@ -1613,7 +1626,7 @@ export namespace Telemetry { clearInterval(flushTimer) flushTimer = undefined } - await flush() + await flush(timeoutMs) enabled = false appInsights = undefined buffer = [] @@ -1624,4 +1637,5 @@ export namespace Telemetry { initPromise = undefined initDone = false } + // altimate_change end } diff --git a/packages/opencode/src/altimate/tools/sample-setup.ts b/packages/opencode/src/altimate/tools/sample-setup.ts index 683ae607d..4423224cb 100644 --- a/packages/opencode/src/altimate/tools/sample-setup.ts +++ b/packages/opencode/src/altimate/tools/sample-setup.ts @@ -109,6 +109,16 @@ export const SampleSetupTool = Tool.define("sample_setup", { `was installed without its wrapper package assets. Reinstall with: ` + `\`npm i -g @altimateai/altimate-code@latest\`\n\n` + `Underlying error: ${message}` + // altimate_change — onboarding funnel: a broken install (sample assets missing) is a + // sample-setup failure too. Without this the failure count only covers materialization + // errors and silently under-reports the "CLI shipped without its assets" case. + void OnboardingTelemetry.emit({ + type: "sample_setup_completed", + success: false, + models: 0, + tables: 0, + reused: false, + }) return { title: "Starter sample unavailable", metadata: { success: false, error: message, targetPath: "", reused: false, suffix: 0, note: "" }, diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index 6eb89241c..8daa005ef 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -234,12 +234,13 @@ export const TuiThreadCommand = cmd({ // thread since the last 5s interval flush is lost — including onboarding_abandoned, which // by definition only fires here. Runs after stop() so the worker has already drained. // - // Bounded: flush() can block for REQUEST_TIMEOUT_MS (10s) on a slow or blackholed network, - // and that would be a visible hang between the user quitting and the shell prompt - // returning. 2s is the same budget the worker gives its trace drain. + // Bounded from the INSIDE (shutdown → flush → AbortController), not by racing a timer: + // a lost race would leave the flush running and resetting module state after we resumed. + // flush() would otherwise block for REQUEST_TIMEOUT_MS (10s) on a blackholed network — a + // visible hang between the user quitting and the shell prompt returning. try { await OnboardingTelemetry.emitAbandonedIfIncomplete() - await withTimeout(Telemetry.shutdown(), 2000) + await Telemetry.shutdown({ timeoutMs: 2000 }) } catch { // Never let telemetry delay or break exit. } diff --git a/packages/opencode/src/cli/tui/worker.ts b/packages/opencode/src/cli/tui/worker.ts index 3d8c64a5c..42f84ff83 100644 --- a/packages/opencode/src/cli/tui/worker.ts +++ b/packages/opencode/src/cli/tui/worker.ts @@ -121,9 +121,10 @@ export const rpc = { // // Bounded at 2s: the caller allows 5s total for this RPC and the trace drain above already // claims up to 2s of it, so an unbounded flush (up to REQUEST_TIMEOUT_MS = 10s) would be - // cut off mid-request by worker.terminate() anyway. + // cut off mid-request by worker.terminate() anyway. The bound is applied inside shutdown() + // rather than by racing a timer here — racing neither cancels the flush nor clears its timer. try { - await Promise.race([Telemetry.shutdown(), new Promise((r) => setTimeout(r, 2000))]) + await Telemetry.shutdown({ timeoutMs: 2000 }) } catch { // Telemetry must never block worker shutdown. } From 2685a9382963c7d1d1d313d978a8552fdbc8bc0e Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 14:45:40 +0530 Subject: [PATCH 03/14] feat(onboarding): activation funnel events + first_prompt_sent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the four session-scoped funnel events, completing the worker-side half of the taxonomy. The three activation events are DERIVED, and the code says so. The activation menu is not UI — it is text the model writes from the `/onboard-connect` template — and the user picks a job by replying in free text, so neither moment is observable. What is deterministic is which command started the session and which tool ran next, so that is what they infer from: - `activation_menu_shown` — for the `skip` branch, at command dispatch, where the variant is known because there is no scan to wait for. For the `scan` branch, when `project_scan` returns, since the menu follows the scan and the variant depends on what the scan found. Emitting both at dispatch would have over-counted sessions that error out before reaching the menu. - `activation_job_selected` / `first_job_completed` — from the first job-shaped tool call in an onboarding session: `sample_setup`, or the `skill` tool carrying `dbt-analyze`, `sql-review`, or `cost-report`. A job that starts and then fails still counts as selected. `something_else` — the "just let me chat" branch — has no tool signature and therefore cannot be detected at all. Those counts are lower bounds, noted in both the event type and the plugin. `first_prompt_sent` excludes slash commands, which matters because the scan gate submits a hidden `/onboard-connect` as an ordinary user message: without the exclusion it would be recorded as the user's first prompt in every single fresh onboarding. Implemented as a plugin (`command.execute.before`, `tool.execute.after`) rather than as edits scattered through `session/prompt.ts` — the hooks already carry everything needed, and this keeps the inference logic in one fork-owned file. Only `first_prompt_sent` lives in the session loop, next to the existing `session_start`/`task_classified` telemetry it belongs with. Session-scoped state is capped and evicts in insertion order: a `serve` process is long-lived and sees unboundedly many sessions. The scan variant sums all four connection buckets from `project_scan` metadata (`existing`, `new_dbt`, `new_docker`, `new_env`) — there is no pre-summed total. A user whose only warehouse was discovered from a dbt profile, docker compose, or env vars still has a warehouse, and reporting `no_data` would send them down the sample-project branch of the menu. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../altimate/plugin/onboarding-telemetry.ts | 119 ++++++++++++++++++ .../src/altimate/telemetry/onboarding.ts | 73 +++++++++++ packages/opencode/src/plugin/index.ts | 4 + packages/opencode/src/session/prompt.ts | 12 ++ 4 files changed, 208 insertions(+) create mode 100644 packages/opencode/src/altimate/plugin/onboarding-telemetry.ts diff --git a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts new file mode 100644 index 000000000..500b50702 --- /dev/null +++ b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts @@ -0,0 +1,119 @@ +// altimate_change start — activation-funnel telemetry, as a plugin. +// +// The three activation events are DERIVED. The activation menu is not UI: it is text the model +// writes from src/command/template/onboard-connect.txt, and the user picks a job by replying in +// free text. Nothing in the codebase observes either moment. What is deterministic is which +// command started the session and which tool ran next, so that is what this infers from — and +// the events are documented as lower bounds rather than exact counts. +// +// Implemented as a plugin rather than as edits inside session/prompt.ts because the hooks +// already exist (`command.execute.before`, `tool.execute.after`) and carry everything needed. +// This keeps the inference logic in one fork-owned file instead of scattering it across the +// session loop. +import type { Hooks, PluginInput } from "@opencode-ai/plugin" +import * as OnboardingTelemetry from "../telemetry/onboarding" + +const ONBOARD_CONNECT = "onboard-connect" + +/** Tool/skill → the spec's activation job enum. */ +type ActivationJob = "sample_duck_db" | "breaks_downstream" | "sql_review" | "cost" | "something_else" + +const JOB_BY_TOOL: Record = { + sample_setup: "sample_duck_db", +} + +const JOB_BY_SKILL: Record = { + "dbt-analyze": "breaks_downstream", + "sql-review": "sql_review", + "cost-report": "cost", +} + +/** + * Which activation job, if any, a tool call represents. Skills arrive as the generic `skill` + * tool with the skill name in `args.name`, so both shapes have to be checked. + * + * Returns undefined for the many tools that are not jobs (reads, writes, project_scan itself). + * Note that `something_else` — the "just let me chat" branch — has no tool signature at all and + * therefore can never be detected here; that branch is systematically missing from these counts. + */ +function jobForTool(tool: string, args: unknown): ActivationJob | undefined { + if (tool === "skill") { + const name = (args as { name?: unknown } | undefined)?.name + if (typeof name === "string") return JOB_BY_SKILL[name] + return undefined + } + return JOB_BY_TOOL[tool] +} + +/** + * Total warehouse connections a project_scan found, from its `metadata.connections` breakdown + * (`{existing, new_dbt, new_docker, new_env}` — there is no pre-summed total). All four count: + * a user whose only warehouse was discovered from a dbt profile, a docker compose file, or env + * vars still has a warehouse, and reporting `no_data` for them would send them down the + * sample-project branch of the activation menu. + */ +function countScanConnections(metadata: unknown): number { + const connections = (metadata as { connections?: Record } | undefined)?.connections + if (!connections) return 0 + return ["existing", "new_dbt", "new_docker", "new_env"].reduce((total, key) => { + const value = connections[key] + return total + (typeof value === "number" ? value : 0) + }, 0) +} + +/** A tool result is a real completion only if the tool did not report its own failure. */ +function succeeded(output: { metadata?: unknown }): boolean { + const success = (output.metadata as { success?: unknown } | undefined)?.success + // Most tools report no `success` field at all; absence means "did not fail". + return success !== false +} + +export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise { + return { + "command.execute.before": async (input) => { + // Any slash command means the next user message was not typed by the user — needed so + // `first_prompt_sent` measures a real first prompt rather than the scan gate's hidden + // `/onboard-connect` submission. + OnboardingTelemetry.noteCommandSubmission(input.sessionID) + + if (input.command !== ONBOARD_CONNECT) return + OnboardingTelemetry.markOnboardingSession(input.sessionID) + + // `skip` renders the menu immediately, with no scan to wait for, so the variant is known + // now. The `scan` branch cannot be resolved here — the menu follows the scan, and the + // variant depends on what the scan finds — so it is emitted from the scan result below. + if (input.arguments?.trim() === "skip" && OnboardingTelemetry.claimActivationMenu(input.sessionID)) { + void OnboardingTelemetry.emit({ type: "activation_menu_shown", variant: "no_data" }) + } + }, + + "tool.execute.after": async (input, output) => { + if (!OnboardingTelemetry.isOnboardingSession(input.sessionID)) return + + // The scan branch: the menu is rendered right after project_scan returns, and the variant + // is whatever the scan found. This is the closest deterministic proxy for the menu + // actually being shown — closer than the command dispatch, which happens before the agent + // has done anything and would over-count sessions that error out first. + if (input.tool === "project_scan" && OnboardingTelemetry.claimActivationMenu(input.sessionID)) { + void OnboardingTelemetry.emit({ + type: "activation_menu_shown", + variant: countScanConnections(output.metadata) > 0 ? "warehouse" : "no_data", + }) + return + } + + const job = jobForTool(input.tool, input.args) + if (!job) return + + // Selection is inferred from the job starting, so both events land on completion of the + // first job-shaped tool call. A job that starts and then fails still counts as selected. + if (OnboardingTelemetry.claimActivationJobSelected(input.sessionID)) { + void OnboardingTelemetry.emit({ type: "activation_job_selected", job }) + } + if (succeeded(output) && OnboardingTelemetry.claimFirstJobCompleted(input.sessionID)) { + void OnboardingTelemetry.emit({ type: "first_job_completed", job }) + } + }, + } +} +// altimate_change end diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts index 5fe23208c..1a6cf02c2 100644 --- a/packages/opencode/src/altimate/telemetry/onboarding.ts +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -139,10 +139,83 @@ export async function emitAbandonedIfIncomplete(): Promise { await emit({ type: "onboarding_abandoned", last_stage: furthestStage }) } +// --------------------------------------------------------------------------- +// Session-scoped state (worker thread) +// +// The activation events are inferred, not observed: the menu is text the model writes from +// src/command/template/onboard-connect.txt, and the user answers in free text. What IS +// deterministic is which command started the session and which tool ran next, so that is what +// these track. Lives here rather than in the plugin so there is one home for onboarding state. +// +// Bounded: a `serve` process is long-lived and sees unboundedly many sessions, so these must not +// grow forever. Sets are capped and evict in insertion order. +// --------------------------------------------------------------------------- + +const MAX_TRACKED_SESSIONS = 64 + +function addBounded(set: Set, value: T) { + if (set.size >= MAX_TRACKED_SESSIONS) { + const oldest = set.values().next() + if (!oldest.done) set.delete(oldest.value) + } + set.add(value) +} + +/** Sessions whose next user message was submitted by a slash command, not typed by the user. */ +const commandSubmissions = new Set() +/** Sessions started by `/onboard-connect` — used to scope events to the onboarding flow. */ +const onboardingSessions = new Set() +/** Sessions that have already emitted each once-per-session activation event. */ +const activationMenuShown = new Set() +const activationJobSelected = new Set() +const activationJobCompleted = new Set() + +/** Record that the next user message in this session comes from a slash command. */ +export function noteCommandSubmission(sessionID: string) { + addBounded(commandSubmissions, sessionID) +} + +/** True (once) if this session's pending user message was command-submitted. */ +export function consumeCommandSubmission(sessionID: string): boolean { + return commandSubmissions.delete(sessionID) +} + +export function markOnboardingSession(sessionID: string) { + addBounded(onboardingSessions, sessionID) +} + +export function isOnboardingSession(sessionID: string): boolean { + return onboardingSessions.has(sessionID) +} + +/** Claim a once-per-session activation milestone. Returns false if already claimed. */ +export function claimActivationMenu(sessionID: string): boolean { + if (activationMenuShown.has(sessionID)) return false + addBounded(activationMenuShown, sessionID) + return true +} + +export function claimActivationJobSelected(sessionID: string): boolean { + if (activationJobSelected.has(sessionID)) return false + addBounded(activationJobSelected, sessionID) + return true +} + +export function claimFirstJobCompleted(sessionID: string): boolean { + if (activationJobCompleted.has(sessionID)) return false + addBounded(activationJobCompleted, sessionID) + return true +} + /** Test seam — module state is per-process by design, so tests need an explicit reset. */ export function resetForTest() { furthestStage = undefined completed = false abandonedEmitted = false + commandSubmissions.clear() + onboardingSessions.clear() + activationMenuShown.clear() + activationJobSelected.clear() + activationJobCompleted.clear() } // altimate_change end diff --git a/packages/opencode/src/plugin/index.ts b/packages/opencode/src/plugin/index.ts index fcb2dff67..fe1cb9abc 100644 --- a/packages/opencode/src/plugin/index.ts +++ b/packages/opencode/src/plugin/index.ts @@ -30,6 +30,9 @@ import { DatabricksAuthPlugin } from "../altimate/plugin/databricks" // altimate_change start — altimate backend auth plugin import { AltimateAuthPlugin } from "../altimate/plugin/altimate" // altimate_change end +// altimate_change start — onboarding activation-funnel telemetry plugin +import { OnboardingTelemetryPlugin } from "../altimate/plugin/onboarding-telemetry" +// altimate_change end // altimate_change start — wire plugin experimental_workspace.register into the // control-plane adapter registry consumed by control-plane/workspace.ts import { registerAdapter } from "../control-plane/adapters" @@ -60,6 +63,7 @@ export namespace Plugin { SnowflakeCortexAuthPlugin, DatabricksAuthPlugin, AltimateAuthPlugin, + OnboardingTelemetryPlugin, ] // altimate_change end diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 289bd8a74..33869dffc 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -75,6 +75,7 @@ import { Tracer } from "../altimate/observability/tracing" import { stampRegistryToolSource, describeMcpTool } from "../altimate/tool-source" // altimate_change end import { Telemetry } from "@/telemetry" // altimate_change — session telemetry +import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altimate_change — onboarding funnel // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -1010,6 +1011,17 @@ export namespace SessionPrompt { } } // altimate_change end — task intent classification + // altimate_change start — onboarding funnel: first free-text prompt. + // "First" means the user actually typed something, so slash commands are excluded: + // the scan gate submits a hidden `/onboard-connect` as a normal user message + // (packages/tui/src/app.tsx), which would otherwise be counted as the user's first + // prompt in every fresh onboarding. The plugin's command.execute.before hook flags + // command-submitted messages; consuming the flag here also clears it. + const fromCommand = OnboardingTelemetry.consumeCommandSubmission(sessionID) + if (!fromCommand) { + void OnboardingTelemetry.emit({ type: "first_prompt_sent" }) + } + // altimate_change end // altimate_change end — session start telemetry } From f64cf0bc5dc2ee0acbaa158de679cec171ad3e59 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 14:52:11 +0530 Subject: [PATCH 04/14] =?UTF-8?q?fix(onboarding):=20activation-funnel=20re?= =?UTF-8?q?view=20fixes=20=E2=80=94=20scoping,=20attribution,=20claims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the activation batch, from a review pass over it. `first_prompt_sent` was wrong in three ways at once: - It fired on every user turn, not once per session. The enclosing `step === 1` block reads like "first message of the session" but is not: `step` is declared inside `loop()`, and `loop()` runs once per user turn, so the block executes every turn. Now claimed once per session explicitly. - It was not scoped to onboarding at all, so an onboarding-taxonomy event fired for every session in the product — TUI, `run`, GitHub, API callers. - Combined with the above, a resumed session could emit it for a historical message. `first_job_completed` no longer fires on skill load. The `skill` tool loads an instruction bundle and returns; the agent then does the actual work with other tools. Treating that as completion reported "downstream impact analysis complete" the moment the instructions were read. Only `sample_setup`, which really does the work and reports `metadata.success`, is evidence of completion. Skill-driven jobs are now absent from this event rather than wrong in it — `activation_job_selected` still covers them, since selection IS observable. Session attribution: `emit()` now takes the caller's session id instead of always reading `Telemetry.getContext()`. That context is process-global and set by the session loop, so a plugin hook firing for session A while the context pointed at session B misattributed the event — and stamped an empty session for the gateway events, which run before any session exists. Session state is now one record per session instead of five parallel sets. Separate sets could evict a session from some and not others, leaving the worst possible state: still considered "onboarding" but with its once-per-session claims forgotten, so it re-emitted `first` events. One map evicts a session's whole state atomically. Cap raised 64 → 256, since a long-lived `serve` process can hold many concurrent sessions. Note for whoever reads the existing telemetry: the `step === 1` misreading above is not limited to this change. `session_start` and `task_classified` are emitted from the same block and therefore also fire once per user turn rather than once per session, which inflates session counts by the number of turns. That is pre-existing and untouched here. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../altimate/plugin/onboarding-telemetry.ts | 45 +++++--- .../src/altimate/telemetry/onboarding.ts | 104 ++++++++++-------- packages/opencode/src/session/prompt.ts | 23 ++-- 3 files changed, 107 insertions(+), 65 deletions(-) diff --git a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts index 500b50702..16e15fc13 100644 --- a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts +++ b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts @@ -61,11 +61,21 @@ function countScanConnections(metadata: unknown): number { }, 0) } -/** A tool result is a real completion only if the tool did not report its own failure. */ -function succeeded(output: { metadata?: unknown }): boolean { - const success = (output.metadata as { success?: unknown } | undefined)?.success - // Most tools report no `success` field at all; absence means "did not fail". - return success !== false +/** + * Whether a completed tool call is evidence the JOB finished, not merely that a tool returned. + * + * This distinction is the whole reason `first_job_completed` is narrower than + * `activation_job_selected`. `sample_setup` genuinely does the work and reports `metadata.success`. + * The `skill` tool does not: it loads an instruction bundle and returns, after which the agent + * does the actual analysis with other tools. Treating a successful skill load as job completion + * would report "downstream impact analysis complete" the moment the instructions were read. + * + * There is no reliable signal for when a skill-driven job finishes, so those jobs are absent from + * `first_job_completed` rather than wrong in it. + */ +function isJobCompletion(tool: string, output: { metadata?: unknown }): boolean { + if (tool !== "sample_setup") return false + return (output.metadata as { success?: unknown } | undefined)?.success !== false } export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise { @@ -83,7 +93,7 @@ export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise 0 ? "warehouse" : "no_data", - }) + void OnboardingTelemetry.emit( + { + type: "activation_menu_shown", + // `no_data` tracks the template's own menu variant, which keys on whether a warehouse + // connection exists. A dbt project with no warehouse gets the no-data menu too. + variant: countScanConnections(output.metadata) > 0 ? "warehouse" : "no_data", + }, + input.sessionID, + ) return } const job = jobForTool(input.tool, input.args) if (!job) return - // Selection is inferred from the job starting, so both events land on completion of the - // first job-shaped tool call. A job that starts and then fails still counts as selected. + // Selection lands on the first job-shaped tool call. A job that starts and then fails still + // counts as selected — the user did choose it. if (OnboardingTelemetry.claimActivationJobSelected(input.sessionID)) { - void OnboardingTelemetry.emit({ type: "activation_job_selected", job }) + void OnboardingTelemetry.emit({ type: "activation_job_selected", job }, input.sessionID) } - if (succeeded(output) && OnboardingTelemetry.claimFirstJobCompleted(input.sessionID)) { - void OnboardingTelemetry.emit({ type: "first_job_completed", job }) + if (isJobCompletion(input.tool, output) && OnboardingTelemetry.claimFirstJobCompleted(input.sessionID)) { + void OnboardingTelemetry.emit({ type: "first_job_completed", job }, input.sessionID) } }, } diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts index 1a6cf02c2..a61e377da 100644 --- a/packages/opencode/src/altimate/telemetry/onboarding.ts +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -100,7 +100,7 @@ function advance(stage: OnboardingStage) { * Emit an onboarding event. Fire-and-forget by design — onboarding must never wait on, or fail * because of, telemetry. Awaiting the returned promise is only useful on an exit path. */ -export async function emit(event: EmitInput): Promise { +export async function emit(event: EmitInput, sessionID?: string): Promise { try { const stage = STAGE_FOR_EVENT[event.type] if (stage) advance(stage) @@ -110,7 +110,11 @@ export async function emit(event: EmitInput): Promise { Telemetry.track({ ...event, timestamp: Date.now(), - session_id: Telemetry.getContext().sessionId, + // Prefer the caller's session over the ambient telemetry context. setContext() is + // process-global and set by the session loop, so a plugin hook firing for session A while + // the context still points at session B would otherwise misattribute the event — or stamp + // "" when no session has run yet, which is the normal case for the gateway events. + session_id: sessionID ?? Telemetry.getContext().sessionId, } as Telemetry.Event) } catch { // Telemetry must never break onboarding. @@ -151,71 +155,85 @@ export async function emitAbandonedIfIncomplete(): Promise { // grow forever. Sets are capped and evict in insertion order. // --------------------------------------------------------------------------- -const MAX_TRACKED_SESSIONS = 64 +const MAX_TRACKED_SESSIONS = 256 -function addBounded(set: Set, value: T) { - if (set.size >= MAX_TRACKED_SESSIONS) { - const oldest = set.values().next() - if (!oldest.done) set.delete(oldest.value) +/** + * One record per tracked session, rather than a set per flag. Separate sets can evict a session + * from some and not others, which produces the worst possible state: a session still considered + * "onboarding" but with its once-per-session claims forgotten, so it re-emits `first` events. A + * single map evicts a session's whole state atomically — it then simply stops being tracked. + */ +type SessionRecord = { + /** Next user message was submitted by a slash command, not typed. */ + commandSubmission: boolean + /** Session was started by `/onboard-connect`. */ + onboarding: boolean + menuShown: boolean + jobSelected: boolean + jobCompleted: boolean + firstPromptSent: boolean +} + +const sessions = new Map() + +function record(sessionID: string): SessionRecord { + const existing = sessions.get(sessionID) + if (existing) return existing + if (sessions.size >= MAX_TRACKED_SESSIONS) { + const oldest = sessions.keys().next() + if (!oldest.done) sessions.delete(oldest.value) } - set.add(value) + const created: SessionRecord = { + commandSubmission: false, + onboarding: false, + menuShown: false, + jobSelected: false, + jobCompleted: false, + firstPromptSent: false, + } + sessions.set(sessionID, created) + return created } -/** Sessions whose next user message was submitted by a slash command, not typed by the user. */ -const commandSubmissions = new Set() -/** Sessions started by `/onboard-connect` — used to scope events to the onboarding flow. */ -const onboardingSessions = new Set() -/** Sessions that have already emitted each once-per-session activation event. */ -const activationMenuShown = new Set() -const activationJobSelected = new Set() -const activationJobCompleted = new Set() +/** Claim a once-per-session flag. Returns false if it was already claimed. */ +function claim(sessionID: string, key: "menuShown" | "jobSelected" | "jobCompleted" | "firstPromptSent"): boolean { + const entry = record(sessionID) + if (entry[key]) return false + entry[key] = true + return true +} /** Record that the next user message in this session comes from a slash command. */ export function noteCommandSubmission(sessionID: string) { - addBounded(commandSubmissions, sessionID) + record(sessionID).commandSubmission = true } /** True (once) if this session's pending user message was command-submitted. */ export function consumeCommandSubmission(sessionID: string): boolean { - return commandSubmissions.delete(sessionID) + const entry = sessions.get(sessionID) + if (!entry?.commandSubmission) return false + entry.commandSubmission = false + return true } export function markOnboardingSession(sessionID: string) { - addBounded(onboardingSessions, sessionID) + record(sessionID).onboarding = true } export function isOnboardingSession(sessionID: string): boolean { - return onboardingSessions.has(sessionID) + return sessions.get(sessionID)?.onboarding === true } -/** Claim a once-per-session activation milestone. Returns false if already claimed. */ -export function claimActivationMenu(sessionID: string): boolean { - if (activationMenuShown.has(sessionID)) return false - addBounded(activationMenuShown, sessionID) - return true -} - -export function claimActivationJobSelected(sessionID: string): boolean { - if (activationJobSelected.has(sessionID)) return false - addBounded(activationJobSelected, sessionID) - return true -} - -export function claimFirstJobCompleted(sessionID: string): boolean { - if (activationJobCompleted.has(sessionID)) return false - addBounded(activationJobCompleted, sessionID) - return true -} +export const claimActivationMenu = (sessionID: string) => claim(sessionID, "menuShown") +export const claimActivationJobSelected = (sessionID: string) => claim(sessionID, "jobSelected") +export const claimFirstJobCompleted = (sessionID: string) => claim(sessionID, "jobCompleted") +export const claimFirstPrompt = (sessionID: string) => claim(sessionID, "firstPromptSent") /** Test seam — module state is per-process by design, so tests need an explicit reset. */ export function resetForTest() { furthestStage = undefined completed = false abandonedEmitted = false - commandSubmissions.clear() - onboardingSessions.clear() - activationMenuShown.clear() - activationJobSelected.clear() - activationJobCompleted.clear() + sessions.clear() } // altimate_change end diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 33869dffc..da3bed08d 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -1012,14 +1012,23 @@ export namespace SessionPrompt { } // altimate_change end — task intent classification // altimate_change start — onboarding funnel: first free-text prompt. - // "First" means the user actually typed something, so slash commands are excluded: - // the scan gate submits a hidden `/onboard-connect` as a normal user message - // (packages/tui/src/app.tsx), which would otherwise be counted as the user's first - // prompt in every fresh onboarding. The plugin's command.execute.before hook flags - // command-submitted messages; consuming the flag here also clears it. + // + // Three guards, each load-bearing: + // - `step === 1` (the enclosing block) is NOT "first message of the session". `step` + // is declared inside loop() and loop() runs once per user turn, so this block runs + // on every turn. `claimFirstPrompt` is what makes this once-per-session. + // - `isOnboardingSession` scopes it to the funnel. Without it, an onboarding-taxonomy + // event would fire for every session in the product — TUI, `run`, GitHub, API. + // - `consumeCommandSubmission` excludes slash commands, because the scan gate submits + // a hidden `/onboard-connect` as an ordinary user message: it would otherwise be + // recorded as the user's first typed prompt in every fresh onboarding. const fromCommand = OnboardingTelemetry.consumeCommandSubmission(sessionID) - if (!fromCommand) { - void OnboardingTelemetry.emit({ type: "first_prompt_sent" }) + if ( + !fromCommand && + OnboardingTelemetry.isOnboardingSession(sessionID) && + OnboardingTelemetry.claimFirstPrompt(sessionID) + ) { + void OnboardingTelemetry.emit({ type: "first_prompt_sent" }, sessionID) } // altimate_change end // altimate_change end — session start telemetry From 6e716cfee31311631098a138d3d4715610f35d93 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 15:07:57 +0530 Subject: [PATCH 05/14] feat(onboarding): TUI funnel events via an injected telemetry callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the eight TUI-side funnel events, completing the taxonomy. `packages/tui` cannot import `packages/opencode`, where the Telemetry module lives, so the host injects a callback through `TuiInput` and a context carries it to the components — the same seam `context/exit.tsx` already uses for the host's exit function. No HTTP route, no SDK surface, no worker round-trip: the TUI renders on the main thread, so this reaches the process's own Telemetry instance, already initialized by the CLI middleware. The provider is mounted above `DialogProvider`, and that placement is load-bearing: `ui/dialog.tsx` renders dialog contents as a *sibling* of its children, so a provider placed around `` is invisible to every dialog. `dialog-scan-gate.tsx` already documents the same trap for the prompt ref. For the scan gate specifically the event is emitted from the `onChoose` closure in `app.tsx`, reusing the prop bridge that exists for exactly this reason. Events and their anchors: - `onboarding_started` — only on the branch that actually opens the first-run gate, so returning users never enter the funnel. - `model_picker_shown` — carries `trigger`. The picker also opens from `/connect`, from declining Big Pickle, and from the prompt gate; without the discriminator every impression would read as a fresh first run. - `provider_selected` — through a single activation choke point so keyboard and mouse cannot diverge, plus the `/` shortcut, which is the same intent as the "Search all providers…" row. Fires on selection, before auth resolves: a cancelled sign-in still counts as a provider having been chosen. - `big_pickle_confirm_shown` / `big_pickle_choice`, `scan_gate_shown` / `scan_gate_choice` — both choice handlers are now guarded against double-submit. Keyboard and mouse both call them directly and nothing stopped two firing before unmount, which would double-count the choice and, for the scan gate, submit `/onboard-connect` twice. - `onboarding_completed` — the readiness false→true transition, which is exactly the spec's "a model is ready and chat is live". Selecting the gateway provider also marks the auth funnel stage on this thread, because the browser flow runs in the worker and cannot reach the main thread's abandonment state. The `name` → `type` remap in `cli/cmd/tui.ts` is the one untyped point in the chain: the package boundary forces `packages/tui` to declare its own mirror of the event union. Both sides are independently type-checked; a test will pin the two lists together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- packages/opencode/src/cli/cmd/tui.ts | 20 +++++ packages/tui/src/app.tsx | 32 +++++++- .../tui/src/component/altimate-onboarding.tsx | 74 +++++++++++++++++-- .../tui/src/component/dialog-scan-gate.tsx | 7 ++ .../tui/src/context/onboarding-telemetry.tsx | 42 +++++++++++ 5 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 packages/tui/src/context/onboarding-telemetry.tsx diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index 8daa005ef..79875a578 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -207,6 +207,26 @@ export const TuiThreadCommand = cmd({ }, config, pluginHost: createLegacyTuiPluginHost(), + // altimate_change start — onboarding funnel seam. The TUI renders on this thread, so + // this reaches the main-process Telemetry module directly (already initialized by the + // CLI middleware) — no HTTP, no worker round-trip. + // + // The `name` → `type` remap is the one untyped point in the chain: packages/tui + // cannot import the Telemetry event union, so it declares its own mirror in + // context/onboarding-telemetry.tsx. A test pins the two lists together. + // + // Selecting the gateway provider also marks the auth stage, because the browser flow + // itself runs in the worker and cannot reach this thread's abandonment state. + onTelemetry: (event) => { + const { name, ...props } = event + if (name === "provider_selected" && event.provider === "altimate_gateway") { + OnboardingTelemetry.markStage("gateway_auth") + } + void OnboardingTelemetry.emit({ type: name, ...props } as Parameters< + typeof OnboardingTelemetry.emit + >[0]) + }, + // altimate_change end directory: cwd, fetch: transport.fetch, events: transport.events, diff --git a/packages/tui/src/app.tsx b/packages/tui/src/app.tsx index 3310a673a..55bccbe7c 100644 --- a/packages/tui/src/app.tsx +++ b/packages/tui/src/app.tsx @@ -9,6 +9,12 @@ import { UPGRADE_KV_KEY } from "./component/upgrade-indicator-utils" // altimate_change end import { ClipboardProvider, useClipboard } from "./context/clipboard" import { ExitProvider, useExit } from "./context/exit" +// altimate_change — onboarding funnel telemetry seam +import { + OnboardingTelemetryProvider, + useOnboardingTelemetry, + type TrackOnboarding, +} from "./context/onboarding-telemetry" import { EpilogueProvider } from "./context/epilogue" import * as Selection from "./util/selection" import { createCliRenderer, MouseButton, type CliRenderer } from "@opentui/core" @@ -153,6 +159,9 @@ export type TuiInput = { headers?: RequestInit["headers"] events?: EventSource pluginHost: TuiPluginHost + // altimate_change — onboarding funnel telemetry, injected by the host (packages/tui cannot + // reach the Telemetry module). Optional: absent means no tracking, not an error. + onTelemetry?: TrackOnboarding } function errorMessage(error: unknown) { @@ -256,6 +265,11 @@ export const run = Effect.fn("Tui.run")(function* (input: TuiInput) { destroyRenderer(renderer) }} > + {/* altimate_change start — onboarding telemetry seam. Mounted here, above + DialogProvider, because dialog contents render as a sibling of DialogProvider's + children and cannot see anything provided inside it. Defaults to a no-op so + hosts that do not supply a tracker (tests, embedders) work unchanged. */} + {})}> (exit.epilogue = value)}> }> + + {/* altimate_change end */} ) }, renderer) @@ -548,6 +564,8 @@ function App(props: { onSnapshot?: () => Promise; pluginHost: TuiPlugi // plugin startup, not onboarding state. const connected = useConnected() const onboardingReady = useReady() + // altimate_change — onboarding funnel tracker (no-op when the host injected none) + const trackOnboarding = useOnboardingTelemetry() // altimate_change end // altimate_change start — AI-7774: first-run onboarding gate. On a fresh launch @@ -561,7 +579,10 @@ function App(props: { onSnapshot?: () => Promise; pluginHost: TuiPlugi if (!ready()) return // wait until providers are loaded before deciding firstRunPickerHandled = true if (onboardingReady()) return // already set up — no gate - dialog.replace(() => ) + // altimate_change — funnel: top of the first-run flow. Emitted only on the branch that + // actually opens the gate, so returning users never enter the funnel. + trackOnboarding({ name: "onboarding_started" }) + dialog.replace(() => ) }) // altimate_change end @@ -577,9 +598,18 @@ function App(props: { onSnapshot?: () => Promise; pluginHost: TuiPlugi if (scanGateShown) return if (isReady && prev === false) { scanGateShown = true + // altimate_change — funnel: Part 1 finished (a model is ready) and the gate is opening. + // This false→true transition is exactly the ticket's "onboarding_completed" definition, + // so both events are emitted from it. + trackOnboarding({ name: "onboarding_completed" }) + trackOnboarding({ name: "scan_gate_shown" }) dialog.replace(() => ( { + // altimate_change — funnel: emitted here rather than inside the gate because the + // dialog overlay renders outside the provider tree; `onChoose` is already the + // established way to hand the gate something it cannot reach itself. + trackOnboarding({ name: "scan_gate_choice", choice: arg }) // Yes → /onboard-connect scan; No → /onboard-connect skip. // Both branches now have a real follow-up: `scan` runs // project_scan and branches into the discovery UX; `skip` diff --git a/packages/tui/src/component/altimate-onboarding.tsx b/packages/tui/src/component/altimate-onboarding.tsx index 5a29e8378..eb73dbc1e 100644 --- a/packages/tui/src/component/altimate-onboarding.tsx +++ b/packages/tui/src/component/altimate-onboarding.tsx @@ -12,6 +12,8 @@ import { useKeyboard } from "@opentui/solid" import { createDialogProviderOptions } from "./dialog-provider" import { DialogModel } from "./dialog-model" import { useConnected } from "./use-connected" +// altimate_change — onboarding funnel telemetry seam +import { useOnboardingTelemetry } from "../context/onboarding-telemetry" // Session-scoped "setup complete" flag. Set when the user picks a ready model, // chooses the free Big Pickle option, or finishes the gateway flow. Combined with @@ -44,18 +46,33 @@ interface WelcomeRow { note: string tone: WelcomeTone activate: () => void + // altimate_change — funnel: the row's identity in the spec's provider enum. Kept as its own + // field rather than derived from providerID/modelID so the enum cannot drift from the row list + // (the "search all" row has no provider at all, and Big Pickle is a model, not a provider). + analyticsProvider: "altimate_gateway" | "anthropic" | "openai" | "google" | "big_pickle" | "search_all" // Identifies the row for the "currently selected" tick. providerID alone matches // any model of that provider; add modelID to match a specific model (Big Pickle). providerID?: string modelID?: string } -export function DialogModelWelcome(props: { intro?: string }) { +export function DialogModelWelcome(props: { + intro?: string + // altimate_change — funnel: which path opened the picker. It also opens from /connect, from + // declining Big Pickle, and from the prompt gate, so without this every impression would read + // as a fresh first run. Defaults to the /connect case since that is the only caller that does + // not pass one explicitly. + trigger?: "first_run" | "connect_command" | "big_pickle_back" | "prompt_gate" +}) { const { theme } = useTheme() const dialog = useDialog() const local = useLocal() const providers = createDialogProviderOptions() const [selected, setSelected] = createSignal(0) + // altimate_change start — funnel: picker impression + provider choice + const trackOnboarding = useOnboardingTelemetry() + onMount(() => trackOnboarding({ name: "model_picker_shown", trigger: props.trigger ?? "connect_command" })) + // altimate_change end onMount(() => dialog.setSize("large")) @@ -82,6 +99,7 @@ export function DialogModelWelcome(props: { intro?: string }) { tone: "success", providerID: "altimate-backend", activate: () => connectProvider("altimate-backend"), + analyticsProvider: "altimate_gateway", }, { name: "Anthropic (Claude)", @@ -89,6 +107,7 @@ export function DialogModelWelcome(props: { intro?: string }) { tone: "muted", providerID: "anthropic", activate: () => connectProvider("anthropic"), + analyticsProvider: "anthropic", }, { name: "OpenAI (GPT)", @@ -96,6 +115,7 @@ export function DialogModelWelcome(props: { intro?: string }) { tone: "muted", providerID: "openai", activate: () => connectProvider("openai"), + analyticsProvider: "openai", }, { name: "Google (Gemini)", @@ -103,6 +123,7 @@ export function DialogModelWelcome(props: { intro?: string }) { tone: "muted", providerID: "google", activate: () => connectProvider("google"), + analyticsProvider: "google", }, { name: "Big Pickle", @@ -111,8 +132,15 @@ export function DialogModelWelcome(props: { intro?: string }) { providerID: "opencode", modelID: "big-pickle", activate: chooseBigPickle, + analyticsProvider: "big_pickle", + }, + { + name: "Search all providers…", + note: "/", + tone: "muted", + activate: openFullCatalog, + analyticsProvider: "search_all", }, - { name: "Search all providers…", note: "/", tone: "muted", activate: openFullCatalog }, ]) // The currently active model → drives the green "selected" tick. @@ -123,6 +151,14 @@ export function DialogModelWelcome(props: { intro?: string }) { return row.modelID ? c.modelID === row.modelID : true } + // altimate_change — funnel: single choke point for row activation so keyboard and mouse + // cannot diverge. Fires on selection, before auth resolves: a cancelled or failed sign-in + // still counts as a provider having been chosen, which is what the funnel step means. + function activateRow(row: WelcomeRow) { + trackOnboarding({ name: "provider_selected", provider: row.analyticsProvider }) + row.activate() + } + // Indices 0-4 are providers, 5 is the search row (rendered below a divider). const COUNT = 6 function move(direction: number) { @@ -135,12 +171,15 @@ export function DialogModelWelcome(props: { intro?: string }) { if (evt.name === "return") { evt.preventDefault() evt.stopPropagation() - rows()[selected()].activate() + activateRow(rows()[selected()]) return } // "/", ctrl+a, or any letter/number reveals the full searchable catalog. if (evt.name === "/" || (evt.ctrl && evt.name === "a") || /^[a-z0-9]$/i.test(evt.name ?? "")) { evt.preventDefault() + // altimate_change — the "/" shortcut is the same intent as the "Search all providers…" + // row, so it records the same choice. + trackOnboarding({ name: "provider_selected", provider: "search_all" }) openFullCatalog() } }) @@ -150,14 +189,14 @@ export function DialogModelWelcome(props: { intro?: string }) { const noteColor = (tone: WelcomeTone) => tone === "success" ? theme.success : tone === "warning" ? theme.warning : theme.textMuted - const Row = (props: { row: WelcomeRow; index: number }) => { + const Row = (props: { row: WelcomeRow; index: number; onActivate: (row: WelcomeRow) => void }) => { const active = createMemo(() => selected() === props.index) return ( setSelected(props.index)} - onMouseUp={() => props.row.activate()} + onMouseUp={() => props.onActivate(props.row)} > {active() ? "›" : " "} @@ -214,10 +253,10 @@ export function DialogModelWelcome(props: { intro?: string }) { — you can change this anytime with /model - {(row, i) => } + {(row, i) => } - + ) @@ -231,11 +270,30 @@ export function DialogBigPickleConfirm(props: { origin: "welcome" | "model" }) { const dialog = useDialog() const local = useLocal() const [selected, setSelected] = createSignal(0) // 0 = No (default) + // altimate_change start — funnel: interstitial impression + decision. + // `decided` guards against a double-submit: keyboard and mouse handlers both call yes()/no() + // directly, and nothing prevents two firing before the dialog unmounts. + const trackOnboarding = useOnboardingTelemetry() + let decided = false + onMount(() => trackOnboarding({ name: "big_pickle_confirm_shown", origin: props.origin })) + // altimate_change end function no() { - dialog.replace(() => (props.origin === "welcome" ? : )) + // altimate_change start + if (decided) return + decided = true + trackOnboarding({ name: "big_pickle_choice", choice: "cancel" }) + // altimate_change end + dialog.replace(() => + props.origin === "welcome" ? : , + ) } function yes() { + // altimate_change start + if (decided) return + decided = true + trackOnboarding({ name: "big_pickle_choice", choice: "accept" }) + // altimate_change end dialog.clear() local.model.set({ providerID: "opencode", modelID: "big-pickle" }, { recent: true }) markSetupComplete() diff --git a/packages/tui/src/component/dialog-scan-gate.tsx b/packages/tui/src/component/dialog-scan-gate.tsx index 6170209da..ab2d79f06 100644 --- a/packages/tui/src/component/dialog-scan-gate.tsx +++ b/packages/tui/src/component/dialog-scan-gate.tsx @@ -20,7 +20,14 @@ export function DialogScanGate(props: { onChoose: (arg: "scan" | "skip") => void onMount(() => dialog.setSize("large")) + // altimate_change — keyboard (return / y / n) and mouse handlers all call run() directly, and + // nothing stops two firing before the dialog unmounts. Without this guard a fast double-press + // submits `/onboard-connect` twice and double-counts the funnel choice. + let chosen = false + function run(arg: "scan" | "skip") { + if (chosen) return + chosen = true dialog.clear() props.onChoose(arg) } diff --git a/packages/tui/src/context/onboarding-telemetry.tsx b/packages/tui/src/context/onboarding-telemetry.tsx new file mode 100644 index 000000000..ad2ccf37f --- /dev/null +++ b/packages/tui/src/context/onboarding-telemetry.tsx @@ -0,0 +1,42 @@ +// altimate_change start — seam for onboarding funnel telemetry. +// +// `packages/tui` cannot import `packages/opencode`, where the Telemetry module lives, so the +// host injects a callback through `TuiInput` and this context carries it to the components that +// need it. Same shape as `./exit.tsx`, which threads the host's exit function the same way. +// +// PLACEMENT MATTERS: the provider must be mounted ABOVE `DialogProvider`. `ui/dialog.tsx` +// renders dialog contents as a sibling of its children, so a provider placed around `` — +// which is inside DialogProvider — is invisible to every dialog. `dialog-scan-gate.tsx` +// documents the same trap for the prompt ref. +// +// The event union is declared here rather than imported because of that package boundary. Field +// names must match the corresponding `Telemetry.Event` variants in +// `packages/opencode/src/altimate/telemetry/index.ts`; the host maps `name` → `type` and spreads +// the rest. A test pins the two together. +import { createSimpleContext } from "./helper" + +export type OnboardingTelemetryEvent = + | { name: "onboarding_started" } + | { + name: "model_picker_shown" + /** The picker also opens from /connect, from declining Big Pickle, and from the prompt + * gate — without this the event reads as a first-run impression every time. */ + trigger: "first_run" | "connect_command" | "big_pickle_back" | "prompt_gate" + } + | { + name: "provider_selected" + provider: "altimate_gateway" | "anthropic" | "openai" | "google" | "big_pickle" | "search_all" + } + | { name: "big_pickle_confirm_shown"; origin: "welcome" | "model" } + | { name: "big_pickle_choice"; choice: "accept" | "cancel" } + | { name: "scan_gate_shown" } + | { name: "scan_gate_choice"; choice: "scan" | "skip" } + | { name: "onboarding_completed" } + +export type TrackOnboarding = (event: OnboardingTelemetryEvent) => void + +export const { use: useOnboardingTelemetry, provider: OnboardingTelemetryProvider } = createSimpleContext({ + name: "OnboardingTelemetry", + init: (input: { track: TrackOnboarding }) => input.track, +}) +// altimate_change end From 80f4a4ba2511c4f1a62c52f8e7c4145bdadc8264 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 15:14:16 +0530 Subject: [PATCH 06/14] =?UTF-8?q?fix(onboarding):=20TUI=20funnel=20review?= =?UTF-8?q?=20fixes=20=E2=80=94=20abandonment=20scope,=20guards,=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the TUI batch, from a review pass over it. Abandonment was going to be mostly noise. Nearly every piece of UI that emits a funnel event is also reachable outside onboarding: `/connect` opens the same welcome picker, `/model` opens the same Big Pickle interstitial. A returning user doing either advanced the funnel stage to `model_picker`, never emitted `onboarding_completed`, and was therefore reported as ABANDONED on exit. The headline drop-off metric would have been dominated by established users opening `/connect`. Stage tracking is now gated on the funnel actually having started — `onboarding_started`, which is emitted only from the branch that opens the first-run gate. Everything downstream is a no-op outside a real first run, which also fixes the same class of problem for the host marking `gateway_auth` on any gateway selection. Other fixes: - `prompt_gate` was never emitted. The prompt gate opened the picker without passing a trigger, so every "typed a prompt before connecting" impression was misattributed to `/connect`. - `provider_selected` was not fire-once. Keyboard return and mouse-up both reach the activation path with nothing stopping two firings before unmount — that double-counted the choice *and* started the provider flow twice. The `/` shortcut now routes through the same guarded path instead of emitting its own event. - The telemetry context no longer throws when absent. It was built on `createSimpleContext`, which throws for consumers outside its provider; these dialogs are rendered directly by tests and can be reused by other hosts, and analytics is not a reason to crash a UI. Missing provider now means no tracking. The host callback is also isolated, so a throwing tracker cannot propagate out of a mount handler or a keypress and take the dialog transition with it. Known and unchanged: under `OPENCODE_FAST_BOOT`, sync reports ready before credentials are populated, so a returning user can transiently look un-onboarded and see the first-run gate. That is a pre-existing product behaviour — the gate itself already had it — and these events faithfully report what the UI did. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../src/altimate/telemetry/onboarding.ts | 14 ++++++++ .../tui/src/component/altimate-onboarding.tsx | 11 +++++-- packages/tui/src/component/prompt/index.tsx | 5 ++- .../tui/src/context/onboarding-telemetry.tsx | 33 ++++++++++++++++--- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts index a61e377da..4ba0c849b 100644 --- a/packages/opencode/src/altimate/telemetry/onboarding.ts +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -76,6 +76,8 @@ type EmitInput = DistributiveOmit> = { @@ -90,6 +92,17 @@ const STAGE_FOR_EVENT: Partial ( - + )) input.clear() input.extmarks.clear() diff --git a/packages/tui/src/context/onboarding-telemetry.tsx b/packages/tui/src/context/onboarding-telemetry.tsx index ad2ccf37f..89d43a83a 100644 --- a/packages/tui/src/context/onboarding-telemetry.tsx +++ b/packages/tui/src/context/onboarding-telemetry.tsx @@ -13,7 +13,7 @@ // names must match the corresponding `Telemetry.Event` variants in // `packages/opencode/src/altimate/telemetry/index.ts`; the host maps `name` → `type` and spreads // the rest. A test pins the two together. -import { createSimpleContext } from "./helper" +import { createContext, useContext, type ParentProps } from "solid-js" export type OnboardingTelemetryEvent = | { name: "onboarding_started" } @@ -35,8 +35,31 @@ export type OnboardingTelemetryEvent = export type TrackOnboarding = (event: OnboardingTelemetryEvent) => void -export const { use: useOnboardingTelemetry, provider: OnboardingTelemetryProvider } = createSimpleContext({ - name: "OnboardingTelemetry", - init: (input: { track: TrackOnboarding }) => input.track, -}) +const OnboardingTelemetryContext = createContext() + +export function OnboardingTelemetryProvider(props: ParentProps<{ track: TrackOnboarding }>) { + return ( + {props.children} + ) +} + +/** + * Deliberately NOT built on `createSimpleContext`, which throws when a consumer sits outside its + * provider. Telemetry is the one context that must never do that: these dialogs are rendered + * directly by tests and can be reused by other hosts, and analytics is not a reason to crash a + * UI. Missing provider means no tracking. + * + * The host callback is also isolated — a throwing tracker would otherwise propagate out of a + * mount handler or a keypress and take the dialog transition with it. + */ +export function useOnboardingTelemetry(): TrackOnboarding { + const track = useContext(OnboardingTelemetryContext) + return (event) => { + try { + track?.(event) + } catch { + // Telemetry must never break the UI. + } + } +} // altimate_change end From 5c7a7df2d46816d39d927e5a7829f579c6a93d85 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 15:35:17 +0530 Subject: [PATCH 07/14] fix(onboarding): add launch correlation id + full-review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A whole-feature review found the events individually correct but the funnel not reconstructable. These are the fixes. Correlation (see AltimateAI/altimate-code#1048): - Every event now carries `launch_id`, a random value generated once per process and shared across threads through the environment. The funnel spans the TUI main thread and the server worker, and most of it runs before any chat session exists — so the first half of the funnel stamped an empty session and the second half a real one, leaving no key to join a single run on. It is not persisted, not derived from the machine or the user, and not reused across launches. Placed in the event envelope alongside `machine_id`, so it applies to all telemetry rather than only the onboarding events. - `~/.altimate/machine-id` is now created exclusively. Both threads initialise their own copy of the telemetry module, and on a genuinely new install both can find the file missing at the same moment; with a plain write the loser overwrote the winner while each kept its own value in memory, so one first run reported two machine ids. The fallback identity broke on exactly the run it mattered most for. `wx` makes one writer fail and re-read. Abandonment: - `last_stage` no longer advertises values it can never report. `scan_gate` and `activation` both occur after `onboarding_completed`, which sets `completed` and suppresses abandonment entirely — abandonment is by definition quitting before connecting. They are removed rather than documented as live. - Added `provider_setup`, advanced by any provider choice. A user who picked Anthropic and quit during key entry was previously reported as abandoning at `model_picker`, as though they had never chosen anything. Shutdown: - `shutdown({timeoutMs})` no longer ignores the budget when a shutdown is already in flight. It returned the existing promise, which may have been running on the default 10s, so the exit path waited past its 2s budget and the worker was terminated mid-flush anyway. The in-flight flush cannot be shortened, but the caller no longer waits on it beyond its own deadline. Known and unchanged: choosing "Search all providers…" and then picking from the full catalogue records only `provider=search_all`. Fixing that means emitting from the full model picker, which is used outside first-run too, so it needs a product call. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- docs/docs/reference/telemetry.md | 28 ++++++++- .../opencode/src/altimate/telemetry/index.ts | 57 ++++++++++++++++++- .../src/altimate/telemetry/onboarding.ts | 12 ++-- 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/docs/docs/reference/telemetry.md b/docs/docs/reference/telemetry.md index 0a65db1af..e20e4dc3a 100644 --- a/docs/docs/reference/telemetry.md +++ b/docs/docs/reference/telemetry.md @@ -47,8 +47,32 @@ We collect the following categories of events: | `schema_complexity` | Warehouse schema structural metrics from introspection — bucketed table, column, and schema counts plus average columns per table. No schema names or content. | | `validator_check` | A completion-gate validator ran on session end — validator name, `ok` boolean, step, retry count, `enforced` flag (false in shadow mode), and structured `details` (model counts, elapsed time, concurrency limit — no SQL or model content). Only emitted when `ALTIMATE_VALIDATORS_ENABLED=1` or `ALTIMATE_VALIDATORS_SHADOW=1`. See [Validators](../data-engineering/validators.md). | | `validator_retries_exhausted` | A session terminated with unresolved validator failures after exhausting the synthetic-retry budget — names of the failing validators (no failure body content). | - -Each event includes a timestamp, anonymous session ID, CLI version, and an anonymous machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and never tied to any personal information). +| `onboarding_started` | The first-run setup gate opened (fresh launch with no usable model). | +| `model_picker_shown` | The provider picker was displayed. `trigger` distinguishes the first run from `/connect`, from declining Big Pickle, and from the prompt gate. | +| `provider_selected` | A provider row was chosen (`altimate_gateway`, `anthropic`, `openai`, `google`, `big_pickle`, `search_all`). Recorded at the moment of choice, so a sign-in that is then cancelled still counts. | +| `big_pickle_confirm_shown` / `big_pickle_choice` | The Big Pickle interstitial was shown, and what the user decided (`accept`/`cancel`). | +| `gateway_device_code_issued` | The Altimate Gateway authorize URL was built and the browser open attempted. **Name note:** the flow is a browser loopback OAuth — there is no device code. The name follows the original event spec. | +| `gateway_auth_completed` / `gateway_auth_failed` | Gateway sign-in outcome. `reason` is `timeout`, `denied`, or `error` — never the underlying message, which can contain the instance name. An unrecognised callback state does not reject the pending attempt, so a CSRF mismatch surfaces as `timeout`. | +| `instance_connected` | Credentials received and saved. `time_to_connect_ms` runs from the start of the authorize call, so it includes the browser launch. No instance or tenant name is sent. | +| `onboarding_completed` | A model is ready and chat is live. | +| `scan_gate_shown` / `scan_gate_choice` | The "scan your environment?" gate appeared, and whether the user chose `scan` or `skip`. | +| `environment_scan_completed` | A `project_scan` finished — `has_dbt`, `has_warehouse`, `is_repo`, `connections_found`, and a bounded list of short `degraded` detection keys. No paths, hostnames, or connection details. Fires for every scan, including `/discover`, not just onboarding. | +| `sample_setup_completed` | The sample dbt project was materialised. `success`, `models`, `tables`, and `reused` — the tool is deliberately re-callable, so this is per invocation. The target path is never sent. | +| `activation_menu_shown` | The activation menu was (very likely) rendered. `variant` is `warehouse` or `no_data`. **Derived** — see the note below. | +| `activation_job_selected` / `first_job_completed` | Which activation job the user started and, where observable, finished. **Derived** — see the note below. | +| `first_prompt_sent` | The user's first typed message in an onboarding session. Slash commands are excluded, so the hidden `/onboard-connect` submission does not count. | +| `onboarding_abandoned` | The CLI exited during a first run without connecting. `last_stage` is the furthest point reached: `started`, `model_picker`, `provider_setup`, `big_pickle_confirm`, `gateway_auth`, or `connected`. Only emitted for a genuine first run — opening `/connect` as an existing user does not enter the funnel, and abandonment after setup completes is out of scope by definition. | + +### A note on the derived activation events + +`activation_menu_shown`, `activation_job_selected`, and `first_job_completed` are **inferred, not observed**. The activation menu is not a UI element: it is text the model writes from a prompt template, and the user picks a job by replying in free text. Nothing in the CLI can see either moment directly. + +They are therefore inferred from the closest deterministic signals — the menu from the command dispatch or the completed environment scan, the job from the first matching tool or skill invocation that follows. Treat the counts as **lower bounds**, and note two specific gaps: + +- The "something else" branch has no tool signature at all and is never counted. +- `first_job_completed` only fires for jobs with a real completion signal. Skill-driven jobs (downstream impact, SQL review, cost) load an instruction bundle and then do their work through other tools, so their completion is not observable and they are absent from this event rather than wrongly counted in it. + +Each event includes a timestamp, anonymous session ID, a per-launch correlation ID (`launch_id` — a random value regenerated every process start, not persisted and not derived from your machine or identity; it exists only to group events from the same run), CLI version, and an anonymous machine ID (a random UUID stored in `~/.altimate/machine-id`, generated once and never tied to any personal information). ## Delivery & Reliability diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index b1a593cf5..02c66fcce 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1351,6 +1351,22 @@ export namespace Telemetry { // above can consult it. See shutdown() for why concurrent shutdowns must be serialized. let shutdownPromise: Promise | undefined + // altimate_change start — per-launch correlation id, shared across threads via the environment. + // The TUI worker is spawned after the CLI middleware has already initialised telemetry on the + // main thread, and a Worker inherits a copy of process.env, so whichever thread runs first + // publishes the value and the other reads it. Lazy rather than module-init so importing the + // telemetry module (in tests, tooling) does not mint ids nobody uses. + const LAUNCH_ID_ENV = "ALTIMATE_LAUNCH_ID" + + function launchId(): string { + const existing = process.env[LAUNCH_ID_ENV] + if (existing) return existing + const created = randomUUID() + process.env[LAUNCH_ID_ENV] = created + return created + } + // altimate_change end + function parseConnectionString(cs: string): AppInsightsConfig | undefined { const parts: Record = {} for (const segment of cs.split(";")) { @@ -1383,6 +1399,12 @@ export namespace Telemetry { source: clientSource, project_id: fields.project_id ?? projectId, ...(machineId && { machine_id: machineId }), + // altimate_change — groups every event from one process launch. The onboarding funnel + // spans the TUI main thread and the server worker, and most of it runs before any chat + // session exists, so `session_id` is empty for the first half and real for the second — + // leaving no key to join a single run on. This is not persisted, not derived from the + // machine or the user, and not reused across launches; it only says "same run". + launch_id: launchId(), } const measurements: Record = {} @@ -1485,9 +1507,22 @@ export namespace Telemetry { try { machineId = fs.readFileSync(machineIdPath, "utf8").trim() } catch { - machineId = randomUUID() + // altimate_change start — create exclusively so two threads cannot mint different ids. + // The TUI main thread and the server worker each initialise their own copy of this + // module, and on a genuinely new install both can find the file missing at the same + // moment. With a plain write, the loser's value overwrites the winner's while both keep + // their own in memory, so a single first run reports two machine_ids — breaking the + // fallback identity exactly on the run that matters most. `wx` makes one of them fail, + // and the loser re-reads what the winner wrote. + const candidate = randomUUID() fs.mkdirSync(path.dirname(machineIdPath), { recursive: true }) - fs.writeFileSync(machineIdPath, machineId, "utf8") + try { + fs.writeFileSync(machineIdPath, candidate, { encoding: "utf8", flag: "wx" }) + machineId = candidate + } catch { + machineId = fs.readFileSync(machineIdPath, "utf8").trim() + } + // altimate_change end } } catch { // Machine ID unavailable — proceed without it @@ -1604,7 +1639,23 @@ export namespace Telemetry { // does NOT cancel it: the losing promise keeps running and resets module state after the // caller has already moved on. Exit paths pass a budget so the fetch itself is aborted. export async function shutdown(opts?: { timeoutMs?: number }) { - if (shutdownPromise) return shutdownPromise + if (shutdownPromise) { + // An earlier caller is mid-flush, possibly on the default 10s budget. Returning its promise + // unchanged would silently ignore this caller's deadline and make the exit path wait past + // it — after which the worker is terminated anyway and the buffer is lost regardless. We + // cannot shorten the in-flight flush, but we can stop making the caller wait for it. + const budget = opts?.timeoutMs + if (budget === undefined) return shutdownPromise + let timer: ReturnType | undefined + await Promise.race([ + shutdownPromise, + new Promise((resolve) => { + timer = setTimeout(resolve, budget) + }), + ]).catch(() => {}) + if (timer) clearTimeout(timer) + return + } shutdownPromise = doShutdown(opts?.timeoutMs).finally(() => { shutdownPromise = undefined }) diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts index 4ba0c849b..fab554c9a 100644 --- a/packages/opencode/src/altimate/telemetry/onboarding.ts +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -29,11 +29,10 @@ import { Telemetry } from "./index" export const ONBOARDING_STAGES = [ "started", "model_picker", + "provider_setup", "big_pickle_confirm", "gateway_auth", "connected", - "scan_gate", - "activation", ] as const export type OnboardingStage = (typeof ONBOARDING_STAGES)[number] @@ -83,12 +82,17 @@ let funnelStarted = false const STAGE_FOR_EVENT: Partial> = { onboarding_started: "started", model_picker_shown: "model_picker", + // Any provider choice enters setup. Without this, a user who picks Anthropic and quits during + // key entry is reported as abandoning at `model_picker` — as if they never chose anything. + provider_selected: "provider_setup", big_pickle_confirm_shown: "big_pickle_confirm", gateway_device_code_issued: "gateway_auth", instance_connected: "connected", onboarding_completed: "connected", - scan_gate_shown: "scan_gate", - activation_menu_shown: "activation", + // Deliberately no entry for scan_gate_shown / activation_menu_shown. Abandonment is defined as + // quitting BEFORE connecting, and both of those only happen after `onboarding_completed` has + // set `completed`, which suppresses abandonment entirely. Stages for them would be values that + // can never be reported. } function advance(stage: OnboardingStage) { From 0ea7981d0bc59770257624662387ecd9c3d652bc Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 16:15:52 +0530 Subject: [PATCH 08/14] fix(onboarding): don't nest altimate_change markers inside the TUI cleanup block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test/cli/tui/command.test.ts` guards the ordering of the TUI worker cleanup by slicing `cli/cmd/tui.ts` between the "clean up TUI worker after failed --session validation" start marker and the next `altimate_change end`. The onboarding telemetry seam added a nested start/end pair inside that region, so the slice ended early and `await stop()` fell outside it — the guard failed with the cleanup index at -1. Uses a single-line marker instead, and says why in the comment so the next person adding something there does not reintroduce it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- packages/opencode/src/cli/cmd/tui.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index 79875a578..d81ba73b1 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -207,7 +207,12 @@ export const TuiThreadCommand = cmd({ }, config, pluginHost: createLegacyTuiPluginHost(), - // altimate_change start — onboarding funnel seam. The TUI renders on this thread, so + // altimate_change — onboarding funnel seam. Deliberately a single-line marker, not a + // start/end pair: this sits inside the "clean up TUI worker after failed --session + // validation" region, and a nested `altimate_change end` truncates the block that + // test/cli/tui/command.test.ts slices to assert cleanup ordering. + // + // The TUI renders on this thread, so // this reaches the main-process Telemetry module directly (already initialized by the // CLI middleware) — no HTTP, no worker round-trip. // @@ -226,7 +231,6 @@ export const TuiThreadCommand = cmd({ typeof OnboardingTelemetry.emit >[0]) }, - // altimate_change end directory: cwd, fetch: transport.fetch, events: transport.events, From fe2545c1d2a23328a912478f18b1feef685eaf78 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 16:33:39 +0530 Subject: [PATCH 09/14] test(onboarding): funnel emission coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers "does this user action emit that event, with those properties" at each emission point, rather than trying to reconstruct a whole cross-thread stream. Expected values are written out from the product spec rather than computed the way the implementation computes them, so a test cannot agree with the code while both are wrong. Every assertion here was checked by breaking the implementation and confirming the test fails: - removing the funnel-start gate → the returning-user test fails - treating a skill load as job completion → the skill test fails - dropping `launch_id` from the envelope → the correlation test fails - removing the scan-gate guard → both double-submit tests fail Coverage of note: - A returning user opening the picker is not enrolled in the funnel and is never reported as abandoned. - Quitting after choosing a provider reports `provider_setup`, not `model_picker`. - A warehouse discovered from a dbt profile counts as having a warehouse — the adversarial fixture for the connection-total bug, where only `existing` was being read. - Loading a skill selects an activation job but does not complete one. - Tools run outside an onboarding session emit nothing. - `launch_id` is asserted at the envelope level, because it is attached during App Insights conversion and is invisible to a `Telemetry.track` spy — the transport-level test is the only place the correlation fix is observable. Not covered yet: mounting the provider picker and the Big Pickle interstitial needs the full sync/SDK stack (`LocalProvider` depends on both), so those belong with the app-lifecycle harness. The scan gate is covered because its choice is handed out through a prop and needs no telemetry context. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../altimate/telemetry/onboarding.test.ts | 340 ++++++++++++++++++ .../test/cli/tui/dialog-scan-gate.test.tsx | 31 ++ 2 files changed, 371 insertions(+) create mode 100644 packages/opencode/test/altimate/telemetry/onboarding.test.ts diff --git a/packages/opencode/test/altimate/telemetry/onboarding.test.ts b/packages/opencode/test/altimate/telemetry/onboarding.test.ts new file mode 100644 index 000000000..ccf403807 --- /dev/null +++ b/packages/opencode/test/altimate/telemetry/onboarding.test.ts @@ -0,0 +1,340 @@ +// altimate_change — coverage for the onboarding funnel taxonomy. +// +// The question these answer is "does user action X emit event Y, with the right properties". +// Emission is verified by spying on Telemetry.track, so the assertions are about what would be +// sent, not about the transport. +// +// Expected values are written out literally from the product spec rather than derived from the +// implementation — a test that computes its expectation the same way the code does would pass +// even when both are wrong. +import { describe, expect, test, beforeEach, afterEach, spyOn, mock } from "bun:test" +import { Telemetry } from "@/altimate/telemetry" +import * as Onboarding from "@/altimate/telemetry/onboarding" +import { OnboardingTelemetryPlugin } from "@/altimate/plugin/onboarding-telemetry" + +type Tracked = Telemetry.Event + +function captureEvents() { + const events: Tracked[] = [] + // init() reads config and touches the filesystem; the funnel logic under test does not care. + spyOn(Telemetry, "init").mockImplementation(async () => {}) + spyOn(Telemetry, "track").mockImplementation((event: Tracked) => { + events.push(event) + }) + return events +} + +/** Wait for the fire-and-forget `void emit(...)` promises to settle. */ +const settle = () => Bun.sleep(0) + +beforeEach(() => { + Onboarding.resetForTest() +}) + +afterEach(() => { + mock.restore() +}) + +// --------------------------------------------------------------------------- +// Abandonment — the funnel must only contain people who were actually in it +// --------------------------------------------------------------------------- +describe("onboarding abandonment", () => { + test("a returning user who opens the picker is not in the funnel", async () => { + const events = captureEvents() + + // No onboarding_started: this is /connect from an established user, which mounts the very + // same picker. Reaching a stage must not enrol them. + Onboarding.markStage("model_picker") + await Onboarding.emitAbandonedIfIncomplete() + await settle() + + expect(events).toEqual([]) + }) + + test("quitting mid first-run reports the furthest stage reached", async () => { + const events = captureEvents() + + await Onboarding.emit({ type: "onboarding_started" }) + await Onboarding.emit({ type: "model_picker_shown", trigger: "first_run" }) + await Onboarding.emit({ type: "provider_selected", provider: "anthropic" }) + await Onboarding.emitAbandonedIfIncomplete() + await settle() + + const abandoned = events.filter((e) => e.type === "onboarding_abandoned") + expect(abandoned).toHaveLength(1) + // Picking a provider and quitting during key entry is "got as far as setting up a provider", + // not "only ever saw the picker". + expect((abandoned[0] as any).last_stage).toBe("provider_setup") + }) + + test("a completed onboarding is never reported as abandoned", async () => { + const events = captureEvents() + + await Onboarding.emit({ type: "onboarding_started" }) + await Onboarding.emit({ type: "onboarding_completed" }) + await Onboarding.emitAbandonedIfIncomplete() + await settle() + + expect(events.some((e) => e.type === "onboarding_abandoned")).toBe(false) + }) + + test("re-opening the picker later does not walk the stage backwards", async () => { + const events = captureEvents() + + await Onboarding.emit({ type: "onboarding_started" }) + await Onboarding.emit({ type: "gateway_device_code_issued" }) + await Onboarding.emit({ type: "model_picker_shown", trigger: "big_pickle_back" }) + await Onboarding.emitAbandonedIfIncomplete() + await settle() + + const abandoned = events.find((e) => e.type === "onboarding_abandoned") + expect((abandoned as any).last_stage).toBe("gateway_auth") + }) + + test("abandonment fires at most once", async () => { + const events = captureEvents() + + await Onboarding.emit({ type: "onboarding_started" }) + await Onboarding.emitAbandonedIfIncomplete() + await Onboarding.emitAbandonedIfIncomplete() + await settle() + + expect(events.filter((e) => e.type === "onboarding_abandoned")).toHaveLength(1) + }) +}) + +// --------------------------------------------------------------------------- +// Activation — inferred from the plugin hooks +// --------------------------------------------------------------------------- +describe("activation events", () => { + const SESSION = "ses_test" + + async function plugin() { + return OnboardingTelemetryPlugin({} as any) + } + + async function startOnboarding(hooks: any, args: "scan" | "skip") { + await hooks["command.execute.before"]!({ command: "onboard-connect", sessionID: SESSION, arguments: args }, { + parts: [], + }) + } + + test("skipping the scan shows the no-data menu immediately", async () => { + const events = captureEvents() + const hooks = await plugin() + + await startOnboarding(hooks, "skip") + await settle() + + const menu = events.filter((e) => e.type === "activation_menu_shown") + expect(menu).toHaveLength(1) + expect((menu[0] as any).variant).toBe("no_data") + }) + + test("a warehouse discovered from dbt profiles still counts as having a warehouse", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "scan") + + // The adversarial case: nothing is already configured, but the scan discovered a connection + // from a dbt profile. This user HAS a warehouse and must get the warehouse menu — reading + // only `existing` would send them down the sample-project branch. + await hooks["tool.execute.after"]!( + { tool: "project_scan", sessionID: SESSION, callID: "c1", args: {} }, + { title: "", output: "", metadata: { connections: { existing: 0, new_dbt: 1, new_docker: 0, new_env: 0 } } }, + ) + await settle() + + const menu = events.filter((e) => e.type === "activation_menu_shown") + expect(menu).toHaveLength(1) + expect((menu[0] as any).variant).toBe("warehouse") + }) + + test("a scan that finds nothing shows the no-data menu", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "scan") + + await hooks["tool.execute.after"]!( + { tool: "project_scan", sessionID: SESSION, callID: "c1", args: {} }, + { title: "", output: "", metadata: { connections: { existing: 0, new_dbt: 0, new_docker: 0, new_env: 0 } } }, + ) + await settle() + + expect((events.find((e) => e.type === "activation_menu_shown") as any).variant).toBe("no_data") + }) + + test("loading a skill selects a job but does not complete one", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + // The skill tool loads an instruction bundle; the analysis itself happens afterwards through + // other tools. Reporting completion here would claim the job finished the moment the + // instructions were read. + await hooks["tool.execute.after"]!( + { tool: "skill", sessionID: SESSION, callID: "c2", args: { name: "dbt-analyze" } }, + { title: "", output: "", metadata: { name: "dbt-analyze" } }, + ) + await settle() + + const selected = events.filter((e) => e.type === "activation_job_selected") + expect(selected).toHaveLength(1) + expect((selected[0] as any).job).toBe("breaks_downstream") + expect(events.some((e) => e.type === "first_job_completed")).toBe(false) + }) + + test("the sample project both selects and completes a job", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + await hooks["tool.execute.after"]!( + { tool: "sample_setup", sessionID: SESSION, callID: "c3", args: {} }, + { title: "", output: "", metadata: { success: true } }, + ) + await settle() + + expect((events.find((e) => e.type === "activation_job_selected") as any).job).toBe("sample_duck_db") + expect((events.find((e) => e.type === "first_job_completed") as any).job).toBe("sample_duck_db") + }) + + test("a failed sample setup counts as selected but not completed", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + await hooks["tool.execute.after"]!( + { tool: "sample_setup", sessionID: SESSION, callID: "c4", args: {} }, + { title: "", output: "", metadata: { success: false } }, + ) + await settle() + + expect(events.some((e) => e.type === "activation_job_selected")).toBe(true) + expect(events.some((e) => e.type === "first_job_completed")).toBe(false) + }) + + test("only the first job counts as the activation job", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + await hooks["tool.execute.after"]!( + { tool: "sample_setup", sessionID: SESSION, callID: "c5", args: {} }, + { title: "", output: "", metadata: { success: true } }, + ) + await hooks["tool.execute.after"]!( + { tool: "skill", sessionID: SESSION, callID: "c6", args: { name: "sql-review" } }, + { title: "", output: "", metadata: {} }, + ) + await settle() + + expect(events.filter((e) => e.type === "activation_job_selected")).toHaveLength(1) + }) + + test("tools run outside an onboarding session emit nothing", async () => { + const events = captureEvents() + const hooks = await plugin() + + // No /onboard-connect for this session — an ordinary chat where someone happens to use a + // reviewable skill must not look like onboarding activation. + await hooks["tool.execute.after"]!( + { tool: "skill", sessionID: "ses_other", callID: "c7", args: { name: "sql-review" } }, + { title: "", output: "", metadata: {} }, + ) + await settle() + + expect(events).toEqual([]) + }) + + test("helper tools are not mistaken for activation jobs", async () => { + const events = captureEvents() + const hooks = await plugin() + await startOnboarding(hooks, "skip") + + for (const tool of ["read", "bash", "warehouse_add"]) { + await hooks["tool.execute.after"]!( + { tool, sessionID: SESSION, callID: tool, args: {} }, + { title: "", output: "", metadata: {} }, + ) + } + await settle() + + expect(events.some((e) => e.type === "activation_job_selected")).toBe(false) + }) +}) + +// --------------------------------------------------------------------------- +// Slash-command suppression, which first_prompt_sent depends on +// --------------------------------------------------------------------------- +describe("command submission tracking", () => { + test("a command-submitted message is flagged, once", () => { + Onboarding.noteCommandSubmission("ses_a") + expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(true) + expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(false) + }) + + test("a session that never ran a command is not flagged", () => { + expect(Onboarding.consumeCommandSubmission("ses_never")).toBe(false) + }) + + test("the flag does not leak between sessions", () => { + Onboarding.noteCommandSubmission("ses_a") + expect(Onboarding.consumeCommandSubmission("ses_b")).toBe(false) + expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(true) + }) + + test("every slash command is flagged, not just /onboard-connect", async () => { + const hooks = await OnboardingTelemetryPlugin({} as any) + await hooks["command.execute.before"]!( + { command: "discover", sessionID: "ses_c", arguments: "" }, + { parts: [] } as any, + ) + expect(Onboarding.consumeCommandSubmission("ses_c")).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// launch_id — added during envelope conversion, so a track() spy cannot see it +// --------------------------------------------------------------------------- +describe("launch correlation id", () => { + afterEach(async () => { + await Telemetry.shutdown() + mock.restore() + }) + + test("every event in a run carries the same launch_id", async () => { + const origDisabled = process.env.ALTIMATE_TELEMETRY_DISABLED + const origCs = process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + const bodies: string[] = [] + const fetchMock = spyOn(global, "fetch").mockImplementation((async (_input: any, init: any) => { + bodies.push(String(init?.body ?? "")) + return new Response("", { status: 200 }) + }) as unknown as typeof fetch) + + try { + delete process.env.ALTIMATE_TELEMETRY_DISABLED + process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = + "InstrumentationKey=k;IngestionEndpoint=https://example.invalid" + await Telemetry.init() + + Telemetry.track({ type: "onboarding_started", timestamp: 1, session_id: "" }) + Telemetry.track({ type: "scan_gate_choice", timestamp: 2, session_id: "ses_1", choice: "scan" }) + await Telemetry.flush() + + const envelopes = JSON.parse(bodies[0]) + expect(envelopes).toHaveLength(2) + + const ids = envelopes.map((e: any) => e.data.baseData.properties.launch_id) + expect(ids[0]).toBeTruthy() + // The whole point: an event emitted before any session exists and one emitted with a real + // session must still be joinable to the same run. + expect(ids[0]).toBe(ids[1]) + } finally { + process.env.ALTIMATE_TELEMETRY_DISABLED = origDisabled + if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs + else delete process.env.APPLICATIONINSIGHTS_CONNECTION_STRING + fetchMock.mockRestore() + } + }) +}) diff --git a/packages/tui/test/cli/tui/dialog-scan-gate.test.tsx b/packages/tui/test/cli/tui/dialog-scan-gate.test.tsx index 280cdc61d..0afa62f88 100644 --- a/packages/tui/test/cli/tui/dialog-scan-gate.test.tsx +++ b/packages/tui/test/cli/tui/dialog-scan-gate.test.tsx @@ -136,3 +136,34 @@ test("scan gate: Enter selects the default (Yes) and chooses scan", async () => await gate.cleanup() } }) + +// altimate_change — the gate submits `/onboard-connect` and records the funnel choice, so a +// double-fire both double-counts the event and runs the command twice. Keyboard and mouse +// handlers call run() directly with nothing between them and the dialog unmounting. +test("scan gate: a rapid second keypress does not choose twice", async () => { + await using tmp = await tmpdir() + const gate = await mountGate(tmp.path) + try { + gate.app.mockInput.pressKey("y") + gate.app.mockInput.pressKey("y") + await wait(() => gate.chosen.length > 0) + await Bun.sleep(50) + expect(gate.chosen).toEqual(["scan"]) + } finally { + await gate.cleanup() + } +}) + +test("scan gate: y then n keeps the first choice", async () => { + await using tmp = await tmpdir() + const gate = await mountGate(tmp.path) + try { + gate.app.mockInput.pressKey("y") + gate.app.mockInput.pressKey("n") + await wait(() => gate.chosen.length > 0) + await Bun.sleep(50) + expect(gate.chosen).toEqual(["scan"]) + } finally { + await gate.cleanup() + } +}) From 9e1a521d74d874896409506296b39ca09df97431 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 16:44:29 +0530 Subject: [PATCH 10/14] test(onboarding): cover the first-run provider picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The picker was the last uncovered emission point. I had written it off as needing the app-lifecycle harness because `LocalProvider` depends on sync and the SDK — that was wrong. `test/fixture/tui-sdk.ts` already provides a fake fetch and event source, so the real provider stack mounts fine. Covered: the impression carries the trigger that opened the picker; an unset trigger is attributed to `/connect` rather than to a first run; choosing the first row records the gateway; a rapid second Enter records one selection, not two; and the `/` shortcut records the same choice as the search row. Verified by mutation, as with the other funnel tests — removing the activation guard fails the double-Enter test, and changing the default trigger fails the attribution test. The trigger parameter is typed by extraction from the event union, so renaming a trigger breaks this test at compile time rather than silently drifting. The fake SDK serves an empty provider list. That is fine here: the curated rows are hardcoded and the funnel event is recorded at the moment of choice, before the provider lookup that would act on it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../cli/tui/dialog-model-welcome.test.tsx | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 packages/tui/test/cli/tui/dialog-model-welcome.test.tsx diff --git a/packages/tui/test/cli/tui/dialog-model-welcome.test.tsx b/packages/tui/test/cli/tui/dialog-model-welcome.test.tsx new file mode 100644 index 000000000..574509030 --- /dev/null +++ b/packages/tui/test/cli/tui/dialog-model-welcome.test.tsx @@ -0,0 +1,185 @@ +/** @jsxImportSource @opentui/solid */ +// altimate_change — onboarding funnel coverage for the curated first-run picker. +// +// Mounts DialogModelWelcome in the real provider stack with a capturing telemetry tracker, and +// checks that a user action produces the event an analyst would expect. The fake SDK serves an +// empty provider list, which is fine here: the curated rows are hardcoded, and the funnel event +// is recorded at the moment of choice — before the provider lookup that would act on it. +import { createDefaultOpenTuiKeymap } from "@opentui/keymap/opentui" +import { testRender, useRenderer } from "@opentui/solid" +import { expect, test } from "bun:test" +import { onCleanup } from "solid-js" +import { createTuiResolvedConfig } from "../../fixture/tui-runtime" +import { TestTuiContexts } from "../../fixture/tui-environment" +import { createEventSource, createFetch, directory } from "../../fixture/tui-sdk" +import type { OnboardingTelemetryEvent } from "../../../src/context/onboarding-telemetry" + +async function wait(fn: () => boolean, timeout = 2000) { + const start = Date.now() + while (!fn()) { + if (Date.now() - start > timeout) throw new Error("timed out waiting for condition") + await Bun.sleep(10) + } +} + +/** Pinned to the real event union, so a renamed trigger breaks this test at compile time. */ +type PickerTrigger = Extract["trigger"] + +async function mountPicker(trigger?: PickerTrigger) { + const [ + { DialogProvider }, + { DialogModelWelcome }, + { OnboardingTelemetryProvider }, + { ArgsProvider }, + { KVProvider }, + { ThemeProvider }, + { TuiConfigProvider }, + { ToastProvider }, + { SDKProvider }, + { ProjectProvider }, + { SyncProvider }, + { LocalProvider }, + { OpencodeKeymapProvider, registerOpencodeKeymap }, + { ExitProvider }, + { RouteProvider }, + ] = await Promise.all([ + import("../../../src/ui/dialog"), + import("../../../src/component/altimate-onboarding"), + import("../../../src/context/onboarding-telemetry"), + import("../../../src/context/args"), + import("../../../src/context/kv"), + import("../../../src/context/theme"), + import("../../../src/config"), + import("../../../src/ui/toast"), + import("../../../src/context/sdk"), + import("../../../src/context/project"), + import("../../../src/context/sync"), + import("../../../src/context/local"), + import("../../../src/keymap"), + import("../../../src/context/exit"), + import("../../../src/context/route"), + ]) + + const events: OnboardingTelemetryEvent[] = [] + const calls = createFetch() + const source = createEventSource() + + function Harness() { + const renderer = useRenderer() + const keymap = createDefaultOpenTuiKeymap(renderer) + const resolvedConfig = createTuiResolvedConfig({ leader_timeout: 1000 }) + const off = registerOpencodeKeymap(keymap, renderer, resolvedConfig) + onCleanup(off) + + return ( + + {}}> + + + + + + + + + + + + {/* above DialogProvider, mirroring app.tsx */} + events.push(e)}> + + + + + + + + + + + + + + + + + + ) + } + + const app = await testRender(() => , { kittyKeyboard: true }) + await app.renderOnce() + await Bun.sleep(50) + await app.renderOnce() + return { + app, + events, + async cleanup() { + app.renderer.destroy() + }, + } +} + +test("picker records an impression with the trigger that opened it", async () => { + const picker = await mountPicker("first_run") + try { + await wait(() => picker.events.length > 0) + expect(picker.events[0]).toEqual({ name: "model_picker_shown", trigger: "first_run" }) + } finally { + await picker.cleanup() + } +}) + +test("picker opened without an explicit trigger is attributed to /connect", async () => { + const picker = await mountPicker() + try { + await wait(() => picker.events.length > 0) + expect(picker.events[0]).toEqual({ name: "model_picker_shown", trigger: "connect_command" }) + } finally { + await picker.cleanup() + } +}) + +test("choosing the first row records the gateway provider", async () => { + const picker = await mountPicker("first_run") + try { + await wait(() => picker.events.length > 0) + picker.app.mockInput.pressEnter() + await wait(() => picker.events.some((e) => e.name === "provider_selected")) + + const selected = picker.events.filter((e) => e.name === "provider_selected") + expect(selected).toEqual([{ name: "provider_selected", provider: "altimate_gateway" }]) + } finally { + await picker.cleanup() + } +}) + +test("a rapid second Enter does not record two selections", async () => { + const picker = await mountPicker("first_run") + try { + await wait(() => picker.events.length > 0) + picker.app.mockInput.pressEnter() + picker.app.mockInput.pressEnter() + await wait(() => picker.events.some((e) => e.name === "provider_selected")) + await Bun.sleep(50) + + expect(picker.events.filter((e) => e.name === "provider_selected")).toHaveLength(1) + } finally { + await picker.cleanup() + } +}) + +test("the / shortcut records the same choice as the search row", async () => { + const picker = await mountPicker("first_run") + try { + await wait(() => picker.events.length > 0) + picker.app.mockInput.pressKey("/") + await wait(() => picker.events.some((e) => e.name === "provider_selected")) + + expect(picker.events.filter((e) => e.name === "provider_selected")).toEqual([ + { name: "provider_selected", provider: "search_all" }, + ]) + } finally { + await picker.cleanup() + } +}) From 4ce542bb2b8ad156d09698f1aaf46dcc24697078 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 18:45:59 +0530 Subject: [PATCH 11/14] fix(onboarding): actually share launch_id between the TUI and worker threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by driving a real CLI process through a PTY against a local telemetry sink: the eight TUI-thread events arrived with one launch_id and the worker-thread events with a different one. The correlation id could not join the two halves of the funnel — which is the only thing it exists to do. Two approaches that look correct are not, both confirmed end to end: - Mutating `process.env` at runtime. A Bun Worker does not observe it, so the worker minted its own id. This is what the original implementation did, and it was lazy as well, so the value did not exist until the first flush — long after the worker had started. - Deriving it from the process (pid + start time). Both threads really are one process and share a pid, but `process.uptime()` is per-THREAD in Bun, so each thread computed a different start time. What works is handing the id to the worker explicitly at construction, which Bun's Worker supports through its `env` option. Verified: all eleven events from a full first run — across both threads and two different session ids — now carry a single launch_id. Also loosens the launch_id unit test to select envelopes by event name instead of by position. The telemetry buffer is module-global, so a sibling test file can leave events in it that flush alongside these, which made the exact-length assertion fail only when the files ran together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- packages/opencode/src/altimate/telemetry/index.ts | 10 +++++++++- packages/opencode/src/cli/cmd/tui.ts | 7 ++++++- .../test/altimate/telemetry/onboarding.test.ts | 15 ++++++++++----- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 02c66fcce..7d02fefd8 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -1356,9 +1356,17 @@ export namespace Telemetry { // main thread, and a Worker inherits a copy of process.env, so whichever thread runs first // publishes the value and the other reads it. Lazy rather than module-init so importing the // telemetry module (in tests, tooling) does not mint ids nobody uses. + // One id per process launch, shared with the TUI's server Worker through its environment. + // + // It has to be handed over explicitly at Worker construction (see cli/cmd/tui.ts). Two things + // that look like they would work do not, both confirmed end to end: + // - mutating process.env after startup: a Bun Worker does not observe it, so the worker mints + // its own id and the two halves of the funnel become unjoinable; + // - deriving it from the process (pid + start time): process.uptime() is per-THREAD in Bun, + // so the worker computes a different start time than the main thread. const LAUNCH_ID_ENV = "ALTIMATE_LAUNCH_ID" - function launchId(): string { + export function launchId(): string { const existing = process.env[LAUNCH_ID_ENV] if (existing) return existing const created = randomUUID() diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index d81ba73b1..403eaadc5 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -135,7 +135,12 @@ export const TuiThreadCommand = cmd({ } const cwd = Filesystem.resolve(process.cwd()) - const worker = new Worker(file) + // altimate_change — hand the launch correlation id to the worker explicitly. A Bun Worker + // does not see runtime mutations to process.env, so without this the worker mints its own + // and the TUI-thread and worker-thread halves of the onboarding funnel cannot be joined. + const worker = new Worker(file, { + env: { ...process.env, ALTIMATE_LAUNCH_ID: Telemetry.launchId() }, + } as WorkerOptions) const client = Rpc.client(worker) const reload = () => { client.call("reload", undefined).catch(() => {}) diff --git a/packages/opencode/test/altimate/telemetry/onboarding.test.ts b/packages/opencode/test/altimate/telemetry/onboarding.test.ts index ccf403807..804304d9c 100644 --- a/packages/opencode/test/altimate/telemetry/onboarding.test.ts +++ b/packages/opencode/test/altimate/telemetry/onboarding.test.ts @@ -322,14 +322,19 @@ describe("launch correlation id", () => { Telemetry.track({ type: "scan_gate_choice", timestamp: 2, session_id: "ses_1", choice: "scan" }) await Telemetry.flush() - const envelopes = JSON.parse(bodies[0]) - expect(envelopes).toHaveLength(2) + // Select by name rather than by position: the telemetry buffer is module-global, so a + // sibling test file can leave events in it and they flush alongside these. + const envelopes = JSON.parse(bodies[0]) as any[] + const byName = (name: string) => envelopes.find((e) => e.data.baseData.name === name) + const preSession = byName("onboarding_started") + const withSession = byName("scan_gate_choice") + expect(preSession).toBeDefined() + expect(withSession).toBeDefined() - const ids = envelopes.map((e: any) => e.data.baseData.properties.launch_id) - expect(ids[0]).toBeTruthy() // The whole point: an event emitted before any session exists and one emitted with a real // session must still be joinable to the same run. - expect(ids[0]).toBe(ids[1]) + expect(preSession.data.baseData.properties.launch_id).toBeTruthy() + expect(preSession.data.baseData.properties.launch_id).toBe(withSession.data.baseData.properties.launch_id) } finally { process.env.ALTIMATE_TELEMETRY_DISABLED = origDisabled if (origCs !== undefined) process.env.APPLICATIONINSIGHTS_CONNECTION_STRING = origCs From 8ac19280c90fbf92df493f359eb244d4ad464b10 Mon Sep 17 00:00:00 2001 From: Haider Date: Wed, 29 Jul 2026 19:47:11 +0530 Subject: [PATCH 12/14] fix(onboarding): attribute tool-emitted funnel events to their own session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `environment_scan_completed` and `sample_setup_completed` were emitted without a session, so they fell back to the process-global telemetry context — which is set per prompt loop. With two concurrent sessions in `serve`, or in the TUI worker, one session overwrites the other's context and a scan or sample setup gets attributed to whichever session most recently started a turn. Both tools have their own `ctx.sessionID`. The activation plugin was already passing it explicitly; these two were not. Same root cause as the launch-id bug: state that looks process-wide is not describing what you assume it describes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- packages/opencode/src/altimate/tools/project-scan.ts | 6 +++++- packages/opencode/src/altimate/tools/sample-setup.ts | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/opencode/src/altimate/tools/project-scan.ts b/packages/opencode/src/altimate/tools/project-scan.ts index de051d2f9..a636a06c8 100644 --- a/packages/opencode/src/altimate/tools/project-scan.ts +++ b/packages/opencode/src/altimate/tools/project-scan.ts @@ -952,7 +952,11 @@ export const ProjectScanTool = Tool.define("project_scan", { is_repo: git.isRepo, connections_found: totalConnections, degraded: degradedList, - }) + // Explicit session: emit() otherwise falls back to the process-global telemetry context, + // which is set per prompt loop. Two concurrent sessions in `serve` or in the TUI worker + // overwrite each other's context, so a scan would be attributed to whichever session most + // recently started a turn. + }, ctx.sessionID) // altimate_change end // Build metadata diff --git a/packages/opencode/src/altimate/tools/sample-setup.ts b/packages/opencode/src/altimate/tools/sample-setup.ts index 4423224cb..161879288 100644 --- a/packages/opencode/src/altimate/tools/sample-setup.ts +++ b/packages/opencode/src/altimate/tools/sample-setup.ts @@ -82,7 +82,7 @@ export const SampleSetupTool = Tool.define("sample_setup", { "conflict prompt. Mutually exclusive with allow_in_place_upgrade; alongside wins.", ), }), - async execute(args, _ctx) { + async execute(args, ctx) { const sampleName = DEFAULT_SAMPLE_NAME // Resolve the sample source ONCE per invocation and pass it forward. // materializeSample() would otherwise call resolveSampleSource() again @@ -118,7 +118,7 @@ export const SampleSetupTool = Tool.define("sample_setup", { models: 0, tables: 0, reused: false, - }) + }, ctx.sessionID) return { title: "Starter sample unavailable", metadata: { success: false, error: message, targetPath: "", reused: false, suffix: 0, note: "" }, @@ -171,7 +171,7 @@ export const SampleSetupTool = Tool.define("sample_setup", { models: sampleContents.models, tables: sampleContents.tables, reused: result.reused, - }) + }, ctx.sessionID) // altimate_change end return { title: result.reused ? `Reused starter sample at ${result.targetPath}` : `Materialized starter sample at ${result.targetPath}`, @@ -201,7 +201,7 @@ export const SampleSetupTool = Tool.define("sample_setup", { models: 0, tables: 0, reused: false, - }) + }, ctx.sessionID) return { title: "Starter materialization failed", metadata: { success: false, error: message, targetPath: "", reused: false, suffix: 0, note: "" }, From 447f8b6ff3068d70ebb08947605395245ddf026b Mon Sep 17 00:00:00 2001 From: Haider Date: Thu, 30 Jul 2026 19:52:02 +0530 Subject: [PATCH 13/14] fix(onboarding): drive the scan gate from setup completion, not readiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Part 2 gate was triggered by useReady(), which is `connected() || setupComplete()`. `connected()` flips as soon as a provider lands in sync data — and the BYOK confirm handlers do exactly that inside `await sync.bootstrap()`, before going on to open the model picker (dialog-provider ApiMethod and CodeMethod). So for every BYOK provider the gate mounted mid-handler, the handler's own `dialog.replace()` destroyed it a moment later, and the one-shot latch meant it never came back. `/onboard-connect` was never submitted, which made activation_menu_shown, activation_job_selected, first_job_completed, sample_setup_completed and first_prompt_sent unreachable on the majority onboarding path — while onboarding_completed and scan_gate_shown were still reported for a gate the user never saw. The gateway path was unaffected because its success branch does not replace the dialog, which is why this survived manual testing and the Big Pickle E2E. Now driven by setup completion alone, which is only set once a model is genuinely chosen — the model picker, the Big Pickle accept path, and the gateway auto-select. That is also what the spec means by "a model is ready". Reported in consensus review of #1049 (CRITICAL), flagged independently by two reviewers, with the ordering verified against sync.tsx: the store write is inside `batch()` within an awaited promise chain, two `.then()` hops before bootstrap() resolves, so the effect flushes before the dialog.replace under either effect-scheduling model. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- packages/tui/src/app.tsx | 29 +++++++++++++------ .../tui/src/component/altimate-onboarding.tsx | 13 +++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/tui/src/app.tsx b/packages/tui/src/app.tsx index 55bccbe7c..f6b8976d7 100644 --- a/packages/tui/src/app.tsx +++ b/packages/tui/src/app.tsx @@ -37,7 +37,7 @@ import { DialogProvider, useDialog } from "./ui/dialog" // altimate_change start — /auth (gateway sign-in) + /connect (curated welcome picker) // + /logout commands import { DialogAltimateAuth } from "./component/dialog-provider" -import { DialogModelWelcome, useReady, resetSetupComplete } from "./component/altimate-onboarding" +import { DialogModelWelcome, useReady, useSetupComplete, resetSetupComplete } from "./component/altimate-onboarding" // altimate_change end // altimate_change — Part 2 scan gate (fires once when Part 1 first completes) import { DialogScanGate } from "./component/dialog-scan-gate" @@ -564,6 +564,8 @@ function App(props: { onSnapshot?: () => Promise; pluginHost: TuiPlugi // plugin startup, not onboarding state. const connected = useConnected() const onboardingReady = useReady() + // altimate_change — setup completion alone (no `connected()` term); see the scan-gate effect + const setupComplete = useSetupComplete() // altimate_change — onboarding funnel tracker (no-op when the host injected none) const trackOnboarding = useOnboardingTelemetry() // altimate_change end @@ -586,17 +588,26 @@ function App(props: { onSnapshot?: () => Promise; pluginHost: TuiPlugi }) // altimate_change end - // altimate_change start — Part 2 scan gate: fire EXACTLY once, on the first time - // this session reaches "ready" from a not-ready start (i.e. the user just finished - // Part 1 in a fresh onboarding). `prev === false` guarantees a genuine false→true - // transition, so a returning user (ready from launch, prev === undefined) never - // sees it, and a later /model change (ready stays true, no transition) never - // re-triggers it. We do NOT auto-scan — the gate just asks. + // altimate_change start — Part 2 scan gate: fire EXACTLY once, when the user has actually + // finished picking a model. `prev === false` guarantees a genuine false→true transition, so a + // returning user (complete from launch, prev === undefined) never sees it, and a later /model + // change (stays true, no transition) never re-triggers it. We do NOT auto-scan — the gate asks. + // + // Driven by setupComplete, NOT useReady(). useReady() also counts `connected()`, which flips as + // soon as a provider lands in sync data — and the BYOK confirm handlers do exactly that inside + // `await sync.bootstrap()` before going on to open the model picker + // (dialog-provider.tsx ApiMethod/CodeMethod). Triggering on readiness therefore mounted this + // gate mid-handler, the handler's own `dialog.replace()` destroyed it a moment + // later, and the one-shot latch meant it never came back: `/onboard-connect` was never + // submitted, so every activation event was unreachable for BYOK users, while + // `onboarding_completed` and `scan_gate_shown` were still reported. setupComplete is only set + // once a model is genuinely chosen (dialog-model.tsx, the Big Pickle accept path, and the + // gateway auto-select), which is what both this gate and the spec actually mean. let scanGateShown = false createEffect( - on(onboardingReady, (isReady, prev) => { + on(setupComplete, (isComplete, prev) => { if (scanGateShown) return - if (isReady && prev === false) { + if (isComplete && prev === false) { scanGateShown = true // altimate_change — funnel: Part 1 finished (a model is ready) and the gate is opening. // This false→true transition is exactly the ticket's "onboarding_completed" definition, diff --git a/packages/tui/src/component/altimate-onboarding.tsx b/packages/tui/src/component/altimate-onboarding.tsx index 7055a752e..755cb2b05 100644 --- a/packages/tui/src/component/altimate-onboarding.tsx +++ b/packages/tui/src/component/altimate-onboarding.tsx @@ -34,6 +34,19 @@ export function useReady() { return createMemo(() => connected() || setupComplete()) } +/** + * Setup completion ONLY — deliberately without the `connected()` term. + * + * `connected()` flips as soon as a provider appears in sync data, which happens inside + * `await sync.bootstrap()` in the BYOK confirm handlers — before those handlers go on to open the + * model picker. Anything driven off `useReady()` therefore fires while the user still has no model + * selected, and is then immediately replaced by that picker. Use this accessor for "the user has + * finished setting up", and `useReady()` only for "is chat usable at all". + */ +export function useSetupComplete() { + return setupComplete +} + // First-run welcome picker (presentation only; reuses the same action handlers as // DialogModel/createDialogProviderOptions). A curated six: five recommended // providers + a "Search all providers…" row that hands off to the full DialogModel From 8e369a8b8fa6f59fc5a90c2760513082168db987 Mon Sep 17 00:00:00 2001 From: Haider Date: Thu, 30 Jul 2026 20:05:53 +0530 Subject: [PATCH 14/14] fix(onboarding): consensus review fixes for #1049 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the findings from the review that fall inside this PR. MAJOR — markSetupComplete() ran before the model-availability check (dialog-provider). The branch below it deliberately refuses to claim success when the gateway connects but offers nothing usable, and said so in a comment — but completion had already been marked, so telemetry reported a finished onboarding while the user was told to go pick a model. Now marked where the model is actually set. This matters more since the scan gate moved onto the same signal. MAJOR — the flushTimer leak. doInit() assigned a new interval without clearing the previous handle, and shutdown() only ever clears the current one, so a second doInit() stranded a timer for the life of the process. doShutdown also nulled initPromise unconditionally, discarding a doInit() that init() had chained onto the in-flight shutdown. Both fixed; see the note in doShutdown on why only the first is covered by a test. MAJOR — instance_connected and onboarding_abandoned could both be reported for one launch. The gateway success events are emitted on the worker thread while abandonment state is main-thread-owned, so a user who finished in the browser and quit before the TUI observed the new provider was reported as abandoning at gateway_auth. The exit path now checks whether credentials landed. MAJOR — environment_scan_completed fired on every project_scan, including /discover and any model-initiated call, so a funnel query could exceed 100% conversion. Now guarded on isOnboardingSession. MINOR — command.execute.before created a tracking record for every slash command in every session, churning the capped map and evicting genuine onboarding sessions, after which their remaining activation events were silently dropped. noteCommandSubmission now only touches sessions already tracked, and /onboard-connect marks the session before flagging its own submission. MINOR — the cross-package event parity test that two comments claimed but which did not exist. Now a compile-time assertion pinning the packages/tui event union to the Telemetry variants; verified by renaming a property and watching the build fail. Required exporting the context subpath from packages/tui. MINOR — a raw HOME path could reach LLM-visible tool output on a sample-setup failure. Paths are masked in `output`; the full message stays in metadata, which the model never sees. MINOR — documented why sample_setup_completed uses a success boolean instead of a _failed sibling event. NIT — launchId() no longer writes process.env. The worker receives the id explicitly through WorkerOptions.env, so the write only leaked it into every subprocess the CLI spawns. Also removes the stray `// scratch` line at the end of cli/cmd/tui.ts. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018SLUQF3xgZHsGZHSjxe7vb --- .../altimate/plugin/onboarding-telemetry.ts | 6 ++- .../opencode/src/altimate/telemetry/index.ts | 39 ++++++++++++--- .../src/altimate/telemetry/onboarding.ts | 30 ++++++++++-- .../src/altimate/tools/project-scan.ts | 31 +++++++----- .../src/altimate/tools/sample-setup.ts | 18 ++++++- packages/opencode/src/cli/cmd/tui.ts | 9 +++- .../altimate/telemetry/onboarding.test.ts | 47 +++++++++++++++++-- packages/tui/package.json | 1 + .../tui/src/component/dialog-provider.tsx | 10 +++- 9 files changed, 160 insertions(+), 31 deletions(-) diff --git a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts index 16e15fc13..a71203742 100644 --- a/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts +++ b/packages/opencode/src/altimate/plugin/onboarding-telemetry.ts @@ -81,13 +81,17 @@ function isJobCompletion(tool: string, output: { metadata?: unknown }): boolean export async function OnboardingTelemetryPlugin(_input: PluginInput): Promise { return { "command.execute.before": async (input) => { + // Mark the session BEFORE flagging the submission: noteCommandSubmission only touches + // sessions already tracked (so ordinary slash commands cannot churn the capped map), and + // /onboard-connect is the command that creates the record in the first place. + if (input.command === ONBOARD_CONNECT) OnboardingTelemetry.markOnboardingSession(input.sessionID) + // Any slash command means the next user message was not typed by the user — needed so // `first_prompt_sent` measures a real first prompt rather than the scan gate's hidden // `/onboard-connect` submission. OnboardingTelemetry.noteCommandSubmission(input.sessionID) if (input.command !== ONBOARD_CONNECT) return - OnboardingTelemetry.markOnboardingSession(input.sessionID) // `skip` renders the menu immediately, with no scan to wait for, so the variant is known // now. The `scan` branch cannot be resolved here — the menu follows the scan, and the diff --git a/packages/opencode/src/altimate/telemetry/index.ts b/packages/opencode/src/altimate/telemetry/index.ts index 7d02fefd8..ec9fbb172 100644 --- a/packages/opencode/src/altimate/telemetry/index.ts +++ b/packages/opencode/src/altimate/telemetry/index.ts @@ -850,6 +850,11 @@ export namespace Telemetry { job: "sample_duck_db" | "breaks_downstream" | "sql_review" | "cost" | "something_else" } | { + /** Deliberately a `success` boolean rather than a `sample_setup_failed` sibling, unlike + * the gateway_auth_completed/_failed pair. The gateway has genuinely distinct failure + * modes worth their own enum (timeout / denied / error); this tool either materialised + * the sample or did not, and the useful breakdown is `reused`, which only exists on the + * success path. Splitting it would duplicate the counts an analyst has to add back up. */ type: "sample_setup_completed" timestamp: number session_id: string @@ -1366,12 +1371,15 @@ export namespace Telemetry { // so the worker computes a different start time than the main thread. const LAUNCH_ID_ENV = "ALTIMATE_LAUNCH_ID" + let cachedLaunchId: string | undefined + export function launchId(): string { - const existing = process.env[LAUNCH_ID_ENV] - if (existing) return existing - const created = randomUUID() - process.env[LAUNCH_ID_ENV] = created - return created + // The worker reads the value the TUI handed it through WorkerOptions.env; the main thread + // generates it. Cached in module scope rather than written back to process.env — the worker + // is given it explicitly, so writing it would only leak the id into every subprocess the CLI + // spawns, for no benefit. + if (!cachedLaunchId) cachedLaunchId = process.env[LAUNCH_ID_ENV] || randomUUID() + return cachedLaunchId } // altimate_change end @@ -1537,6 +1545,11 @@ export namespace Telemetry { } enabled = true log.info("telemetry initialized", { mode: "appinsights" }) + // altimate_change — clear any existing interval before installing a new one. doInit() can + // run more than once per process (init/shutdown cycles per session in prompt.ts), and + // without this each extra run strands the previous timer: shutdown() only ever clears the + // current handle, so orphans accumulate for the life of a `serve` process. + if (flushTimer) clearInterval(flushTimer) const timer = setInterval(flush, FLUSH_INTERVAL_MS) if (typeof timer === "object" && timer && "unref" in timer) (timer as any).unref() flushTimer = timer @@ -1671,6 +1684,7 @@ export namespace Telemetry { } async function doShutdown(timeoutMs?: number) { + const initPromiseAtShutdown = initPromise // Wait for init to complete so we know whether telemetry is enabled // and have a valid endpoint to flush to. init() is fire-and-forget // in CLI middleware, so it may still be in-flight when shutdown runs. @@ -1693,8 +1707,19 @@ export namespace Telemetry { sessionId = "" projectId = "" machineId = "" - initPromise = undefined - initDone = false + // altimate_change — only clear initPromise if it is still the one this shutdown began with. + // init() can set `initPromise = shutdownPromise.then(doInit)`; nulling that unconditionally + // discards a doInit() which has not run yet, so the next init() starts a second one. + // + // Not covered by a test: that assignment requires initPromise to be undefined while + // shutdownPromise is still live, and those two are cleared one statement apart — a window I + // could not reach deterministically. Kept because it is free and obviously correct; the + // clear-before-assign in doInit() is what actually prevents an orphaned interval, whatever + // path leads to a second doInit. + if (initPromise === initPromiseAtShutdown) { + initPromise = undefined + initDone = false + } } // altimate_change end } diff --git a/packages/opencode/src/altimate/telemetry/onboarding.ts b/packages/opencode/src/altimate/telemetry/onboarding.ts index fab554c9a..0c1253e1d 100644 --- a/packages/opencode/src/altimate/telemetry/onboarding.ts +++ b/packages/opencode/src/altimate/telemetry/onboarding.ts @@ -70,6 +70,9 @@ type DistributiveOmit = T extends unknown ? Omit type EmitInput = DistributiveOmit +/** Exported so the cross-package parity assertion in the tests can pin the TUI union to this. */ +export type OnboardingEmitInput = EmitInput + // Module-global, per thread. Resets on every process launch, which is correct: a fresh launch // is a fresh onboarding attempt. let furthestStage: OnboardingStage | undefined @@ -154,8 +157,18 @@ export function isCompleted() { * Call on the exit path, before the final flush. No-ops when the user never started (a * returning user with credentials), already completed, or when it has already fired. */ -export async function emitAbandonedIfIncomplete(): Promise { +export async function emitAbandonedIfIncomplete(opts?: { connected?: boolean }): Promise { if (completed || abandonedEmitted || !furthestStage) return + // A gateway sign-in that succeeded is not an abandonment, even if the TUI never observed it. + // `gateway_auth_completed` and `instance_connected` are emitted on the WORKER thread, and this + // state lives on the main thread — so a user who finishes in the browser and quits before the + // TUI notices the new provider would otherwise be reported as abandoning at `gateway_auth`, in + // the same launch that already reported a successful connection. Two contradictory terminal + // states for one run, and gateway auth is the slowest step so it is the likeliest to hit this. + // + // The caller passes whether credentials now exist, which is the main thread's own view of the + // same fact and needs no cross-thread channel. + if (opts?.connected) return abandonedEmitted = true await emit({ type: "onboarding_abandoned", last_stage: furthestStage }) } @@ -220,9 +233,20 @@ function claim(sessionID: string, key: "menuShown" | "jobSelected" | "jobComplet return true } -/** Record that the next user message in this session comes from a slash command. */ +/** + * Record that the next user message in this session comes from a slash command. + * + * Only touches sessions already tracked. `command.execute.before` fires for EVERY slash command + * in every session, so creating a record here made ordinary `/discover`, `/model` and so on churn + * the capped map and evict genuine onboarding sessions — after which `isOnboardingSession()` + * returns false and the rest of that user's activation events are silently dropped. + * + * An untracked session has no onboarding state to protect, so skipping it loses nothing: + * `first_prompt_sent` is gated on `isOnboardingSession` anyway. + */ export function noteCommandSubmission(sessionID: string) { - record(sessionID).commandSubmission = true + const entry = sessions.get(sessionID) + if (entry) entry.commandSubmission = true } /** True (once) if this session's pending user message was command-submitted. */ diff --git a/packages/opencode/src/altimate/tools/project-scan.ts b/packages/opencode/src/altimate/tools/project-scan.ts index a636a06c8..31a770ee1 100644 --- a/packages/opencode/src/altimate/tools/project-scan.ts +++ b/packages/opencode/src/altimate/tools/project-scan.ts @@ -945,18 +945,25 @@ export const ProjectScanTool = Tool.define("project_scan", { // otherwise be recorded as having none. // // degraded[] is a sorted list of short detection-failure keys — no paths, hosts, or messages. - void OnboardingTelemetry.emit({ - type: "environment_scan_completed", - has_dbt: dbtProject.found, - has_warehouse: totalConnections > 0, - is_repo: git.isRepo, - connections_found: totalConnections, - degraded: degradedList, - // Explicit session: emit() otherwise falls back to the process-global telemetry context, - // which is set per prompt loop. Two concurrent sessions in `serve` or in the TUI worker - // overwrite each other's context, so a scan would be attributed to whichever session most - // recently started a turn. - }, ctx.sessionID) + // Only for onboarding runs. project_scan is also reachable via /discover and any + // model-initiated call; without this guard an event in the onboarding taxonomy fires for all + // of them, and a funnel query on scan_gate_shown → environment_scan_completed can exceed 100%. + // + // Explicit session id as well: emit() otherwise falls back to the process-global telemetry + // context, which is set per prompt loop, so two concurrent sessions overwrite each other's. + if (OnboardingTelemetry.isOnboardingSession(ctx.sessionID)) { + void OnboardingTelemetry.emit( + { + type: "environment_scan_completed", + has_dbt: dbtProject.found, + has_warehouse: totalConnections > 0, + is_repo: git.isRepo, + connections_found: totalConnections, + degraded: degradedList, + }, + ctx.sessionID, + ) + } // altimate_change end // Build metadata diff --git a/packages/opencode/src/altimate/tools/sample-setup.ts b/packages/opencode/src/altimate/tools/sample-setup.ts index 161879288..67575b8ba 100644 --- a/packages/opencode/src/altimate/tools/sample-setup.ts +++ b/packages/opencode/src/altimate/tools/sample-setup.ts @@ -205,7 +205,11 @@ export const SampleSetupTool = Tool.define("sample_setup", { return { title: "Starter materialization failed", metadata: { success: false, error: message, targetPath: "", reused: false, suffix: 0, note: "" }, - output: `status: error\nreason: materialize_failed\n\n${message}`, + // Paths are masked in `output` but kept verbatim in `metadata.error`. `output` is what + // the model sees (session/message-v2.ts), so it lands in conversation context and is sent + // to the provider on every later turn — and rejectUnsafeHome messages embed HOME + // verbatim. metadata never reaches the model, so the full text stays available locally. + output: `status: error\nreason: materialize_failed\n\n${redactPaths(message)}`, } } }, @@ -229,6 +233,18 @@ export const SampleSetupTool = Tool.define("sample_setup", { // // Best-effort by construction: telemetry must never fail a sample setup, so any fs error // yields 0 rather than propagating. +/** + * Replace absolute filesystem paths with a placeholder. + * + * Deliberately local rather than reusing Telemetry.maskString: importing the telemetry module + * here drags in Config/Account and hangs this tool's tests. This only needs to handle paths. + */ +function redactPaths(message: string): string { + return message + .replace(/(?:[A-Za-z]:)?[\\/](?:[\w.\-~ ]+[\\/])+[\w.\-~ ]*/g, "") + .replace(/~[\\/][^\s'"]*/g, "") +} + function countFilesWithExtension(dir: string, extension: string): number { let total = 0 let entries: fs.Dirent[] diff --git a/packages/opencode/src/cli/cmd/tui.ts b/packages/opencode/src/cli/cmd/tui.ts index 403eaadc5..20ab747e0 100644 --- a/packages/opencode/src/cli/cmd/tui.ts +++ b/packages/opencode/src/cli/cmd/tui.ts @@ -19,6 +19,7 @@ import { win32InstallCtrlCGuard } from "@opencode-ai/tui/terminal-win32" // altimate_change start — onboarding telemetry: main-thread flush on the TUI exit path import { Telemetry } from "@/altimate/telemetry" import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" +import { AltimateApi } from "@/altimate/api/client" // altimate_change end declare global { @@ -268,7 +269,12 @@ export const TuiThreadCommand = cmd({ // flush() would otherwise block for REQUEST_TIMEOUT_MS (10s) on a blackholed network — a // visible hang between the user quitting and the shell prompt returning. try { - await OnboardingTelemetry.emitAbandonedIfIncomplete() + // Ask whether gateway credentials landed. The success events are emitted on the worker + // thread and this state is main-thread-owned, so without this a browser sign-in that + // completed just before the user quit is reported as an abandonment in the same launch + // that already reported instance_connected. + const connected = await AltimateApi.isConfigured().catch(() => false) + await OnboardingTelemetry.emitAbandonedIfIncomplete({ connected }) await Telemetry.shutdown({ timeoutMs: 2000 }) } catch { // Never let telemetry delay or break exit. @@ -278,4 +284,3 @@ export const TuiThreadCommand = cmd({ process.exit(0) }, }) -// scratch diff --git a/packages/opencode/test/altimate/telemetry/onboarding.test.ts b/packages/opencode/test/altimate/telemetry/onboarding.test.ts index 804304d9c..8a641c06d 100644 --- a/packages/opencode/test/altimate/telemetry/onboarding.test.ts +++ b/packages/opencode/test/altimate/telemetry/onboarding.test.ts @@ -11,6 +11,7 @@ import { describe, expect, test, beforeEach, afterEach, spyOn, mock } from "bun: import { Telemetry } from "@/altimate/telemetry" import * as Onboarding from "@/altimate/telemetry/onboarding" import { OnboardingTelemetryPlugin } from "@/altimate/plugin/onboarding-telemetry" +import type { OnboardingTelemetryEvent } from "@opencode-ai/tui/context/onboarding-telemetry" type Tracked = Telemetry.Event @@ -268,28 +269,45 @@ describe("activation events", () => { // Slash-command suppression, which first_prompt_sent depends on // --------------------------------------------------------------------------- describe("command submission tracking", () => { - test("a command-submitted message is flagged, once", () => { + test("a command-submitted message in an onboarding session is flagged, once", () => { + Onboarding.markOnboardingSession("ses_a") Onboarding.noteCommandSubmission("ses_a") expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(true) expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(false) }) test("a session that never ran a command is not flagged", () => { + Onboarding.markOnboardingSession("ses_never") expect(Onboarding.consumeCommandSubmission("ses_never")).toBe(false) }) test("the flag does not leak between sessions", () => { + Onboarding.markOnboardingSession("ses_a") + Onboarding.markOnboardingSession("ses_b") Onboarding.noteCommandSubmission("ses_a") expect(Onboarding.consumeCommandSubmission("ses_b")).toBe(false) expect(Onboarding.consumeCommandSubmission("ses_a")).toBe(true) }) - test("every slash command is flagged, not just /onboard-connect", async () => { + test("an untracked session is not created just by running a slash command", () => { + // The anti-churn property. command.execute.before fires for every slash command in every + // session; if that created a record, ordinary /discover and /model traffic in a long-lived + // `serve` process would evict genuine onboarding sessions from the capped map and silently + // drop those users' remaining activation events. + Onboarding.noteCommandSubmission("ses_unrelated") + expect(Onboarding.consumeCommandSubmission("ses_unrelated")).toBe(false) + expect(Onboarding.isOnboardingSession("ses_unrelated")).toBe(false) + }) + + test("/onboard-connect marks the session before flagging its own submission", async () => { + // Ordering matters: the command that creates the record must be flagged too, or the hidden + // scan-gate submission counts as the user's first typed prompt. const hooks = await OnboardingTelemetryPlugin({} as any) await hooks["command.execute.before"]!( - { command: "discover", sessionID: "ses_c", arguments: "" }, + { command: "onboard-connect", sessionID: "ses_c", arguments: "skip" }, { parts: [] } as any, ) + expect(Onboarding.isOnboardingSession("ses_c")).toBe(true) expect(Onboarding.consumeCommandSubmission("ses_c")).toBe(true) }) }) @@ -343,3 +361,26 @@ describe("launch correlation id", () => { } }) }) + +// --------------------------------------------------------------------------- +// Cross-package event parity +// --------------------------------------------------------------------------- +// +// packages/tui cannot import the Telemetry event union, so it declares its own mirror and the +// host remaps `name` → `type` through an unchecked cast in cli/cmd/tui.ts. Without this, a rename +// or an added property on either side compiles clean and ships malformed events. +// +// Compile-time only: tsgo runs over test files, so drift is a build failure. There is nothing to +// assert at runtime — a type is not a value. +type TuiEventAsEmitInput = E extends { name: infer N } ? Omit & { type: N } : never + +// Fails to compile if any TUI event lacks a matching Telemetry variant, or if their property +// names or enum values diverge. +const _tuiEventsMatchTelemetry: Onboarding.OnboardingEmitInput = null as unknown as TuiEventAsEmitInput +void _tuiEventsMatchTelemetry + +describe("cross-package event parity", () => { + test("is enforced by the type assertion above, not at runtime", () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/tui/package.json b/packages/tui/package.json index f2822e4a6..ad6cb0887 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -17,6 +17,7 @@ "./context/epilogue": "./src/context/epilogue.tsx", "./context/exit": "./src/context/exit.tsx", "./context/kv": "./src/context/kv.tsx", + "./context/onboarding-telemetry": "./src/context/onboarding-telemetry.tsx", "./context/project": "./src/context/project.tsx", "./context/runtime": "./src/context/runtime.tsx", "./context/sdk": "./src/context/sdk.tsx", diff --git a/packages/tui/src/component/dialog-provider.tsx b/packages/tui/src/component/dialog-provider.tsx index 9620dbf13..3b23489c4 100644 --- a/packages/tui/src/component/dialog-provider.tsx +++ b/packages/tui/src/component/dialog-provider.tsx @@ -398,8 +398,10 @@ function AutoMethod(props: AutoMethodProps) { await sdk.client.instance.dispose() await sync.bootstrap() if (disposed) return - // altimate_change start — mark setup complete (flips useReady → unlocks first-run chat/tips) - markSetupComplete() + // altimate_change start — setup is marked complete once a model is actually selected, not + // here. The branch below deliberately refuses to claim success when the gateway connects but + // offers nothing usable ("Connected, but no model is available yet"); marking completion up + // front contradicted that, and it drives onboarding_completed and the Part 2 scan gate. // The gateway sign-in already shows the auth URL + "Waiting for authorization…". // On success, confirm inline (green) and auto-close after a moment rather than // opening the model picker. Auto-select a model so the user can chat right away. @@ -419,6 +421,9 @@ function AutoMethod(props: AutoMethodProps) { return } local.model.set({ providerID: props.providerID, modelID: model }, { recent: true }) + // A model is chosen — this is the real end of setup (flips useReady → unlocks first-run + // chat/tips, and opens the Part 2 scan gate). + markSetupComplete() setConnected(true) closeTimer = setTimeout(() => { if (!disposed) dialog.clear() @@ -427,6 +432,7 @@ function AutoMethod(props: AutoMethodProps) { } // altimate_change end toast.show({ message: `Connected to ${props.title}`, variant: "success" }) + // No markSetupComplete() here: this opens the model picker, which marks it on selection. dialog.replace(() => ) })