From 141a3861f8469559777bcd6910b8b39337ef9df8 Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:46:38 -0400 Subject: [PATCH 1/7] feat(observability): connect API and Temporal spans through shared OTLP export --- .github/workflows/ci.yml | 18 + apps/api/package.json | 3 +- apps/api/src/dot-goals.ts | 15 +- .../src/memory-consolidation-coordinator.ts | 3 + apps/api/src/memory-mutation-coordinator.ts | 3 + apps/api/src/otlp-span-exporter.ts | 162 +-------- apps/api/src/room-agent.ts | 1 + apps/api/src/trace-context.ts | 4 +- apps/api/tests/dot-goals.test.ts | 2 +- .../memory-consolidation-coordinator.test.ts | 1 + .../tests/memory-mutation-coordinator.test.ts | 1 + apps/api/tests/trace-correlation.test.ts | 2 + apps/worker/package.json | 11 +- apps/worker/src/activity-observability.ts | 16 +- apps/worker/src/memory-maintenance-worker.ts | 11 +- apps/worker/src/memory-mutation-worker.ts | 28 +- apps/worker/src/temporal-trace-context.ts | 110 ++++++ apps/worker/src/worker-telemetry.ts | 95 +++++ apps/worker/src/worker.ts | 13 +- apps/worker/src/workflow-telemetry.ts | 80 +++++ .../tests/support/telemetry-workflows.ts | 6 + apps/worker/tests/worker-telemetry.test.ts | 326 ++++++++++++++++++ apps/worker/tsconfig.telemetry-tests.json | 5 + docs/configuration-reference.md | 24 ++ docs/quality/observability.md | 123 ++++--- package-lock.json | 38 +- package.json | 6 +- packages/telemetry/package.json | 22 ++ packages/telemetry/src/index.ts | 164 +++++++++ packages/telemetry/tsconfig.json | 12 + 30 files changed, 1071 insertions(+), 234 deletions(-) create mode 100644 apps/worker/src/temporal-trace-context.ts create mode 100644 apps/worker/src/worker-telemetry.ts create mode 100644 apps/worker/src/workflow-telemetry.ts create mode 100644 apps/worker/tests/support/telemetry-workflows.ts create mode 100644 apps/worker/tests/worker-telemetry.test.ts create mode 100644 apps/worker/tsconfig.telemetry-tests.json create mode 100644 packages/telemetry/package.json create mode 100644 packages/telemetry/src/index.ts create mode 100644 packages/telemetry/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82b1903c0..eb41c52df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,24 @@ concurrency: cancel-in-progress: true jobs: + worker-tracing: + name: Connected Temporal tracing + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version-file: .nvmrc + cache: npm + - run: ELECTRON_SKIP_BINARY_DOWNLOAD=1 npm ci + - run: npm run check + - run: npm run telemetry:check -w @space/worker + - run: npm test -w @space/worker -- tests/worker-telemetry.test.ts tests/activity-observability.test.ts + - run: npm test -w @space/api -- otlp trace-correlation memory-consolidation-coordinator memory-mutation-coordinator dot-goals + public-quality: name: Sanitized public quality gates runs-on: ubuntu-latest diff --git a/apps/api/package.json b/apps/api/package.json index 5f6485b27..5a46402d7 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -45,7 +45,8 @@ "node-pty": "^1.1.0", "pdfjs-dist": "6.3.289", "sanitize-html": "^2.17.7", - "zod": "^4.2.1" + "zod": "^4.2.1", + "@space/telemetry": "0.2.5" }, "devDependencies": { "@types/pg": "^8.23.1", diff --git a/apps/api/src/dot-goals.ts b/apps/api/src/dot-goals.ts index 30dd69dee..064f00b4e 100644 --- a/apps/api/src/dot-goals.ts +++ b/apps/api/src/dot-goals.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; import { z } from "zod"; import { Client, Connection, WorkflowExecutionAlreadyStartedError, WorkflowNotFoundError } from "@temporalio/client"; import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from "fastify"; @@ -26,10 +27,10 @@ import type { DotGoalsRepository, DotsRepository } from "@space/db"; * blocking question, cancellation, three consecutive failures or the budget. */ export interface DotGoalWorkflowCoordinator { - start(goalId: string): Promise; + start(goalId: string, traceId?: string): Promise; wake(goalId: string): Promise; /** Starts the Goal's workflow when none is running; true when it had to start one. */ - ensure?(goalId: string): Promise; + ensure?(goalId: string, traceId?: string): Promise; } export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordinator { constructor(private readonly options: { address: string; namespace: string; taskQueue: string }) {} @@ -38,11 +39,11 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi try { return await action(new Client({ connection, namespace: this.options.namespace })); } finally { await connection.close(); } } - start(goalId: string) { + start(goalId: string, traceId?: string) { return this.client(async (client) => { try { await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue, - args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE" }); + args: [{ goalId }], memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }), workflowIdReusePolicy: "REJECT_DUPLICATE" }); } catch (error) { if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; } }); } @@ -52,11 +53,11 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi catch (error) { if (!(error instanceof WorkflowNotFoundError)) throw error; } }); } - ensure(goalId: string) { + ensure(goalId: string, traceId?: string) { return this.client(async (client) => { try { await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue, - args: [{ goalId }], workflowIdReusePolicy: "ALLOW_DUPLICATE" }); + args: [{ goalId }], memo: workflowCorrelationMemo({ traceId, traceparent: traceparentForTraceId(traceId) }), workflowIdReusePolicy: "ALLOW_DUPLICATE" }); return true; } catch (error) { if (error instanceof WorkflowExecutionAlreadyStartedError) return false; @@ -125,7 +126,7 @@ export function createDotGoalService(deps: DotGoalServiceDeps) { if ((error as { code?: string }).code === "DOT_GOAL_OPEN") throw new DotGoalError(409, "DOT_GOAL_OPEN", "This conversation already has an open Goal."); throw error; } - try { await deps.workflow.start(goal.goalId); } + try { await deps.workflow.start(goal.goalId, request?.id); } catch { goal = await finish(goal, "FAILED", "The Goal runtime could not start. Try again."); throw new DotGoalError(503, "DOT_GOAL_RUNTIME", "The Goal runtime could not start. Try again."); diff --git a/apps/api/src/memory-consolidation-coordinator.ts b/apps/api/src/memory-consolidation-coordinator.ts index df237c648..9be32d10f 100644 --- a/apps/api/src/memory-consolidation-coordinator.ts +++ b/apps/api/src/memory-consolidation-coordinator.ts @@ -6,6 +6,7 @@ import { type MemoryConsolidationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; export { MEMORY_CONSOLIDATION_TASK_QUEUE, MEMORY_CONSOLIDATION_WORKFLOW_TYPE }; @@ -16,6 +17,7 @@ export interface MemoryConsolidationStartResult { } interface WorkflowStartOptions { + memo: Record; args: [MemoryConsolidationWorkflowInput]; taskQueue: typeof MEMORY_CONSOLIDATION_TASK_QUEUE; workflowId: string; @@ -46,6 +48,7 @@ export async function scheduleMemoryConsolidation( try { const handle = await startWorkflow(MEMORY_CONSOLIDATION_WORKFLOW_TYPE, { args: [input], + memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) }), taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/src/memory-mutation-coordinator.ts b/apps/api/src/memory-mutation-coordinator.ts index f7d3d48bb..cfecc8073 100644 --- a/apps/api/src/memory-mutation-coordinator.ts +++ b/apps/api/src/memory-mutation-coordinator.ts @@ -7,6 +7,7 @@ import { type MemoryMutationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; export { MEMORY_MUTATION_TASK_QUEUE, MEMORY_MUTATION_WORKFLOW_TYPE, buildMemoryMutationWorkflowId }; @@ -17,6 +18,7 @@ export interface MemoryMutationStartResult { } interface WorkflowStartOptions { + memo: Record; args: [MemoryMutationWorkflowInput]; taskQueue: typeof MEMORY_MUTATION_TASK_QUEUE; workflowId: string; @@ -48,6 +50,7 @@ export async function scheduleMemoryMutation( try { const handle = await startWorkflow(MEMORY_MUTATION_WORKFLOW_TYPE, { args: [input], + memo: workflowCorrelationMemo({ traceId: input.traceId, traceparent: traceparentForTraceId(input.traceId) }), taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/src/otlp-span-exporter.ts b/apps/api/src/otlp-span-exporter.ts index 53a5f16b0..ba5eb7593 100644 --- a/apps/api/src/otlp-span-exporter.ts +++ b/apps/api/src/otlp-span-exporter.ts @@ -1,160 +1,2 @@ -/** - * Optional OTLP/HTTP (JSON) export of API server spans. Off unless - * SPACE_OTEL_EXPORTER_OTLP_ENDPOINT names a collector base URL (spans go to - * `/v1/traces`). No dependency, no collector required: Space logs stay - * joinable by W3C trace id without it. - * - * Spans carry only the method, route template, status code, service name and - * Space request id; never URLs with parameters, headers, bodies or user data. - * Export is best effort: a bounded queue drops (and counts) spans instead of - * growing, each POST has a timeout, and a failing collector never affects a - * request. - */ - -export interface ServerSpan { - traceId: string; - spanId: string; - parentSpanId: string | null; - name: string; - startTimeUnixNano: bigint; - endTimeUnixNano: bigint; - attributes: Readonly>; - error: boolean; -} - -export interface SpanExporterStats { - exported: number; - dropped: number; - failed: number; -} - -export interface SpanExporter { - record(span: ServerSpan): void; - flush(): Promise; - close(): Promise; - stats(): SpanExporterStats; -} - -type FetchLike = (url: string, init: { method: string; headers: Record; body: string; signal: AbortSignal }) => Promise<{ ok: boolean }>; - -export interface OtlpSpanExporterOptions { - endpoint: string; - serviceName: string; - fetch?: FetchLike; - batchSize?: number; - maxQueue?: number; - flushIntervalMs?: number; - timeoutMs?: number; -} - -const hexId = (length: number) => new RegExp(`^[0-9a-f]{${length}}$`); -const traceIdPattern = hexId(32); -const spanIdPattern = hexId(16); - -/** The collector base URL, or null when export is off or the value is unsafe. */ -export function otlpTracesUrl(value: string | undefined): string | null { - if (!value?.trim()) return null; - let url: URL; - try { - url = new URL(value.trim()); - } catch { - return null; - } - if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password || url.search || url.hash) return null; - return `${url.origin}${url.pathname.replace(/\/+$/, "")}/v1/traces`; -} - -function attributeValue(value: string | number) { - return typeof value === "number" && Number.isInteger(value) ? { intValue: String(value) } : { stringValue: String(value) }; -} - -export function otlpTraceRequest(spans: readonly ServerSpan[], serviceName: string) { - return { - resourceSpans: [{ - resource: { attributes: [{ key: "service.name", value: { stringValue: serviceName } }] }, - scopeSpans: [{ - scope: { name: "space-api" }, - spans: spans.map(span => ({ - traceId: span.traceId, - spanId: span.spanId, - ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}), - name: span.name, - // SPAN_KIND_SERVER - kind: 2, - startTimeUnixNano: span.startTimeUnixNano.toString(), - endTimeUnixNano: span.endTimeUnixNano.toString(), - attributes: Object.entries(span.attributes).map(([key, value]) => ({ key, value: attributeValue(value) })), - // STATUS_CODE_ERROR or STATUS_CODE_UNSET - status: { code: span.error ? 2 : 0 } - })) - }] - }] - }; -} - -export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanExporter { - const url = otlpTracesUrl(options.endpoint); - if (!url) throw new Error("Invalid OTLP endpoint."); - const send: FetchLike = options.fetch ?? ((target, init) => fetch(target, init)); - const batchSize = options.batchSize ?? 256; - const maxQueue = options.maxQueue ?? 4096; - const timeoutMs = options.timeoutMs ?? 3000; - const queue: ServerSpan[] = []; - const stats: SpanExporterStats = { exported: 0, dropped: 0, failed: 0 }; - let inFlight: Promise | null = null; - - const flushOnce = async () => { - const batch = queue.splice(0, batchSize); - if (!batch.length) return; - try { - const response = await send(url, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(otlpTraceRequest(batch, options.serviceName)), - signal: AbortSignal.timeout(timeoutMs) - }); - if (response.ok) stats.exported += batch.length; - else stats.failed += batch.length; - } catch { - stats.failed += batch.length; - } - }; - const flush = (): Promise => { - if (!inFlight) { - inFlight = (async () => { - try { - while (queue.length) await flushOnce(); - } finally { - inFlight = null; - } - })(); - } - return inFlight; - }; - const timer = setInterval(() => void flush(), options.flushIntervalMs ?? 5000); - timer.unref?.(); - - return { - record(span) { - if (!traceIdPattern.test(span.traceId) || !spanIdPattern.test(span.spanId)) return; - if (queue.length >= maxQueue) { - stats.dropped += 1; - return; - } - queue.push(span); - if (queue.length >= batchSize) void flush(); - }, - flush, - async close() { - clearInterval(timer); - await flush(); - }, - stats: () => ({ ...stats }) - }; -} - -/** The exporter configured by SPACE_OTEL_EXPORTER_OTLP_ENDPOINT, or null when export is off. */ -export function otlpSpanExporterFromEnv(env: NodeJS.ProcessEnv, serviceName: string): SpanExporter | null { - const endpoint = env.SPACE_OTEL_EXPORTER_OTLP_ENDPOINT; - return otlpTracesUrl(endpoint) ? createOtlpSpanExporter({ endpoint: endpoint!, serviceName }) : null; -} +// Preserve the API import surface; API and workers share one bounded exporter. +export * from "@space/telemetry"; diff --git a/apps/api/src/room-agent.ts b/apps/api/src/room-agent.ts index f784f44c5..bf22d670b 100644 --- a/apps/api/src/room-agent.ts +++ b/apps/api/src/room-agent.ts @@ -225,6 +225,7 @@ export class TemporalRoomAgentWorkflowCoordinator implements RoomAgentWorkflowCo workflowId, taskQueue: this.options.taskQueue, args: [{ roomId: item.turn.roomId, pending: [], processedCount: 0 }], + memo: workflowCorrelationMemo({ traceId: item.turn.traceId, traceparent: item.turn.traceparent, missionId: item.missionId }), signal: ROOM_AGENT_ENQUEUE_SIGNAL, signalArgs: [item] }); diff --git a/apps/api/src/trace-context.ts b/apps/api/src/trace-context.ts index af9e55efd..ff7335df0 100644 --- a/apps/api/src/trace-context.ts +++ b/apps/api/src/trace-context.ts @@ -96,7 +96,9 @@ export const requestTraceContexts = new TraceContextRegistry(); /** traceparent for work started on behalf of this Space request id, if known. */ export function traceparentForTraceId(traceId: string | null | undefined, registry = requestTraceContexts): string | null { - return childTraceparent(registry.lookup(traceId)?.traceparent ?? null); + // A propagated parent must name the exported HTTP span. Creating another + // unrecorded span here would leave every workflow attached to a missing parent. + return registry.lookup(traceId)?.traceparent ?? null; } /** Adds the request's traceparent to a turn input unless it already has a valid one. */ diff --git a/apps/api/tests/dot-goals.test.ts b/apps/api/tests/dot-goals.test.ts index 0c4a38e11..4385e2c27 100644 --- a/apps/api/tests/dot-goals.test.ts +++ b/apps/api/tests/dot-goals.test.ts @@ -73,7 +73,7 @@ describe("Dot Goals", () => { const { service, deps } = fixture(); const goal = await service.create("user:1", dotId, input, null); expect(goal).toMatchObject({ status: "ACTIVE", budget: { maxSteps: 2, maxMinutes: 180 }, milestones: [{ id: "m1", title: "Draft" }, { id: "m2" }] }); - expect(deps.workflow!.start).toHaveBeenCalledWith(goal.goalId); + expect(deps.workflow!.start).toHaveBeenCalledWith(goal.goalId, undefined); await expect(service.create("user:1", dotId, input, null)).rejects.toMatchObject({ status: 409, code: "DOT_GOAL_OPEN" }); await expect(service.create("user:2", dotId, input, null)).rejects.toBeInstanceOf(DotGoalError); }); diff --git a/apps/api/tests/memory-consolidation-coordinator.test.ts b/apps/api/tests/memory-consolidation-coordinator.test.ts index 3ff07aecb..d18216c64 100644 --- a/apps/api/tests/memory-consolidation-coordinator.test.ts +++ b/apps/api/tests/memory-consolidation-coordinator.test.ts @@ -21,6 +21,7 @@ describe("Memory consolidation coordinator", () => { workflowType: MEMORY_CONSOLIDATION_WORKFLOW_TYPE, options: { args: [input], + memo: { spaceTraceId: input.traceId }, taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/tests/memory-mutation-coordinator.test.ts b/apps/api/tests/memory-mutation-coordinator.test.ts index e07db2b56..90e203e1c 100644 --- a/apps/api/tests/memory-mutation-coordinator.test.ts +++ b/apps/api/tests/memory-mutation-coordinator.test.ts @@ -27,6 +27,7 @@ describe("Memory mutation coordinator", () => { workflowType: MEMORY_MUTATION_WORKFLOW_TYPE, options: { args: [input], + memo: { spaceTraceId: input.traceId }, taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowId: result.workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/tests/trace-correlation.test.ts b/apps/api/tests/trace-correlation.test.ts index 7d89bfbdc..bde111f5d 100644 --- a/apps/api/tests/trace-correlation.test.ts +++ b/apps/api/tests/trace-correlation.test.ts @@ -50,6 +50,7 @@ describe("request trace context", () => { registry.remember("req:dddddddddddd", createRequestTraceContext(inbound)); const filled = withTurnTraceparent({ traceId: "req:dddddddddddd" }, registry); expect(parseTraceparent(filled.traceparent)?.traceId).toBe(inboundTraceId); + expect(filled.traceparent).toBe(registry.lookup("req:dddddddddddd")?.traceparent); const kept = withTurnTraceparent({ traceId: "req:dddddddddddd", traceparent: inbound }, registry); expect(kept.traceparent).toBe(inbound); expect(withTurnTraceparent({ traceId: "req:unknown00000" }, registry)).toEqual({ traceId: "req:unknown00000" }); @@ -150,6 +151,7 @@ describe("workflow starters", () => { expect(options.workflowId).toContain(turn.traceId); expect(options.args[0]?.traceId).toBe(turn.traceId); expect(parseTraceparent(options.args[0]?.traceparent)?.traceId).toBe(inboundTraceId); + expect(options.args[0]?.traceparent).toBe(requestTraceContexts.lookup(turn.traceId)?.traceparent); expect(options.memo).toMatchObject({ spaceTraceId: turn.traceId, w3cTraceId: inboundTraceId }); expect(JSON.stringify(options.memo)).not.toContain("PRIVATE PROMPT"); expect(close).toHaveBeenCalledOnce(); diff --git a/apps/worker/package.json b/apps/worker/package.json index abd012313..1fcce9389 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -10,7 +10,8 @@ "start": "node dist/worker.js", "build": "tsc -p tsconfig.json", "check": "tsc -p tsconfig.json --noEmit", - "test": "vitest run" + "test": "vitest run", + "telemetry:check": "tsc -p tsconfig.telemetry-tests.json" }, "dependencies": { "@space/codex-app-server": "0.2.5", @@ -23,11 +24,15 @@ "@temporalio/client": "1.24.0", "@temporalio/worker": "1.24.0", "@temporalio/workflow": "1.24.0", - "undici": "8.10.2" + "undici": "8.10.2", + "@space/telemetry": "0.2.5", + "@temporalio/common": "1.24.0" }, "devDependencies": { "typescript": "^5.9.3", - "vitest": "^4.1.11" + "vitest": "^4.1.11", + "@temporalio/testing": "1.24.0", + "@temporalio/proto": "1.24.0" }, "license": "BUSL-1.1" } diff --git a/apps/worker/src/activity-observability.ts b/apps/worker/src/activity-observability.ts index 519f0fd71..cc07ba298 100644 --- a/apps/worker/src/activity-observability.ts +++ b/apps/worker/src/activity-observability.ts @@ -29,6 +29,7 @@ export type ActivityOutcome = "COMPLETED" | "FAILED_RESULT" | "FAILED" | "CANCEL interface ActivityObservation { reasonCode: string | null; + traceparent?: string; } const observations = new AsyncLocalStorage(); @@ -54,8 +55,10 @@ export function failureReasonScope(execute: () => Promise): { promise: Pro return { promise: observations.run(observation, execute), reasonCode: () => observation.reasonCode }; } -/** traceparent for an outbound call: same W3C trace, new span id; null if none. */ +/** Parent an outbound HTTP call to the observed activity span when available. */ export function childTraceparent(traceparent: string | null | undefined): string | null { + const current = observations.getStore()?.traceparent; + if (current && parseTraceparent(current)) return current; const parsed = parseTraceparent(traceparent); if (!parsed) return null; let spanId = randomBytes(8).toString("hex"); @@ -134,6 +137,11 @@ export interface ObserveActivityDependencies { metrics?: WorkerMetrics; now?: () => number; cancelled?: () => boolean; + traceparent?: string; + onFinish?: (observation: { + outcome: ActivityOutcome; reasonCode: string | null; category: FailureCategory | null; + startTimeMs: number; endTimeMs: number; durationMs: number; + }) => void; } export async function observeActivityExecution( @@ -163,7 +171,7 @@ export async function observeActivityExecution( if (queueMs !== null) metrics.activityScheduleToStart.observe(labels, queueMs / 1000); metrics.activitiesInFlight.inc(labels); workerLog("info", "activity.started", { ...base, queueMs }); - const observation: ActivityObservation = { reasonCode: null }; + const observation: ActivityObservation = { reasonCode: null, traceparent: dependencies.traceparent }; const finish = (outcome: ActivityOutcome, reasonCode: string | null, errorName: string | null) => { const durationMs = Math.max(0, now() - startedAt); @@ -185,6 +193,10 @@ export async function observeActivityExecution( queueMs, rssBytes: process.memoryUsage().rss }); + // Telemetry is best effort; an observer must never change activity results + // or create a retry (and must not return a promise to await). + try { dependencies.onFinish?.({ outcome, reasonCode, category, startTimeMs: startedAt, endTimeMs: startedAt + durationMs, durationMs }); } + catch { /* Keep activity execution independent from instrumentation. */ } }; try { diff --git a/apps/worker/src/memory-maintenance-worker.ts b/apps/worker/src/memory-maintenance-worker.ts index 78b413aae..12947913d 100644 --- a/apps/worker/src/memory-maintenance-worker.ts +++ b/apps/worker/src/memory-maintenance-worker.ts @@ -1,4 +1,5 @@ import { fileURLToPath } from "node:url"; +import { createWorkerTelemetry } from "./worker-telemetry.js"; import { NativeConnection, Worker } from "@temporalio/worker"; import * as activities from "./memory-maintenance-activities.js"; import { @@ -17,15 +18,18 @@ if (!memoryMaintenanceEnabled(process.env)) { const namespace = process.env.SPACE_TEMPORAL_NAMESPACE ?? DEFAULT_TEMPORAL_NAMESPACE; const workflowsPath = fileURLToPath(new URL("./memory-maintenance-workflows.js", import.meta.url)); + const telemetry = createWorkerTelemetry({ serviceName: service }); + let connection: NativeConnection | undefined; try { - const connection = await NativeConnection.connect({ address }); + connection = await NativeConnection.connect({ address }); const worker = await Worker.create({ connection, namespace, taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE, workflowsPath, activities, - ...memoryMaintenanceWorkerOptions + ...memoryMaintenanceWorkerOptions, + ...telemetry.workerOptions }); console.log(JSON.stringify({ service, @@ -43,5 +47,8 @@ if (!memoryMaintenanceEnabled(process.env)) { error: error instanceof Error ? error.message : "Unknown maintenance worker failure." })); process.exitCode = 1; + } finally { + await telemetry.close(); + await connection?.close(); } } diff --git a/apps/worker/src/memory-mutation-worker.ts b/apps/worker/src/memory-mutation-worker.ts index dea89d513..f7307e71e 100644 --- a/apps/worker/src/memory-mutation-worker.ts +++ b/apps/worker/src/memory-mutation-worker.ts @@ -1,4 +1,5 @@ import { fileURLToPath } from "node:url"; +import { createWorkerTelemetry } from "./worker-telemetry.js"; import { NativeConnection, Worker } from "@temporalio/worker"; import { executePersistedMemoryChangeSet } from "./memory-mutation-activity.js"; import { MEMORY_MUTATION_TASK_QUEUE, memoryMutationsEnabled, memoryMutationWorkerOptions } from "./memory-mutation-config.js"; @@ -12,14 +13,21 @@ if (!memoryMutationsEnabled(process.env)) { const address = process.env.SPACE_TEMPORAL_ADDRESS ?? DEFAULT_TEMPORAL_ADDRESS; const namespace = process.env.SPACE_TEMPORAL_NAMESPACE ?? DEFAULT_TEMPORAL_NAMESPACE; const connection = await NativeConnection.connect({ address }); - const worker = await Worker.create({ - connection, - namespace, - taskQueue: MEMORY_MUTATION_TASK_QUEUE, - workflowsPath: fileURLToPath(new URL("./memory-mutation-workflows.js", import.meta.url)), - activities: { executePersistedMemoryChangeSet }, - ...memoryMutationWorkerOptions - }); - console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE })); - await worker.run(); + const telemetry = createWorkerTelemetry({ serviceName: service }); + try { + const worker = await Worker.create({ + connection, + namespace, + taskQueue: MEMORY_MUTATION_TASK_QUEUE, + workflowsPath: fileURLToPath(new URL("./memory-mutation-workflows.js", import.meta.url)), + activities: { executePersistedMemoryChangeSet }, + ...memoryMutationWorkerOptions, + ...telemetry.workerOptions + }); + console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE })); + await worker.run(); + } finally { + await telemetry.close(); + await connection.close(); + } } diff --git a/apps/worker/src/temporal-trace-context.ts b/apps/worker/src/temporal-trace-context.ts new file mode 100644 index 000000000..d55a1b3f2 --- /dev/null +++ b/apps/worker/src/temporal-trace-context.ts @@ -0,0 +1,110 @@ +import { defaultPayloadConverter, type Headers } from "@temporalio/common"; +import { parseTraceparent } from "@space/contracts"; + +// This module is safe inside the workflow sandbox: no I/O, random number +// generation, environment reads, or mutable process-global state. +export const TRACE_HEADER = "space-trace-context-v1"; +export interface SpanLink { traceId: string; spanId: string } +export interface TemporalTraceContext { + traceId: string; + spanId: string; + parentSpanId: string | null; + links: SpanLink[]; +} +export interface PropagatedTrace { + traceparent: string; + links?: SpanLink[]; +} + +const tracePattern = /^(?!0{32}$)[0-9a-f]{32}$/; +const spanPattern = /^(?!0{16}$)[0-9a-f]{16}$/; +const identifierPattern = /^[A-Za-z0-9][A-Za-z0-9:._-]{0,199}$/; + +export function traceIdentifier(value: unknown): string | undefined { + return typeof value === "string" && identifierPattern.test(value) ? value : undefined; +} + +export function validLink(value: unknown): value is SpanLink { + if (!value || typeof value !== "object") return false; + const link = value as SpanLink; + return tracePattern.test(link.traceId) && spanPattern.test(link.spanId); +} + +export function readTraceHeader(headers: Headers): PropagatedTrace | null { + try { + const payload = headers[TRACE_HEADER]; + if (!payload) return null; + const value = defaultPayloadConverter.fromPayload(payload) as PropagatedTrace; + if (!value || !parseTraceparent(value.traceparent)) return null; + return { traceparent: value.traceparent, links: Array.isArray(value.links) ? value.links.filter(validLink).slice(0, 2) : [] }; + } catch { return null; } +} + +export function traceHeaders(headers: Headers, context: PropagatedTrace): Headers { + return { ...headers, [TRACE_HEADER]: defaultPayloadConverter.toPayload(context) }; +} + +export function inputTraceparent(args: readonly unknown[]): string | undefined { + const read = (value: unknown, depth: number): string | undefined => { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const object = value as Record; + if (typeof object.traceparent === "string" && parseTraceparent(object.traceparent)) return object.traceparent; + if (depth > 0) for (const key of ["turn", "item", "bridge", "input"]) { + const found = read(object[key], depth - 1); + if (found) return found; + } + return undefined; + }; + for (const arg of args) { const found = read(arg, 2); if (found) return found; } + return undefined; +} + +/** Stable per-run ID; does not consume the workflow's deterministic PRNG. */ +export function workflowSpanId(runId: string): string { + const hex = runId.replace(/-/g, "").toLowerCase(); + if (!tracePattern.test(hex)) throw new Error("Temporal supplied an invalid run id"); + const span = hex.slice(16); + return spanPattern.test(span) ? span : hex.slice(0, 16); +} + +export function workflowTraceContext(runId: string, propagated?: PropagatedTrace | null): TemporalTraceContext { + const parent = parseTraceparent(propagated?.traceparent); + return { + traceId: parent?.traceId ?? runId.replace(/-/g, "").toLowerCase(), + spanId: workflowSpanId(runId), + parentSpanId: parent?.parentId ?? null, + links: propagated?.links?.filter(validLink).slice(0, 2) ?? [] + }; +} + +export const temporalAttributeKeys = [ + "temporal.workflow_id", "temporal.run_id", "temporal.workflow_type", "temporal.activity_type", + "temporal.activity_id", "temporal.attempt", "space.trace_id", "space.room_id", "space.pane_id", + "space.mission_id", "space.goal_id", "space.outcome", "space.reason_code", "space.category", "duration_ms" +] as const; + +export function temporalAttributes(input: Readonly>): Record { + const attributes: Record = {}; + for (const key of temporalAttributeKeys) { + const value = input[key]; + if ((key === "duration_ms" || key === "temporal.attempt") && typeof value === "number" && Number.isFinite(value) && value >= 0) attributes[key] = value; + else { + const identifier = traceIdentifier(value); + if (identifier) attributes[key] = identifier; + } + } + return attributes; +} + +export function workflowInputAttributes(args: readonly unknown[]): Record { + const attrs: Record = {}; + for (const arg of args) { + if (!arg || typeof arg !== "object" || Array.isArray(arg)) continue; + const object = arg as Record; + for (const [input, output] of [["traceId", "space.trace_id"], ["roomId", "space.room_id"], ["paneId", "space.pane_id"], ["missionId", "space.mission_id"], ["goalId", "space.goal_id"]]) { + const value = traceIdentifier(object[input!]); + if (value) attrs[output!] = value; + } + } + return attrs; +} diff --git a/apps/worker/src/worker-telemetry.ts b/apps/worker/src/worker-telemetry.ts new file mode 100644 index 000000000..595a3e600 --- /dev/null +++ b/apps/worker/src/worker-telemetry.ts @@ -0,0 +1,95 @@ +import { randomBytes } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import type { WorkerOptions } from "@temporalio/worker"; +import { otlpSpanExporterFromEnv, type SpanExporter } from "@space/telemetry"; +import { formatTraceparent, parseTraceparent } from "@space/contracts"; +import { activityCorrelation, createActivityObservabilityInterceptors, observeActivityExecution } from "./activity-observability.js"; +import { + readTraceHeader, temporalAttributes, traceIdentifier, workflowTraceContext +} from "./temporal-trace-context.js"; +import type { TelemetrySinks } from "./workflow-telemetry.js"; +import type { InjectedSinks } from "@temporalio/worker"; + +const nanos = (milliseconds: number) => BigInt(Math.trunc(milliseconds * 1e3)) * 1_000n; +function spanId(): string { + for (;;) { const id = randomBytes(8).toString("hex"); if (!/^0+$/.test(id)) return id; } +} + +/** Host-only exporter and interceptors, shared by all three worker entrypoints. */ +export function createWorkerTelemetry(options: { + serviceName: "space-worker" | "space-memory-maintenance-worker" | "space-memory-mutation-worker"; + env?: NodeJS.ProcessEnv; + exporter?: SpanExporter | null; +}) { + const exporter = options.exporter === undefined + ? otlpSpanExporterFromEnv(options.env ?? process.env, options.serviceName) + : options.exporter; + const sinks: InjectedSinks = { + spaceTelemetry: { record: { + callDuringReplay: false, + fn(info, record) { + if (!exporter) return; + const { context } = record; + const name = traceIdentifier(info.workflowType) ?? "unknown"; + exporter.record({ + traceId: context.traceId, spanId: context.spanId, parentSpanId: context.parentSpanId, + name: `workflow.${name}`, kind: 1, links: context.links, + startTimeUnixNano: nanos(record.startTimeMs), endTimeUnixNano: nanos(record.endTimeMs), + error: record.outcome === "FAILED", + attributes: temporalAttributes({ + ...record.attributes, "temporal.workflow_id": info.workflowId, "temporal.run_id": info.runId, + "temporal.workflow_type": name, "space.outcome": record.outcome, + duration_ms: Math.max(0, record.endTimeMs - record.startTimeMs) + }) + }); + } + } } + }; + const workerOptions: Pick = exporter ? { + sinks, + interceptors: { + workflowModules: [fileURLToPath(new URL("./workflow-telemetry.js", import.meta.url))], + activity: [(context) => ({ inbound: { execute(input, next) { + const info = context.info; + const run = info.workflowExecution; + const parent = readTraceHeader(input.headers); + const propagated = parseTraceparent(parent?.traceparent); + const fallback = run ? workflowTraceContext(run.runId) : null; + const traceId = propagated?.traceId ?? fallback?.traceId; + const parentSpanId = propagated?.parentId ?? fallback?.spanId; + const correlation = activityCorrelation(input.args); + const id = spanId(); + return observeActivityExecution(info, input.args, () => next(input), { + cancelled: () => context.cancellationSignal.aborted, + traceparent: traceId ? formatTraceparent(traceId, id, "01") : undefined, + onFinish(observation) { + if (!traceId || !parentSpanId) return; + exporter.record({ + traceId, spanId: id, parentSpanId, + name: `activity.${traceIdentifier(info.activityType) ?? "unknown"}`, kind: 1, + startTimeUnixNano: nanos(observation.startTimeMs), endTimeUnixNano: nanos(observation.endTimeMs), + error: observation.outcome === "FAILED" || observation.outcome === "FAILED_RESULT", + attributes: temporalAttributes({ + "temporal.workflow_id": run?.workflowId, "temporal.run_id": run?.runId, + "temporal.workflow_type": info.workflowType, "temporal.activity_type": info.activityType, + "temporal.activity_id": info.activityId, "temporal.attempt": info.attempt, + "space.trace_id": correlation.traceId, "space.room_id": correlation.roomId, + "space.pane_id": correlation.paneId, "space.mission_id": correlation.missionId, + "space.goal_id": correlation.goalId, "space.outcome": observation.outcome, + "space.reason_code": observation.reasonCode, "space.category": observation.category, + duration_ms: observation.durationMs + }) + }); + } + }); + } } })] + } + } : { interceptors: { activity: [createActivityObservabilityInterceptors()] } }; + return { + workerOptions, + sinks, + stats: () => exporter?.stats() ?? { exported: 0, dropped: 0, failed: 0 }, + flush: async () => { await exporter?.flush(); }, + close: async () => { await exporter?.close(); } + }; +} diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index c0f0b9d47..96f7b0d8d 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -6,7 +6,7 @@ import { StreamingCredentialStore } from "@space/streaming"; import { TelegramDeliveryWorker, TelegramSecretStore } from "@space/runtime"; import { NativeConnection, Runtime, Worker } from "@temporalio/worker"; import * as activities from "./activities.js"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { createWorkerTelemetry } from "./worker-telemetry.js"; import { resolveTemporalCoreMetricsBind, resolveWorkerMetricsListen, startWorkerMetricsServer } from "./worker-metrics.js"; import { workerLog } from "./worker-log.js"; import { sweepCodexCliCompletions } from "./codex-cli-completion.js"; @@ -35,15 +35,18 @@ if (temporalCoreMetricsBind) { } const connection = await NativeConnection.connect({ address }); +const telemetry = createWorkerTelemetry({ serviceName: "space-worker" }); const worker = await Worker.create({ connection, namespace, taskQueue, workflowsPath, activities, - // Structured, correlated activity records and worker metrics. Activity-side - // only: the workflow sandbox and its replay behaviour are unchanged. - interceptors: { activity: [createActivityObservabilityInterceptors()] } + ...telemetry.workerOptions +}).catch(async (error: unknown) => { + await telemetry.close(); + await connection.close(); + throw error; }); // Optional loopback-only Prometheus endpoint for worker metrics. @@ -172,4 +175,6 @@ try { telegramNotificationLoop?.stop(); streamingBotLoop?.stop(); chessMoveServer?.close(); + await telemetry.close(); + await connection.close(); } diff --git a/apps/worker/src/workflow-telemetry.ts b/apps/worker/src/workflow-telemetry.ts new file mode 100644 index 000000000..896f59fe5 --- /dev/null +++ b/apps/worker/src/workflow-telemetry.ts @@ -0,0 +1,80 @@ +import { formatTraceparent, parseTraceparent } from "@space/contracts"; +import { + isCancellation, proxySinks, workflowInfo, + type Sinks, type WorkflowInterceptorsFactory +} from "@temporalio/workflow"; +import { + inputTraceparent, readTraceHeader, traceHeaders, workflowInputAttributes, workflowTraceContext, + type SpanLink, type TemporalTraceContext +} from "./temporal-trace-context.js"; + +export interface WorkflowSpanRecord { + context: TemporalTraceContext; + startTimeMs: number; + endTimeMs: number; + outcome: "COMPLETED" | "FAILED" | "CANCELLED" | "CONTINUED_AS_NEW"; + attributes: Record; +} +export interface TelemetrySinks extends Sinks { + spaceTelemetry: { record(span: WorkflowSpanRecord): void }; +} +const { spaceTelemetry } = proxySinks(); + +export const interceptors: WorkflowInterceptorsFactory = () => { + let context: TemporalTraceContext; + let outcome: WorkflowSpanRecord["outcome"] = "COMPLETED"; + let attributes: Record = {}; + const parent = () => formatTraceparent(context.traceId, context.spanId, "01"); + return { + inbound: [{ + async execute(input, next) { + // Memo decoding happens after interceptor construction. Read the + // current WorkflowInfo here so starts that propagate through memo work. + const info = workflowInfo(); + const fromInput = inputTraceparent(input.args); + const fromMemo = typeof info.memo?.traceparent === "string" ? info.memo.traceparent : undefined; + context = workflowTraceContext(info.runId, readTraceHeader(input.headers) ?? + (fromInput || fromMemo ? { traceparent: (fromInput ?? fromMemo)! } : null)); + attributes = workflowInputAttributes(input.args); + const startTimeMs = Date.now(); + try { return await next(input); } + catch (error) { + if (outcome !== "CONTINUED_AS_NEW") outcome = isCancellation(error) ? "CANCELLED" : "FAILED"; + throw error; + } finally { + // The host sink is registered with callDuringReplay:false. This call + // enqueues a record without I/O or an await in workflow code. + spaceTelemetry.record({ context, startTimeMs, endTimeMs: Date.now(), outcome, attributes }); + } + } + }], + outbound: [{ + scheduleActivity: (input, next) => next({ ...input, headers: traceHeaders(input.headers, { traceparent: parent() }) }), + scheduleLocalActivity: (input, next) => next({ ...input, headers: traceHeaders(input.headers, { traceparent: parent() }) }), + startChildWorkflowExecution(input, next) { + const args = input.options.args ?? []; + const source = inputTraceparent(args); + const links: SpanLink[] = []; + const first = args[0] as { roomAgentContinuation?: unknown } | undefined; + // A later operator request can start a mission in an older supervisor. + // Preserve its request trace and link to that supervising run. + if (source && (first?.roomAgentContinuation || source.split("-")[1] !== context.traceId)) { + links.push({ traceId: context.traceId, spanId: context.spanId }); + if (first?.roomAgentContinuation) { + const origin = parseTraceparent(source)!; + links.push({ traceId: origin.traceId, spanId: origin.parentId }); + } + return next({ ...input, headers: traceHeaders(input.headers, { traceparent: source, links }) }); + } + return next({ ...input, headers: traceHeaders(input.headers, { traceparent: parent() }) }); + }, + continueAsNew(input, next) { + outcome = "CONTINUED_AS_NEW"; + return next({ ...input, headers: traceHeaders(input.headers, { + traceparent: formatTraceparent(context.traceId, context.parentSpanId ?? context.spanId, "01"), + links: [{ traceId: context.traceId, spanId: context.spanId }] + }) }); + } + }] + }; +}; diff --git a/apps/worker/tests/support/telemetry-workflows.ts b/apps/worker/tests/support/telemetry-workflows.ts new file mode 100644 index 000000000..161be6e1b --- /dev/null +++ b/apps/worker/tests/support/telemetry-workflows.ts @@ -0,0 +1,6 @@ +export { + dummyTurnWorkflow, codexAppServerTurnWorkflow, dotGoalWorkflow, + roomAgentSupervisorWorkflow, roomAgentActionWorkflow +} from "../../src/workflows.js"; +export { memoryMaintenanceWorkflow, memoryConsolidationWorkflow } from "../../src/memory-maintenance-workflows.js"; +export { memoryMutationWorkflow } from "../../src/memory-mutation-workflows.js"; diff --git a/apps/worker/tests/worker-telemetry.test.ts b/apps/worker/tests/worker-telemetry.test.ts new file mode 100644 index 000000000..9736f7c3f --- /dev/null +++ b/apps/worker/tests/worker-telemetry.test.ts @@ -0,0 +1,326 @@ +import { createServer, type Server, type ServerResponse } from "node:http"; +import { fileURLToPath } from "node:url"; +import { randomUUID } from "node:crypto"; +import Fastify from "fastify"; +import { afterEach, beforeAll, beforeEach, expect, it } from "vitest"; +import { Context } from "@temporalio/activity"; +import { ApplicationFailure } from "@temporalio/common"; +import { Client } from "@temporalio/client"; +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { bundleWorkflowCode, DefaultLogger, Runtime, Worker } from "@temporalio/worker"; +import { createOtlpSpanExporter } from "@space/telemetry"; +import { roomAgentSupervisorInputSchema, roomAgentSupervisorQueueItemSchema } from "@space/contracts"; +import { createWorkerTelemetry } from "../src/worker-telemetry.js"; +import { childTraceparent, noteActivityFailureReason } from "../src/activity-observability.js"; +import { temporalAttributeKeys, temporalAttributes, workflowSpanId } from "../src/temporal-trace-context.js"; +import { MEMORY_MAINTENANCE_TASK_QUEUE } from "../src/memory-maintenance-config.js"; +import { MEMORY_MUTATION_TASK_QUEUE } from "../src/memory-mutation-config.js"; +import { createHttpObservability } from "../../api/src/observability.js"; +import { requestTraceContexts, traceContextForRawRequest } from "../../api/src/trace-context.js"; +import { TemporalCodexAppServerTurnStarter } from "../../api/src/turns.js"; + +type Span = { traceId: string; spanId: string; parentSpanId?: string; name: string; kind: number; + links?: Array<{ traceId: string; spanId: string }>; + attributes: Array<{ key: string; value: { stringValue?: string; intValue?: string } }>; service: string }; +const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; +const apiParent = "00f067aa0ba902b7"; +const traceparent = `00-${traceId}-${apiParent}-01`; +const secret = "PRIVATE PROMPT sk-live-not-real /home/operator/private https://secret.invalid"; +const complete = { status: "COMPLETED", message: secret }; +let collector: Server; +let endpoint: string; +let requests: any[]; +let collectorResponds: boolean; +let heldResponses: ServerResponse[]; +let env: TestWorkflowEnvironment; +let bundle: Awaited>; +let workers: Worker[]; +let running: Promise[]; +let telemetries: ReturnType[]; +let handles: Array>>; +let handlers: Record unknown>; +let workerFailures: unknown[]; +const mainQueue = "synthetic-telemetry"; +const serviceQueues = [ + ["space-worker", mainQueue], + ["space-memory-maintenance-worker", MEMORY_MAINTENANCE_TASK_QUEUE], + ["space-memory-mutation-worker", MEMORY_MUTATION_TASK_QUEUE] +] as const; + +beforeAll(async () => { + Runtime.install({ logger: new DefaultLogger("ERROR") }); + bundle = await bundleWorkflowCode({ + workflowsPath: fileURLToPath(new URL("./support/telemetry-workflows.ts", import.meta.url)), + workflowInterceptorModules: [fileURLToPath(new URL("../src/workflow-telemetry.ts", import.meta.url))], + logger: new DefaultLogger("ERROR") + }); +}, 60_000); + +beforeEach(async () => { + requests = []; workers = []; running = []; telemetries = []; handles = []; workerFailures = []; collectorResponds = true; heldResponses = []; + collector = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + requests.push({ path: request.url, ...JSON.parse(body) }); + if (collectorResponds) response.writeHead(200, { "content-type": "application/json" }).end("{}"); + else heldResponses.push(response); + }); + await new Promise(resolve => collector.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${(collector.address() as { port: number }).port}`; + env = await TestWorkflowEnvironment.createTimeSkipping(); + handlers = { + recordDummyTurnStarted: () => undefined, recordDummyTurnCompleted: () => complete, + runCodexAppServerTurn: () => complete, runOpenCodeAgentTurn: () => complete, + runRoomAgentBackgroundAction: () => complete, failRoomAgentBackgroundAction: () => undefined, + dotGoalStep: () => ({ state: "DONE", goalStatus: "COMPLETED", runId: null }), + checkRoomAgentSupervisorTurn: () => "RUN", markRoomAgentMissionStarted: () => undefined, + markRoomAgentMissionFinished: () => undefined, finishRoomAgentSupervisorTurn: () => undefined, + refreshMemoryGraphSnapshot: () => ({ sourceHash: "synthetic", nodeCount: 0, edgeCount: 0 }), + refreshAllMonthsMemoryGraphSnapshot: () => undefined, reconcileStaleSpaceAgentSessions: () => undefined, + createScheduledMemoryAudit: () => ({ id: "consolidation:synthetic", workflowId: `synthetic-child-${randomUUID()}` }), + prepareMemoryConsolidation: () => ({ changeSetId: "change:synthetic", run: { id: "consolidation:synthetic" } }), + executePersistedMemoryChangeSet: () => ({ status: "APPLIED" }), finalizeMemoryRepair: () => ({ status: "COMPLETED" }) + }; + for (const [serviceName, taskQueue] of serviceQueues) { + // Exercise the same environment configuration and options as the real entrypoints. + const telemetry = createWorkerTelemetry({ serviceName, env: { SPACE_OTEL_EXPORTER_OTLP_ENDPOINT: endpoint } }); + telemetries.push(telemetry); + const worker = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue, workflowBundle: bundle, + sinks: telemetry.workerOptions.sinks, + interceptors: { activity: telemetry.workerOptions.interceptors!.activity }, + activities: Object.fromEntries(Object.keys(handlers).map(name => [name, (...args: any[]) => handlers[name]!(...args)])), + maxCachedWorkflows: 16, maxConcurrentWorkflowTaskExecutions: 8, maxConcurrentActivityTaskExecutions: 8, + maxConcurrentWorkflowTaskPolls: 2, maxConcurrentActivityTaskPolls: 2, shutdownGraceTime: "1 second" + }); + workers.push(worker); + running.push(worker.run().catch(error => { workerFailures.push(error); })); + } +}, 30_000); + +afterEach(async () => { + try { + for (const handle of handles) if ((await handle.describe()).status.name === "RUNNING") await handle.terminate("Synthetic telemetry cleanup"); + for (const worker of workers) if (worker.getState() === "RUNNING") worker.shutdown(); + await Promise.all(running); + expect(workerFailures).toEqual([]); + } finally { + collectorResponds = true; + for (const response of heldResponses) response.writeHead(503).end(); + await Promise.all(telemetries.map(telemetry => telemetry.close())); + await env?.teardown(); + collector.closeAllConnections(); + await new Promise(resolve => collector.close(() => resolve())); + } +}, 15_000); + +function spans(): Span[] { + return requests.flatMap(request => request.resourceSpans.flatMap((resource: any) => + resource.scopeSpans.flatMap((scope: any) => scope.spans.map((span: Span) => ({ + ...span, service: resource.resource.attributes.find((attr: any) => attr.key === "service.name").value.stringValue + }))))); +} +const attrs = (span: Span) => Object.fromEntries(span.attributes.map(({ key, value }) => [key, value.stringValue ?? Number(value.intValue)])); +async function flush() { await Promise.all(telemetries.map(telemetry => telemetry.flush())); } +async function start(type: string, input: unknown, taskQueue = mainQueue) { + const handle = await env.client.workflow.start(type, { + workflowId: `synthetic-${type}-${randomUUID()}`, taskQueue, args: [input], memo: { traceparent } + }); + handles.push(handle); + return handle; +} +async function until(predicate: () => boolean) { + const deadline = Date.now() + 10_000; + while (!predicate() && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + expect(predicate()).toBe(true); +} + +it("connects the real API SERVER span to the production turn workflow and every retry attempt", async () => { + const outboundParents: Array = []; + handlers.runCodexAppServerTurn = () => { + outboundParents.push(childTraceparent(traceparent)); + if (Context.current().info.attempt === 1) throw ApplicationFailure.retryable(secret, "UPSTREAM_RETRYABLE_UNAVAILABLE"); + noteActivityFailureReason("OPENCODE_TURN_TIMEOUT"); + return { status: "FAILED", message: secret }; + }; + const apiExporter = createOtlpSpanExporter({ endpoint, serviceName: "space-api" }); + const observability = createHttpObservability({ serviceName: "space-api", spanExporter: apiExporter }); + const app = Fastify({ genReqId: () => "req:synthetic0001" }); + const starter = new TemporalCodexAppServerTurnStarter({ address: "unused", namespace: env.namespace ?? "default", taskQueue: mainQueue, + clientFactory: async () => ({ client: env.client, close: async () => undefined }) }); + app.addHook("onRequest", async request => { + observability.onRequest(request); + requestTraceContexts.remember(request.id, traceContextForRawRequest(request.raw)); + }); + app.addHook("onResponse", async (request, reply) => observability.onResponse(request, reply)); + app.post("/synthetic/turn", request => starter.start({ + roomId: "room:synthetic0001", paneId: "pane:synthetic0001", traceId: request.id, prompt: secret, artifactIds: [], + permissionMode: null, collaborationMode: "default" + })); + try { + const address = await app.listen({ port: 0, host: "127.0.0.1" }); + const response = await fetch(`${address}/synthetic/turn`, { method: "POST", headers: { traceparent } }); + expect(response.status).toBe(200); + const result = await response.json() as { workflowId: string; runId: string }; + // The manual UI proof uses wall-clock time so Jaeger does not need to + // correct clock skew between skipped workflow time and host activity time. + const resultClient = process.env.SPACE_TELEMETRY_PROOF_ENDPOINT + ? new Client({ connection: env.connection, namespace: env.namespace }) : env.client; + const handle = resultClient.workflow.getHandle(result.workflowId); + expect(await handle.result()).toMatchObject({ status: "FAILED" }); + await flush(); await apiExporter.flush(); + const recorded = spans(); + expect(recorded).toHaveLength(4); + expect(new Set(recorded.map(span => span.traceId))).toEqual(new Set([traceId])); + const server = recorded.find(span => span.kind === 2)!; + const workflow = recorded.find(span => span.name === "workflow.codexAppServerTurnWorkflow")!; + expect(server.parentSpanId).toBe(apiParent); + expect(workflow.parentSpanId).toBe(server.spanId); + expect(workflow.spanId).toBe(workflowSpanId(result.runId)); + const activities = recorded.filter(span => span.name.startsWith("activity.")); + expect(activities.map(span => span.parentSpanId)).toEqual([workflow.spanId, workflow.spanId]); + expect(new Set(activities.map(span => span.spanId)).size).toBe(2); + expect(outboundParents).toEqual(activities.map(span => `00-${span.traceId}-${span.spanId}-01`)); + expect(activities.map(attrs)).toEqual([ + expect.objectContaining({ "temporal.attempt": 1, "space.outcome": "FAILED", "space.reason_code": "UPSTREAM_RETRYABLE_UNAVAILABLE" }), + expect.objectContaining({ "temporal.attempt": 2, "space.outcome": "FAILED_RESULT", "space.reason_code": "OPENCODE_TURN_TIMEOUT", "space.category": "TIMEOUT" }) + ]); + for (const span of recorded.filter(span => span.kind === 1)) expect(span.attributes.every(attribute => temporalAttributeKeys.includes(attribute.key as any))).toBe(true); + expect(JSON.stringify(requests)).not.toMatch(/PRIVATE PROMPT|sk-live|operator\/private|secret.invalid|message|stack/); + expect(requests.every(request => request.path === "/v1/traces")).toBe(true); + + // Explicit opt-in manual proof: forward this already-checked synthetic + // trace to a local Jaeger. Normal tests contact only their fake collector. + if (process.env.SPACE_TELEMETRY_PROOF_ENDPOINT) { + const target = new URL(process.env.SPACE_TELEMETRY_PROOF_ENDPOINT); + if (target.protocol !== "http:" || !["127.0.0.1", "localhost", "[::1]"].includes(target.hostname)) { + throw new Error("The synthetic proof collector must be loopback HTTP"); + } + for (const { path: _path, ...body } of requests) { + const forwarded = await fetch(new URL("/v1/traces", target), { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify(body), signal: AbortSignal.timeout(3_000) }); + expect(forwarded.ok).toBe(true); + } + } + + // Replaying the actual history with the same sinks must enqueue no spans. + requests.length = 0; + await Worker.runReplayHistory({ workflowBundle: bundle, sinks: telemetries[0]!.sinks }, await handle.fetchHistory(), handle.workflowId); + await flush(); + expect(spans()).toEqual([]); + } finally { await app.close(); observability.close(); await apiExporter.close(); } +}, 30_000); + +it("nests the memory child workflow and its cross-queue mutation under the API parent", async () => { + const handle = await start("memoryMaintenanceWorkflow", { traceId: "req:syntheticmemory", month: "2025-01" }, MEMORY_MAINTENANCE_TASK_QUEUE); + await handle.result(); await flush(); + const recorded = spans(); + const root = recorded.find(span => span.name === "workflow.memoryMaintenanceWorkflow")!; + const child = recorded.find(span => span.name === "workflow.memoryConsolidationWorkflow")!; + expect(root.parentSpanId).toBe(apiParent); + expect(child.parentSpanId).toBe(root.spanId); + const mutation = recorded.find(span => span.name === "activity.executePersistedMemoryChangeSet")!; + expect(mutation.parentSpanId).toBe(child.spanId); + expect(mutation.service).toBe("space-memory-mutation-worker"); + expect(root.service).toBe("space-memory-maintenance-worker"); + expect(new Set(recorded.map(span => span.traceId))).toEqual(new Set([traceId])); + for (const activity of recorded.filter(span => span.name.startsWith("activity."))) { + expect([root.spanId, child.spanId]).toContain(activity.parentSpanId); + expect(attrs(activity)).toMatchObject({ "temporal.attempt": 1, "space.outcome": "COMPLETED" }); + } +}, 20_000); + +it.each(["dummyTurnWorkflow", "roomAgentActionWorkflow", "memoryMutationWorkflow"])("exports %s with an input or memo parent", async type => { + const handle = await start(type, { traceparent, roomId: "room:synthetic", paneId: "pane:synthetic", traceId: "req:synthetic", prompt: secret, + actionId: "action:synthetic", bridge: { roomId: "room:synthetic", missionId: "mission:synthetic" }, changeSetId: "change:synthetic" }, + type === "memoryMutationWorkflow" ? MEMORY_MUTATION_TASK_QUEUE : mainQueue); + await handle.result(); await flush(); + const workflow = spans().find(span => span.name === `workflow.${type}`)!; + expect(workflow).toMatchObject({ traceId, parentSpanId: apiParent }); + expect(spans().filter(span => span.name.startsWith("activity.")).every(span => span.parentSpanId === workflow.spanId)).toBe(true); +}, 15_000); + +it("links a real dot continue-as-new run to its previous run", async () => { + let steps = 0; + handlers.dotGoalStep = () => ({ state: ++steps <= 300 ? "WAITING" : "DONE", goalStatus: steps <= 300 ? "ACTIVE" : "COMPLETED", runId: null }); + const handle = await start("dotGoalWorkflow", { goalId: "goal:synthetic" }); + await handle.result(); await flush(); + const workflows = spans().filter(span => span.name === "workflow.dotGoalWorkflow"); + expect(workflows).toHaveLength(2); + const first = workflows.find(span => attrs(span)["space.outcome"] === "CONTINUED_AS_NEW")!; + const second = workflows.find(span => span !== first)!; + expect(first.parentSpanId).toBe(apiParent); + expect(second).toMatchObject({ traceId, parentSpanId: apiParent, links: [{ traceId, spanId: first.spanId }] }); + expect(spans().filter(span => span.name === "activity.dotGoalStep")).toHaveLength(301); +}, 30_000); + +it("links a background continuation to its supervising run and keeps its originating request trace", async () => { + const childTrace = "1234567890abcdef1234567890abcdef"; + const item = roomAgentSupervisorQueueItemSchema.parse({ missionId: "mission:synthetic", turn: { + roomId: "room:synthetic", paneId: "pane:synthetic", traceId: "trace:bg-synthetic", prompt: secret, + traceparent: `00-${childTrace}-${apiParent}-01`, agentRunId: "agent_run:synthetic", + roomAgentContinuation: { sourceRunId: "agent_run:source", generation: 1 } + } }); + const handle = await start("roomAgentSupervisorWorkflow", roomAgentSupervisorInputSchema.parse({ roomId: "room:synthetic", pending: [item] })); + await until(() => telemetries[0]!.stats().exported > 0 || requests.length > 0); + await flush(); + const child = spans().find(span => span.name === "workflow.codexAppServerTurnWorkflow"); + expect(child).toMatchObject({ traceId: childTrace, parentSpanId: apiParent, + links: [{ traceId, spanId: workflowSpanId(handle.firstExecutionRunId) }, { traceId: childTrace, spanId: apiParent }] }); +}, 20_000); + +it("rejects non-identifier attributes and makes the unset exporter a no-op", async () => { + const telemetry = createWorkerTelemetry({ serviceName: "space-worker", env: {} }); + expect(telemetry.workerOptions.sinks).toBeUndefined(); + expect(telemetry.workerOptions.interceptors?.workflowModules).toBeUndefined(); + expect(temporalAttributes({ "space.room_id": secret, "space.pane_id": "/home/secret", "space.mission_id": "https://secret.invalid", + "temporal.attempt": Infinity, prompt: secret, message: secret, error: secret, "space.trace_id": "req:synthetic" })) + .toEqual({ "space.trace_id": "req:synthetic" }); + await telemetry.flush(); await telemetry.close(); + expect(telemetry.stats()).toEqual({ exported: 0, dropped: 0, failed: 0 }); + expect(requests).toEqual([]); +}); + +it("completes real workflows before a stalled collector responds, with bounded drops", async () => { + collectorResponds = false; + const exporter = createOtlpSpanExporter({ endpoint, serviceName: "space-worker", batchSize: 1, maxQueue: 2, + timeoutMs: 30_000, flushIntervalMs: 60_000 }); + const telemetry = createWorkerTelemetry({ serviceName: "space-worker", exporter }); + telemetries.push(telemetry); + const taskQueue = "synthetic-backpressure"; + const worker = await Worker.create({ + connection: env.nativeConnection, namespace: env.namespace, taskQueue, workflowBundle: bundle, + sinks: telemetry.workerOptions.sinks, interceptors: { activity: telemetry.workerOptions.interceptors!.activity }, + activities: { recordDummyTurnStarted: () => undefined, recordDummyTurnCompleted: () => complete }, + maxCachedWorkflows: 16, shutdownGraceTime: "1 second", + maxConcurrentWorkflowTaskExecutions: 4, maxConcurrentActivityTaskExecutions: 4, + maxConcurrentWorkflowTaskPolls: 2, maxConcurrentActivityTaskPolls: 2 + }); + workers.push(worker); running.push(worker.run().catch(error => { workerFailures.push(error); })); + const started = await Promise.all(Array.from({ length: 2 }, () => start("dummyTurnWorkflow", { traceparent }, taskQueue))); + // The test deadline is shorter than the POST timeout. The collector cannot + // release its first request until BOTH workflows have already completed. + // No workflow timers are involved here. Keep the global test clock locked + // while concurrent activity attempts complete instead of auto-skipping it. + const realTime = new Client({ connection: env.connection, namespace: env.namespace }); + expect(await Promise.all(started.map(handle => realTime.workflow.getHandle(handle.workflowId).result()))).toHaveLength(2); + await until(() => heldResponses.length > 0); + expect(telemetry.stats()).toMatchObject({ exported: 0, failed: 0 }); + expect(telemetry.stats().dropped).toBe(3); + collectorResponds = true; + for (const response of heldResponses.splice(0)) response.writeHead(503).end(); + await telemetry.flush(); + expect(telemetry.stats()).toEqual({ exported: 2, dropped: 3, failed: 1 }); +}, 15_000); + +it("times out a real HTTP POST and counts its failure", async () => { + collectorResponds = false; + const exporter = createOtlpSpanExporter({ endpoint, serviceName: "space-worker", timeoutMs: 50 }); + exporter.record({ traceId, spanId: apiParent, parentSpanId: null, name: "synthetic-timeout", startTimeUnixNano: 1n, + endTimeUnixNano: 2n, attributes: {}, error: false }); + try { + await exporter.flush(); + expect(exporter.stats()).toEqual({ exported: 0, dropped: 0, failed: 1 }); + expect(heldResponses).toHaveLength(1); + } finally { await exporter.close(); } +}); diff --git a/apps/worker/tsconfig.telemetry-tests.json b/apps/worker/tsconfig.telemetry-tests.json new file mode 100644 index 000000000..d3fd7eb19 --- /dev/null +++ b/apps/worker/tsconfig.telemetry-tests.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "rootDir": "../..", "noEmit": true }, + "include": ["tests/worker-telemetry.test.ts", "tests/support/telemetry-workflows.ts"] +} diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index e432af4c6..5f0586f03 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -52,3 +52,27 @@ background work, and is hidden in the web app. Public installs upgraded from 1.1.x that used Job Copilot, the games, channels, Telegram or the desktop routes set the modules they keep, for example `SPACE_OPTIONAL_MODULES=job-copilot,games`. + +## Optional API and worker tracing + +`SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` is a collector **base URL**, for example +`http://127.0.0.1:4318`. Set it separately in the environment of the API, main +Temporal worker, memory-maintenance worker, and memory-mutation worker. Each +posts OTLP/HTTP JSON to `/v1/traces` under its own service name. In containers, +use a collector address reachable from that container; its loopback is not the +host loopback. This setting is outside `SpaceApiConfig` and its registry. + +Export is off by default. Empty/invalid URLs or URLs with credentials, query +parameters, or fragments disable it. No tracing SDK, provider credential, or +paid backend is required. API spans parent workflow spans, which parent activity +attempts and child workflows; continuations include span links. Only allowlisted +identifiers, status/error codes, and durations are exported, never prompt/output +text, paths, full URLs, or exception messages. + +The shared exporter bounds each queue at 4,096 spans, batches up to 256, flushes +every 5 seconds, and times out each POST after 3 seconds. It drops/counts excess +spans and counts failed batches. Collection never blocks workflow or activity +execution. Workflow spans appear at completion, failure, cancellation or +continue-as-new; an open supervisor has not emitted its final span yet. +See [observability](quality/observability.md#traces-w3c-trace-context) for replay +behavior, limits, and integration-test coverage. diff --git a/docs/quality/observability.md b/docs/quality/observability.md index 6016baa92..e9581447e 100644 --- a/docs/quality/observability.md +++ b/docs/quality/observability.md @@ -212,42 +212,86 @@ Without the endpoint, every activity still writes `activity.started` and ## Traces (W3C trace context) -The trace context is local by default. An optional OTLP/HTTP (JSON) exporter sends one -SERVER span per API request when `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` is set (no -OpenTelemetry dependency; `apps/api/src/otlp-span-exporter.ts`). -- **API:** a valid inbound `traceparent` is continued (same trace ID, new span). - An invalid or missing header gets a new trace ID. Every API log line carries - `w3cTraceId` (Fastify `childLoggerFactory`). -- **Propagation into turns:** API turn starters and the Room Agent coordinator add - the request's `traceparent` to the turn input (an optional contract field) and - to the Temporal memo. A bounded in-memory map from request ID to trace context - (`apps/api/src/trace-context.ts`) does this without passing the request object - through every service. -- **Worker:** worker logs read the trace ID from the activity input. Internal - room-action calls from the worker send a child `traceparent`, so the internal - API request joins the same trace. -- **Workflow code is unchanged.** Correlation is done in the API starter - (input/memo) and in an activity interceptor. The workflow bundle only gains the - optional contract field. +The trace context is local by default. Set `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` +to a collector base URL in the API and each worker process to export OTLP/HTTP +JSON to `/v1/traces`. The API, main worker, memory-maintenance worker, and +memory-mutation worker share `@space/telemetry`; no OpenTelemetry SDK is needed. +An unset or invalid endpoint disables export. The URL must be HTTP(S), without +credentials, query parameters, or a fragment. Export is optional and does not +require an account with a tracing service. + +- The API emits one SERVER span per request. A valid inbound `traceparent` + continues that trace; otherwise the API generates a trace ID. +- Turn, supervisor, action, Dot goal, and memory starters propagate the actual + API span ID through input or memo. The bounded request-context registry only + retains identifiers. A scheduled workflow without a request starts a trace + from its Temporal run ID. +- Each workflow run emits an INTERNAL span when it completes, fails, is + cancelled, or continues as new. Its stable span ID is derived from the run + UUID without consuming the workflow's random-number sequence. An active, + long-lived supervisor's span is available when that run finishes; its activity + spans can arrive first. Server-side termination cannot run the workflow's + completion interceptor and therefore cannot emit its final span. +- Every activity attempt has its own INTERNAL span under the workflow span, + including retries, failed-result returns, and cancellations. It reuses the + existing outcome, stable reason code, category, and duration observation. + Child workflows nest below the parent workflow, including memory consolidation + and work on the separate mutation queue. +- Continue-as-new preserves the original request parent and links to the prior + run. Background continuations retain their originating W3C trace and link to + both its request span and the supervising run. Space's `trace:bg-` + remains an application correlation ID, distinct from the W3C trace ID. + +Outbound internal HTTP calls made inside an observed activity propagate that +activity span as their parent, so a nested API request does not reference an +unexported intermediate span. + +Each process has a bounded queue (4,096 waiting spans, batches of 256), a +3-second timeout per HTTP POST, and a 5-second flush interval. `record()` only +queues; workflow and activity execution never await collector I/O. Full queues +increment `dropped`; failed batches increment `failed` and are not retried. +`createWorkerTelemetry().stats()` exposes exported, dropped, and failed counts. +Shutdown drains the bounded queue; a failing collector cannot fail work. ### Replay compatibility -No file under `apps/worker/src` that the workflow bundle imports was changed -(`workflows.ts`, `room-supervisor-state.ts`, `turn-runtime-policy.ts`, -`dot-goal-loop.ts`). The `agent-turns` workflow bundle was built for the base -commit and for this change and compared after normalizing paths and webpack -module numbers. The only differences are in `@space/contracts`: -- the new `observability` module; -- the optional `traceparent` turn-input field; -- the optional file search and readiness fields. +Workflow tracing uses sandbox-safe interceptors for context propagation and a +sink registered with `callDuringReplay: false`. HTTP export, timers for flushing, +process environment reads, and random activity span IDs live in the worker +process. Production workflow functions, retry policies, and patch decisions do +not change. No network call occurs in the workflow sandbox. -A history without `traceparent` parses exactly as before. Activity inputs are -not part of Temporal's determinism check. +`apps/worker/tests/worker-telemetry.test.ts` connects real HTTP request +instrumentation and the production turn starter to a time-skipping Temporal +server and local HTTP collector. It checks the complete parent tree, retries, +redaction, child workflows on separate workers, continuation links, and zero +spans when the same history is replayed. The existing API exporter tests also +run unchanged. These are synthetic tests without a database or AI provider. +The SDK downloads its test server on first use; subsequent runs use its cache. -The read-only live replay (`node scripts/temporal-replay-check.mjs --limit 30`) -could not be run from the agent sandbox that made this change. Its network -namespace has no route to the Temporal container (`172.18.0.5:7233`), and -`127.0.0.1:7233` refuses connections. Run it before deploying this change. +### Local Jaeger proof + +Start a disposable collector using the official Jaeger all-in-one image: + +```sh +docker run --rm --name spaceapp-trace-proof -e COLLECTOR_OTLP_ENABLED=true \ + -p 127.0.0.1:16686:16686 -p 127.0.0.1:4318:4318 jaegertracing/all-in-one:1.76.0 +``` + +In the checkout, after building dependencies, run: + +```sh +SPACE_TELEMETRY_PROOF_ENDPOINT=http://127.0.0.1:4318 \ + npm test -w @space/worker -- tests/worker-telemetry.test.ts -t 'connects the real API' +``` + +Open `http://127.0.0.1:16686`, choose `space-api`, and find the trace. It contains +an HTTP SERVER span, the production turn workflow, and two real activity attempts +with injected failures. The activities are synthetic: this proof does not call +an AI provider. The opt-in helper forwards only the trace already checked by the +redaction assertions, accepts loopback HTTP only, and uses wall-clock workflow +execution for the UI proof. Normal tests do not contact Jaeger. Stop the disposable +collector when finished; its in-memory trace data is discarded. ## Redaction policy @@ -295,9 +339,10 @@ No credential or transcript export was added. have no `trace_id` or error-code column, and the store ignores the `traceId` argument of those methods. Adding them needs a migration. Today the link is `TURN_FAILED` (with the run ID) plus the workflow ID suffix. -- **Background continuations start a new trace.** They create - `trace:bg-`, linked only through `roomAgentContinuation.sourceRunId`. - Chat-pane sends from room actions use `:chat`. +- **Application IDs remain distinct.** Background continuations use + `trace:bg-` and chat-pane sends use `:chat`. OTLP continuation + links preserve their W3C trace relationship; mission/task database rows still + do not have trace columns. - **CLI terminal task IDs are not linked.** `cliTaskId` and `revisionId` are not tied to request IDs. - **Hosts are observed from the API, not instrumented.** Since 2026-10-12 the @@ -309,14 +354,14 @@ No credential or transcript export was added. The CLI hosts are checked every 30 s by the terminal manager; the Browser Host at each `/readyz`. Memory and CPU come from `/proc` only when the PID's command line is that host. The hosts themselves still have no metrics endpoint or - spans (a normal deploy never restarts them), and the memory-maintenance and - memory-mutation workers have no activity interceptor or metrics. + spans (a normal deploy never restarts them). Memory workers now use the shared + activity observation interceptor but do not expose their own metrics endpoint. - **Some workflow-level faults are not in the trace lookup.** Workflow-task failures and nondeterminism appear only in Temporal history. The lookup returns events, audit entries and workflow rows, not `space_agent_runs` rows. -- **Span export covers API requests only.** The optional OTLP exporter sends API - SERVER spans (method, route template, status, request id); workers and hosts emit - no spans, so worker and host work joins traces through log lines only. +- **CLI and Browser Host span export remains open.** API and all three Temporal + workers export spans. CLI/Browser Host operations still join traces through + correlation in logs rather than their own spans. - **Recovery counters are partial.** Recovery counters exist for plan model retries, CLI terminal reconnects and worker activity retries only. - **Worker endpoints are not scraped yet.** Since 2026-10-14 stable's diff --git a/package-lock.json b/package-lock.json index 8fb65e91c..db934a165 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,6 +63,7 @@ "@space/opencode-control": "0.2.5", "@space/runtime": "0.2.5", "@space/streaming": "0.2.5", + "@space/telemetry": "0.2.5", "@t3-oss/env-core": "^0.13.11", "@temporalio/client": "1.24.0", "@xmldom/xmldom": "^0.8.15", @@ -203,13 +204,17 @@ "@space/memory-graph": "0.2.5", "@space/opencode-control": "0.2.5", "@space/streaming": "0.2.5", + "@space/telemetry": "0.2.5", "@temporalio/activity": "1.24.0", "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", "@temporalio/worker": "1.24.0", "@temporalio/workflow": "1.24.0", "undici": "8.10.2" }, "devDependencies": { + "@temporalio/proto": "1.24.0", + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.11" } @@ -3249,6 +3254,10 @@ "resolved": "packages/streaming", "link": true }, + "node_modules/@space/telemetry": { + "resolved": "packages/telemetry", + "link": true + }, "node_modules/@space/web": { "resolved": "apps/web", "link": true @@ -3685,6 +3694,25 @@ "node": ">=12.0.0" } }, + "node_modules/@temporalio/testing": { + "version": "1.24.0", + "resolved": "https://registry.npmjs.org/@temporalio/testing/-/testing-1.24.0.tgz", + "integrity": "sha512-XKCuvgb7smKR3oXy5nDgwYSFX/hmg9zTJ2rZQQsVwJzFkga1TbnWDiuT5YMU0INKjoLhnM6c5YA/uLKlp8sdfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@temporalio/activity": "1.24.0", + "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", + "@temporalio/core-bridge": "1.24.0", + "@temporalio/proto": "1.24.0", + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0" + }, + "engines": { + "node": ">= 20.3.0" + } + }, "node_modules/@temporalio/worker": { "version": "1.24.0", "resolved": "https://registry.npmjs.org/@temporalio/worker/-/worker-1.24.0.tgz", @@ -11643,7 +11671,7 @@ } }, "packages/run-spaceapp": { - "version": "1.0.35", + "version": "1.1.5", "cpu": [ "x64", "arm64" @@ -11721,6 +11749,14 @@ "engines": { "node": ">=12.0.0" } + }, + "packages/telemetry": { + "name": "@space/telemetry", + "version": "0.2.5", + "license": "BUSL-1.1", + "devDependencies": { + "typescript": "^5.9.3" + } } } } diff --git a/package.json b/package.json index b488fbbcb..44907fd77 100644 --- a/package.json +++ b/package.json @@ -47,12 +47,12 @@ "dev:web": "npm run dev -w @space/web", "dev:worker": "npm run dev -w @space/worker", "db:migrate": "npm run migrate -w @space/db", - "build": "npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && node scripts/build-demo-projects.mjs && npm run build -w @space/api && npm run build -w @space/worker && npm run build -w @space/web && node scripts/fix-build-permissions.mjs", + "build": "npm run build -w @space/telemetry && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && node scripts/build-demo-projects.mjs && npm run build -w @space/api && npm run build -w @space/worker && npm run build -w @space/web && node scripts/fix-build-permissions.mjs", "public:export:verify": "node scripts/public-export.mjs --verify", - "check": "npm run test:architecture && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm run check -w @space/api && npm run check -w @space/worker && npm run check -w @space/web", + "check": "npm run build -w @space/telemetry && npm run test:architecture && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm run check -w @space/api && npm run check -w @space/worker && npm run check -w @space/web", "test": "npm run test:public", "test:deploy": "node --test scripts/tests/composition-root-size.test.mjs scripts/tests/domain-boundaries.test.mjs", - "test:public": "node --test scripts/tests/public-suite.test.mjs && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm test --workspaces --if-present", + "test:public": "npm run build -w @space/telemetry && node --test scripts/tests/public-suite.test.mjs && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm test --workspaces --if-present", "hygiene:status": "node scripts/space-hygiene-check.mjs", "hygiene:preflight": "node scripts/space-hygiene-check.mjs --preflight", "build:demos": "node scripts/build-demo-projects.mjs", diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json new file mode 100644 index 000000000..ee3b60065 --- /dev/null +++ b/packages/telemetry/package.json @@ -0,0 +1,22 @@ +{ + "name": "@space/telemetry", + "version": "0.2.5", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "check": "tsc -p tsconfig.json --noEmit" + }, + "devDependencies": { + "typescript": "^5.9.3" + }, + "license": "BUSL-1.1" +} diff --git a/packages/telemetry/src/index.ts b/packages/telemetry/src/index.ts new file mode 100644 index 000000000..9320db225 --- /dev/null +++ b/packages/telemetry/src/index.ts @@ -0,0 +1,164 @@ +/** + * Optional OTLP/HTTP (JSON) export shared by the API and Temporal workers. Off unless + * SPACE_OTEL_EXPORTER_OTLP_ENDPOINT names a collector base URL (spans go to + * `/v1/traces`). No dependency, no collector required: Space logs stay + * joinable by W3C trace id without it. + * + * Callers supply their own strict attribute allowlist. They never pass URLs + * with parameters, headers, bodies, prompts, error messages or other user data. + * Export is best effort: a bounded queue drops (and counts) spans instead of + * growing, each POST has a timeout, and a failing collector never affects a + * request. + */ + +export interface ServerSpan { + traceId: string; + spanId: string; + parentSpanId: string | null; + name: string; + startTimeUnixNano: bigint; + endTimeUnixNano: bigint; + attributes: Readonly>; + error: boolean; + /** OTLP SpanKind. Defaults to SERVER for the existing API callers. */ + kind?: 1 | 2 | 3 | 4 | 5; + links?: readonly { traceId: string; spanId: string }[]; +} + +export interface SpanExporterStats { + exported: number; + dropped: number; + failed: number; +} + +export interface SpanExporter { + record(span: ServerSpan): void; + flush(): Promise; + close(): Promise; + stats(): SpanExporterStats; +} + +type FetchLike = (url: string, init: { method: string; headers: Record; body: string; signal: AbortSignal }) => Promise<{ ok: boolean }>; + +export interface OtlpSpanExporterOptions { + endpoint: string; + serviceName: string; + fetch?: FetchLike; + batchSize?: number; + maxQueue?: number; + flushIntervalMs?: number; + timeoutMs?: number; +} + +const hexId = (length: number) => new RegExp(`^[0-9a-f]{${length}}$`); +const traceIdPattern = hexId(32); +const spanIdPattern = hexId(16); + +/** The collector base URL, or null when export is off or the value is unsafe. */ +export function otlpTracesUrl(value: string | undefined): string | null { + if (!value?.trim()) return null; + let url: URL; + try { + url = new URL(value.trim()); + } catch { + return null; + } + if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password || url.search || url.hash) return null; + return `${url.origin}${url.pathname.replace(/\/+$/, "")}/v1/traces`; +} + +function attributeValue(value: string | number) { + return typeof value === "number" && Number.isInteger(value) ? { intValue: String(value) } : { stringValue: String(value) }; +} + +export function otlpTraceRequest(spans: readonly ServerSpan[], serviceName: string) { + return { + resourceSpans: [{ + resource: { attributes: [{ key: "service.name", value: { stringValue: serviceName } }] }, + scopeSpans: [{ + scope: { name: serviceName === "space-api" ? "space-api" : "space-temporal-worker" }, + spans: spans.map(span => ({ + traceId: span.traceId, + spanId: span.spanId, + ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}), + name: span.name, + // SPAN_KIND_SERVER + kind: span.kind ?? 2, + ...(span.links?.length ? { links: span.links.filter(link => traceIdPattern.test(link.traceId) && spanIdPattern.test(link.spanId)) } : {}), + startTimeUnixNano: span.startTimeUnixNano.toString(), + endTimeUnixNano: span.endTimeUnixNano.toString(), + attributes: Object.entries(span.attributes).map(([key, value]) => ({ key, value: attributeValue(value) })), + // STATUS_CODE_ERROR or STATUS_CODE_UNSET + status: { code: span.error ? 2 : 0 } + })) + }] + }] + }; +} + +export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanExporter { + const url = otlpTracesUrl(options.endpoint); + if (!url) throw new Error("Invalid OTLP endpoint."); + const send: FetchLike = options.fetch ?? ((target, init) => fetch(target, init)); + const batchSize = options.batchSize ?? 256; + const maxQueue = options.maxQueue ?? 4096; + const timeoutMs = options.timeoutMs ?? 3000; + const queue: ServerSpan[] = []; + const stats: SpanExporterStats = { exported: 0, dropped: 0, failed: 0 }; + let inFlight: Promise | null = null; + + const flushOnce = async () => { + const batch = queue.splice(0, batchSize); + if (!batch.length) return; + try { + const response = await send(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(otlpTraceRequest(batch, options.serviceName)), + signal: AbortSignal.timeout(timeoutMs) + }); + if (response.ok) stats.exported += batch.length; + else stats.failed += batch.length; + } catch { + stats.failed += batch.length; + } + }; + const flush = (): Promise => { + if (!inFlight) { + inFlight = (async () => { + try { + while (queue.length) await flushOnce(); + } finally { + inFlight = null; + } + })(); + } + return inFlight; + }; + const timer = setInterval(() => void flush(), options.flushIntervalMs ?? 5000); + timer.unref?.(); + + return { + record(span) { + if (!traceIdPattern.test(span.traceId) || !spanIdPattern.test(span.spanId)) return; + if (queue.length >= maxQueue) { + stats.dropped += 1; + return; + } + queue.push(span); + if (queue.length >= batchSize) void flush(); + }, + flush, + async close() { + clearInterval(timer); + await flush(); + }, + stats: () => ({ ...stats }) + }; +} + +/** The exporter configured by SPACE_OTEL_EXPORTER_OTLP_ENDPOINT, or null when export is off. */ +export function otlpSpanExporterFromEnv(env: NodeJS.ProcessEnv, serviceName: string): SpanExporter | null { + const endpoint = env.SPACE_OTEL_EXPORTER_OTLP_ENDPOINT; + return otlpTracesUrl(endpoint) ? createOtlpSpanExporter({ endpoint: endpoint!, serviceName }) : null; +} diff --git a/packages/telemetry/tsconfig.json b/packages/telemetry/tsconfig.json new file mode 100644 index 000000000..778e54839 --- /dev/null +++ b/packages/telemetry/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true + }, + "include": [ + "src/**/*.ts" + ] +} From d8cc0981dcfee51b197c1af043f81b18c393f5a0 Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:48:24 -0400 Subject: [PATCH 2/7] fix(ci): preserve public validation and workspace build ordering --- .github/workflows/ci.yml | 11 ++++++++--- package.json | 4 ++-- scripts/tests/public-package-metadata.test.mjs | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb41c52df..7ef282df6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,19 +19,24 @@ jobs: name: Connected Temporal tracing runs-on: ubuntu-latest timeout-minutes: 10 + defaults: + run: + working-directory: ${{ runner.temp }}/spaceapp-tracing-export steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm + - name: Create sanitized tracing candidate + working-directory: ${{ github.workspace }} + run: node scripts/public-export.mjs --output "$RUNNER_TEMP/spaceapp-tracing-export" - run: ELECTRON_SKIP_BINARY_DOWNLOAD=1 npm ci - run: npm run check - run: npm run telemetry:check -w @space/worker - run: npm test -w @space/worker -- tests/worker-telemetry.test.ts tests/activity-observability.test.ts - - run: npm test -w @space/api -- otlp trace-correlation memory-consolidation-coordinator memory-mutation-coordinator dot-goals public-quality: name: Sanitized public quality gates diff --git a/package.json b/package.json index 44907fd77..031d4dcbb 100644 --- a/package.json +++ b/package.json @@ -49,10 +49,10 @@ "db:migrate": "npm run migrate -w @space/db", "build": "npm run build -w @space/telemetry && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && node scripts/build-demo-projects.mjs && npm run build -w @space/api && npm run build -w @space/worker && npm run build -w @space/web && node scripts/fix-build-permissions.mjs", "public:export:verify": "node scripts/public-export.mjs --verify", - "check": "npm run build -w @space/telemetry && npm run test:architecture && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm run check -w @space/api && npm run check -w @space/worker && npm run check -w @space/web", + "check": "npm run test:architecture && npm run build -w @space/telemetry && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm run check -w @space/api && npm run check -w @space/worker && npm run check -w @space/web", "test": "npm run test:public", "test:deploy": "node --test scripts/tests/composition-root-size.test.mjs scripts/tests/domain-boundaries.test.mjs", - "test:public": "npm run build -w @space/telemetry && node --test scripts/tests/public-suite.test.mjs && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm test --workspaces --if-present", + "test:public": "node --test scripts/tests/public-suite.test.mjs && npm run build -w @space/telemetry && npm run build -w @space/contracts && npm run build -w @space/icons && npm run build -w @space/streaming && npm run build -w @space/opencode-control && npm run build -w @space/memory-graph && npm run build -w @space/browser-host && npm run build -w @space/cli-host && npm run build -w @space/codex-app-server && npm run build -w @space/runtime && npm run build -w @space/db && npm run build -w @space/gateway && npm test --workspaces --if-present", "hygiene:status": "node scripts/space-hygiene-check.mjs", "hygiene:preflight": "node scripts/space-hygiene-check.mjs --preflight", "build:demos": "node scripts/build-demo-projects.mjs", diff --git a/scripts/tests/public-package-metadata.test.mjs b/scripts/tests/public-package-metadata.test.mjs index cf92bdea8..5183030e7 100644 --- a/scripts/tests/public-package-metadata.test.mjs +++ b/scripts/tests/public-package-metadata.test.mjs @@ -83,6 +83,7 @@ test("public test command uses a portable one-file suite that imports every publ .filter((name) => name.startsWith("@space/")) .sort(); const publicBuildOrder = [ + "@space/telemetry", "@space/contracts", "@space/streaming", "@space/opencode-control", From 6ee1f29a582522e5abe8b8d556b18fd8469ab7cc Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:53:24 -0400 Subject: [PATCH 3/7] fix(ci): resolve runner paths at step scope --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ef282df6..94dd92421 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,9 +19,6 @@ jobs: name: Connected Temporal tracing runs-on: ubuntu-latest timeout-minutes: 10 - defaults: - run: - working-directory: ${{ runner.temp }}/spaceapp-tracing-export steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -34,9 +31,13 @@ jobs: working-directory: ${{ github.workspace }} run: node scripts/public-export.mjs --output "$RUNNER_TEMP/spaceapp-tracing-export" - run: ELECTRON_SKIP_BINARY_DOWNLOAD=1 npm ci + working-directory: ${{ runner.temp }}/spaceapp-tracing-export - run: npm run check + working-directory: ${{ runner.temp }}/spaceapp-tracing-export - run: npm run telemetry:check -w @space/worker + working-directory: ${{ runner.temp }}/spaceapp-tracing-export - run: npm test -w @space/worker -- tests/worker-telemetry.test.ts tests/activity-observability.test.ts + working-directory: ${{ runner.temp }}/spaceapp-tracing-export public-quality: name: Sanitized public quality gates From 10b501cf970a2be0ba8ea261b2a9d9bbeed88847 Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:35:19 -0400 Subject: [PATCH 4/7] Keep idle OTLP flushes from pinning the export queue --- apps/api/tests/otlp-span-exporter.test.ts | 15 +++++++++++++++ packages/runtime/src/otlp-span-exporter.ts | 3 +++ 2 files changed, 18 insertions(+) diff --git a/apps/api/tests/otlp-span-exporter.test.ts b/apps/api/tests/otlp-span-exporter.test.ts index d94ec6539..869566253 100644 --- a/apps/api/tests/otlp-span-exporter.test.ts +++ b/apps/api/tests/otlp-span-exporter.test.ts @@ -53,6 +53,21 @@ describe("OTLP span export configuration", () => { }); describe("OTLP span exporter", () => { + it("exports new spans after an empty idle flush and after a drained batch", async () => { + const fetchImpl = vi.fn(async () => ({ ok: true })); + const created = exporter(fetchImpl); + await created.flush(); + await created.flush(); + created.record(span()); + await created.flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + await created.flush(); + created.record(span({ spanId: "1111111111111111" })); + await created.close(); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(created.stats()).toEqual({ exported: 2, dropped: 0, failed: 0 }); + }); + it("posts OTLP/HTTP JSON server spans with only route-level attributes", async () => { const fetchImpl = vi.fn(async () => ({ ok: true })); const created = exporter(fetchImpl); diff --git a/packages/runtime/src/otlp-span-exporter.ts b/packages/runtime/src/otlp-span-exporter.ts index 9320db225..0ec3b031c 100644 --- a/packages/runtime/src/otlp-span-exporter.ts +++ b/packages/runtime/src/otlp-span-exporter.ts @@ -124,6 +124,9 @@ export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanEx } }; const flush = (): Promise => { + // An empty async IIFE can finish its finally block before its promise is + // assigned, leaving inFlight permanently set after an idle timer tick. + if (!inFlight && queue.length === 0) return Promise.resolve(); if (!inFlight) { inFlight = (async () => { try { From c76b6ef5dffda29bcf9c4c301e9957094f8a2f1e Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:52:38 -0400 Subject: [PATCH 5/7] Normalize credential fixture byte counts on BSD and GNU systems --- scripts/tests/public-credential-install.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/public-credential-install.test.mjs b/scripts/tests/public-credential-install.test.mjs index 1f6a36bdd..b5e63c814 100644 --- a/scripts/tests/public-credential-install.test.mjs +++ b/scripts/tests/public-credential-install.test.mjs @@ -84,7 +84,7 @@ const ask = (socketPath, body) => new Promise((resolve, reject) => { test('the credential agent runs only a wrapper\'s fixed credential actions, input on stdin', async () => { // Short path: Unix socket paths are limited to about 100 bytes. const root = await mkdtemp('/tmp/spaceapp-agent-'); - await writeFile(join(root, 'codex-vscode-parity'), '#!/bin/sh\nprintf "%s %s\\n" "$1" "$(wc -c)"\n'); + await writeFile(join(root, 'codex-vscode-parity'), '#!/bin/sh\nprintf "%s %s\\n" "$1" "$(( $(wc -c) ))"\n'); await chmod(join(root, 'codex-vscode-parity'), 0o755); const socketPath = join(root, 'credentials.sock'); const server = await startCredentialAgent({ socketPath, commandPath: root }); From 7d2f0a43abb05cd9539c5b2685856c8a87e746ed Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:58:17 -0400 Subject: [PATCH 6/7] Classify the actual workflow outcome after caught continuation attempts --- apps/worker/src/workflow-telemetry.ts | 5 ++--- apps/worker/tests/support/telemetry-workflows.ts | 14 ++++++++++++++ apps/worker/tests/worker-telemetry.test.ts | 9 +++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/apps/worker/src/workflow-telemetry.ts b/apps/worker/src/workflow-telemetry.ts index 896f59fe5..b1efe7444 100644 --- a/apps/worker/src/workflow-telemetry.ts +++ b/apps/worker/src/workflow-telemetry.ts @@ -1,6 +1,6 @@ import { formatTraceparent, parseTraceparent } from "@space/contracts"; import { - isCancellation, proxySinks, workflowInfo, + ContinueAsNew, isCancellation, proxySinks, workflowInfo, type Sinks, type WorkflowInterceptorsFactory } from "@temporalio/workflow"; import { @@ -39,7 +39,7 @@ export const interceptors: WorkflowInterceptorsFactory = () => { const startTimeMs = Date.now(); try { return await next(input); } catch (error) { - if (outcome !== "CONTINUED_AS_NEW") outcome = isCancellation(error) ? "CANCELLED" : "FAILED"; + outcome = error instanceof ContinueAsNew ? "CONTINUED_AS_NEW" : isCancellation(error) ? "CANCELLED" : "FAILED"; throw error; } finally { // The host sink is registered with callDuringReplay:false. This call @@ -69,7 +69,6 @@ export const interceptors: WorkflowInterceptorsFactory = () => { return next({ ...input, headers: traceHeaders(input.headers, { traceparent: parent() }) }); }, continueAsNew(input, next) { - outcome = "CONTINUED_AS_NEW"; return next({ ...input, headers: traceHeaders(input.headers, { traceparent: formatTraceparent(context.traceId, context.parentSpanId ?? context.spanId, "01"), links: [{ traceId: context.traceId, spanId: context.spanId }] diff --git a/apps/worker/tests/support/telemetry-workflows.ts b/apps/worker/tests/support/telemetry-workflows.ts index 161be6e1b..aa2e43eaf 100644 --- a/apps/worker/tests/support/telemetry-workflows.ts +++ b/apps/worker/tests/support/telemetry-workflows.ts @@ -4,3 +4,17 @@ export { } from "../../src/workflows.js"; export { memoryMaintenanceWorkflow, memoryConsolidationWorkflow } from "../../src/memory-maintenance-workflows.js"; export { memoryMutationWorkflow } from "../../src/memory-mutation-workflows.js"; + +import { ApplicationFailure, CancelledFailure, ContinueAsNew, continueAsNew } from "@temporalio/workflow"; + +// A continue-as-new attempt can be caught by workflow code. Telemetry must +// classify the actual completion boundary, not the attempted outbound command. +export async function caughtContinuationWorkflow(outcome: "COMPLETED" | "FAILED" | "CANCELLED") { + try { await continueAsNew(outcome); } + catch (error) { + if (!(error instanceof ContinueAsNew)) throw error; + if (outcome === "FAILED") throw ApplicationFailure.nonRetryable("Synthetic terminal failure"); + if (outcome === "CANCELLED") throw new CancelledFailure("Synthetic cancellation"); + return "recovered"; + } +} diff --git a/apps/worker/tests/worker-telemetry.test.ts b/apps/worker/tests/worker-telemetry.test.ts index 2f45d2910..9215c125c 100644 --- a/apps/worker/tests/worker-telemetry.test.ts +++ b/apps/worker/tests/worker-telemetry.test.ts @@ -254,6 +254,15 @@ it("links a real dot continue-as-new run to its previous run", async () => { expect(spans().filter(span => span.name === "activity.dotGoalStep")).toHaveLength(301); }, 30_000); +it.each(["COMPLETED", "FAILED", "CANCELLED"])("records %s when a workflow catches its continue-as-new attempt", async outcome => { + const handle = await start("caughtContinuationWorkflow", outcome); + if (outcome === "COMPLETED") expect(await handle.result()).toBe("recovered"); + else await expect(handle.result()).rejects.toThrow(); + await flush(); + const workflow = spans().find(span => span.name === "workflow.caughtContinuationWorkflow")!; + expect(attrs(workflow)["space.outcome"]).toBe(outcome); +}, 15_000); + it("links a background continuation to its supervising run and keeps its originating request trace", async () => { const childTrace = "1234567890abcdef1234567890abcdef"; const item = roomAgentSupervisorQueueItemSchema.parse({ missionId: "mission:synthetic", turn: { From 2ab60c095f16c11b936b7a5754978ad34bda4f60 Mon Sep 17 00:00:00 2001 From: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:18:54 -0400 Subject: [PATCH 7/7] Keep telemetry PR credential fixture identical to upstream Remove the one-line Mac wc-output normalization introduced while resolving the shared fixture conflict. This addresses the automated review's scope warning without changing telemetry code, test assertions or security rules. Validation: node --check and git diff --check pass. The file is byte-identical to upstream main f20c00a. Its 10 tests run on macOS with 9 passing and the unchanged upstream byte-count assertion failing because BSD wc pads 0 with spaces. This Linux-oriented fixture limitation is recorded, not suppressed. All earlier telemetry implementation checks and integration results remain applicable because no telemetry implementation or tests changed here. Signed-off-by: Marc Faustin <79724402+Xgoyeezy@users.noreply.github.com> --- scripts/tests/public-credential-install.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/public-credential-install.test.mjs b/scripts/tests/public-credential-install.test.mjs index 096ddb9b5..1837201dd 100644 --- a/scripts/tests/public-credential-install.test.mjs +++ b/scripts/tests/public-credential-install.test.mjs @@ -85,7 +85,7 @@ test('the credential agent runs only a wrapper\'s fixed credential actions, inpu // Short path: Unix socket paths are limited to about 100 bytes. const root = await mkdtemp('/tmp/spaceapp-agent-'); for (const name of ['codex-vscode-parity', 'opencode-vscode-parity']) { - await writeFile(join(root, name), '#!/bin/sh\nprintf "%s %s\\n" "$1" "$(( $(wc -c) ))"\n'); + await writeFile(join(root, name), '#!/bin/sh\nprintf "%s %s\\n" "$1" "$(wc -c)"\n'); await chmod(join(root, name), 0o755); } const socketPath = join(root, 'credentials.sock');