diff --git a/apps/api/src/dot-goals.ts b/apps/api/src/dot-goals.ts index 30dd69dee..5b2755f16 100644 --- a/apps/api/src/dot-goals.ts +++ b/apps/api/src/dot-goals.ts @@ -16,6 +16,7 @@ import { } from "@space/contracts"; import { SpaceConflictError, SpaceNotFoundError, type SpaceStore } from "@space/runtime"; import type { DotGoalsRepository, DotsRepository } from "@space/db"; +import { workflowCorrelationMemo } from "./trace-context.js"; /** * Space Dots Stage 4: Goals. @@ -26,7 +27,8 @@ import type { DotGoalsRepository, DotsRepository } from "@space/db"; * blocking question, cancellation, three consecutive failures or the budget. */ export interface DotGoalWorkflowCoordinator { - start(goalId: string): Promise; + /** `traceparent`: the starting request's W3C context, kept in the memo for worker spans. */ + start(goalId: string, traceparent?: string): Promise; wake(goalId: string): Promise; /** Starts the Goal's workflow when none is running; true when it had to start one. */ ensure?(goalId: string): Promise; @@ -38,11 +40,12 @@ export class TemporalDotGoalWorkflowCoordinator implements DotGoalWorkflowCoordi try { return await action(new Client({ connection, namespace: this.options.namespace })); } finally { await connection.close(); } } - start(goalId: string) { + start(goalId: string, traceparent?: string) { + const memo = traceparent ? workflowCorrelationMemo({ traceId: null, traceparent }) : {}; return this.client(async (client) => { try { await client.workflow.start("dotGoalWorkflow", { workflowId: dotGoalWorkflowId(goalId), taskQueue: this.options.taskQueue, - args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE" }); + args: [{ goalId }], workflowIdReusePolicy: "REJECT_DUPLICATE", ...(Object.keys(memo).length ? { memo } : {}) }); } catch (error) { if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; } }); } @@ -125,7 +128,8 @@ export function createDotGoalService(deps: DotGoalServiceDeps) { if ((error as { code?: string }).code === "DOT_GOAL_OPEN") throw new DotGoalError(409, "DOT_GOAL_OPEN", "This conversation already has an open Goal."); throw error; } - try { await deps.workflow.start(goal.goalId); } + const traceparent = request?.traceContext?.traceparent; + try { await (traceparent ? deps.workflow.start(goal.goalId, traceparent) : deps.workflow.start(goal.goalId)); } catch { goal = await finish(goal, "FAILED", "The Goal runtime could not start. Try again."); throw new DotGoalError(503, "DOT_GOAL_RUNTIME", "The Goal runtime could not start. Try again."); diff --git a/apps/api/src/memory-consolidation-coordinator.ts b/apps/api/src/memory-consolidation-coordinator.ts index df237c648..198528153 100644 --- a/apps/api/src/memory-consolidation-coordinator.ts +++ b/apps/api/src/memory-consolidation-coordinator.ts @@ -6,6 +6,14 @@ import { type MemoryConsolidationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; + +// Only when the starting request's trace context is known (an API request in +// flight); otherwise the start options are unchanged. +function correlationMemo(traceId: string): { memo?: Record } { + const traceparent = traceparentForTraceId(traceId); + return traceparent ? { memo: workflowCorrelationMemo({ traceId, traceparent }) } : {}; +} export { MEMORY_CONSOLIDATION_TASK_QUEUE, MEMORY_CONSOLIDATION_WORKFLOW_TYPE }; @@ -46,6 +54,8 @@ export async function scheduleMemoryConsolidation( try { const handle = await startWorkflow(MEMORY_CONSOLIDATION_WORKFLOW_TYPE, { args: [input], + // Correlation ids and the starting request's W3C context (worker spans). + ...correlationMemo(input.traceId), taskQueue: MEMORY_CONSOLIDATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/src/memory-mutation-coordinator.ts b/apps/api/src/memory-mutation-coordinator.ts index f7d3d48bb..0035850b8 100644 --- a/apps/api/src/memory-mutation-coordinator.ts +++ b/apps/api/src/memory-mutation-coordinator.ts @@ -7,6 +7,14 @@ import { type MemoryMutationWorkflowInput } from "@space/contracts"; import { SpaceFeatureDisabledError } from "@space/runtime"; +import { traceparentForTraceId, workflowCorrelationMemo } from "./trace-context.js"; + +// Only when the starting request's trace context is known (an API request in +// flight); otherwise the start options are unchanged. +function correlationMemo(traceId: string): { memo?: Record } { + const traceparent = traceparentForTraceId(traceId); + return traceparent ? { memo: workflowCorrelationMemo({ traceId, traceparent }) } : {}; +} export { MEMORY_MUTATION_TASK_QUEUE, MEMORY_MUTATION_WORKFLOW_TYPE, buildMemoryMutationWorkflowId }; @@ -48,6 +56,8 @@ export async function scheduleMemoryMutation( try { const handle = await startWorkflow(MEMORY_MUTATION_WORKFLOW_TYPE, { args: [input], + // Correlation ids and the starting request's W3C context (worker spans). + ...correlationMemo(input.traceId), taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowId, workflowIdReusePolicy: "REJECT_DUPLICATE", diff --git a/apps/api/src/trace-context.ts b/apps/api/src/trace-context.ts index af9e55efd..41134fe69 100644 --- a/apps/api/src/trace-context.ts +++ b/apps/api/src/trace-context.ts @@ -94,9 +94,15 @@ export class TraceContextRegistry { export const requestTraceContexts = new TraceContextRegistry(); -/** traceparent for work started on behalf of this Space request id, if known. */ +/** + * traceparent for work started on behalf of this Space request id, if known. + * Its parent id is the request's own SERVER span id (the span the API exports), + * so a workflow span that names it as parent joins the request's trace instead + * of pointing at a span that is never exported. + */ export function traceparentForTraceId(traceId: string | null | undefined, registry = requestTraceContexts): string | null { - return childTraceparent(registry.lookup(traceId)?.traceparent ?? null); + const traceparent = registry.lookup(traceId)?.traceparent ?? null; + return parseTraceparent(traceparent) ? traceparent : null; } /** Adds the request's traceparent to a turn input unless it already has a valid one. */ diff --git a/apps/api/tests/trace-context-workflow-parent.test.ts b/apps/api/tests/trace-context-workflow-parent.test.ts new file mode 100644 index 000000000..563481929 --- /dev/null +++ b/apps/api/tests/trace-context-workflow-parent.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { parseTraceparent } from "@space/contracts"; +import { createRequestTraceContext, requestTraceContexts, traceparentForTraceId, TraceContextRegistry } from "../src/trace-context.js"; +import { scheduleMemoryMutation } from "../src/memory-mutation-coordinator.js"; +import { scheduleMemoryConsolidation } from "../src/memory-consolidation-coordinator.js"; + +describe("trace context handed to Temporal workflows", () => { + it("names the request's exported SERVER span as the workflow's parent", () => { + const registry = new TraceContextRegistry(); + const context = createRequestTraceContext(undefined); + registry.remember("req:parent-span-01", context); + const traceparent = traceparentForTraceId("req:parent-span-01", registry); + expect(parseTraceparent(traceparent)).toEqual({ traceId: context.w3cTraceId, parentId: context.spanId, flags: "01" }); + expect(traceparentForTraceId("req:unknown-000001", registry)).toBeNull(); + }); + + it("adds the correlation memo to memory workflows started during a traced request", async () => { + const context = createRequestTraceContext(undefined); + requestTraceContexts.remember("req:memory-start-01", context); + const options: unknown[] = []; + const start = async (_type: string, option: { workflowId: string }) => { options.push(option); return { workflowId: option.workflowId, firstExecutionRunId: "run:1" }; }; + await scheduleMemoryMutation({ changeSetId: "memory_change:traced", traceId: "req:memory-start-01" }, start as never); + await scheduleMemoryConsolidation({ runId: "memory_run:traced", traceId: "req:memory-start-01" }, "space-memory-consolidation:traced", start as never); + for (const option of options) { + expect((option as { memo: Record }).memo).toEqual({ + spaceTraceId: "req:memory-start-01", traceparent: context.traceparent, w3cTraceId: context.w3cTraceId + }); + } + expect(options).toHaveLength(2); + }); +}); diff --git a/apps/worker/package.json b/apps/worker/package.json index ec58125fa..6345dd5f5 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -21,11 +21,13 @@ "@space/streaming": "0.2.5", "@temporalio/activity": "1.24.0", "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", "@temporalio/worker": "1.24.0", "@temporalio/workflow": "1.24.0", "undici": "8.12.3" }, "devDependencies": { + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.12" }, diff --git a/apps/worker/src/activity-observability.ts b/apps/worker/src/activity-observability.ts index ea4d021d1..dc15ae94f 100644 --- a/apps/worker/src/activity-observability.ts +++ b/apps/worker/src/activity-observability.ts @@ -11,6 +11,7 @@ import { type FailureCategory } from "@space/contracts"; import type { SpanExporter } from "@space/runtime"; +import { allowlistSpanAttributes, correlationAttributes, readTraceHeader } from "./trace-carrier.js"; import { logIdentifier, workerLog, type WorkerLogValue } from "./worker-log.js"; import { workerMetrics, type WorkerMetrics } from "./worker-metrics.js"; @@ -135,8 +136,15 @@ export interface ObserveActivityDependencies { metrics?: WorkerMetrics; now?: () => number; cancelled?: () => boolean; - /** OTLP export (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT): one span per activity whose input carries a traceparent. */ + /** + * OTLP export (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT): one INTERNAL span per + * activity attempt, child of the scheduling workflow span (the `space-trace` + * Temporal header), or of the input's traceparent for work scheduled by a + * workflow without tracing. + */ spans?: SpanExporter | null; + /** traceparent from the `space-trace` header; its parent id is the workflow span. */ + parentTraceparent?: string | null; } function activityTraceparent(args: readonly unknown[]) { @@ -194,19 +202,38 @@ export async function observeActivityExecution( queueMs, rssBytes: process.memoryUsage().rss }); - const parent = dependencies.spans ? activityTraceparent(args) : null; + const parent = dependencies.spans + ? parseTraceparent(dependencies.parentTraceparent) ?? activityTraceparent(args) + : null; if (parent) { - dependencies.spans!.record({ - traceId: parent.traceId, - spanId: randomBytes(8).toString("hex"), - parentSpanId: parent.parentId, - name: `activity ${activityType}`, - startTimeUnixNano: BigInt(startedAt) * 1_000_000n, - endTimeUnixNano: BigInt(startedAt + durationMs) * 1_000_000n, - attributes: { "space.activity_type": activityType, "space.outcome": outcome, "space.attempt": info.attempt }, - error: outcome === "FAILED" || outcome === "FAILED_RESULT", - kind: 1 - }); + try { + dependencies.spans!.record({ + traceId: parent.traceId, + spanId: randomBytes(8).toString("hex"), + parentSpanId: parent.parentId, + name: `activity ${activityType}`, + startTimeUnixNano: BigInt(startedAt) * 1_000_000n, + endTimeUnixNano: BigInt(startedAt + durationMs) * 1_000_000n, + attributes: allowlistSpanAttributes({ + ...correlationAttributes(args), + "space.activity_type": activityType, + "space.activity_id": info.activityId, + "space.workflow_type": info.workflowType, + "space.workflow_id": info.workflowExecution?.workflowId, + "space.run_id": info.workflowExecution?.runId, + "space.attempt": info.attempt, + "space.outcome": outcome, + "space.reason_code": reasonCode, + "space.category": category, + "space.duration_ms": durationMs, + "space.queue_ms": queueMs + }), + error: outcome === "FAILED" || outcome === "FAILED_RESULT", + kind: 1 + }); + } catch { + // Span export is best effort and never affects the activity. + } } }; @@ -229,6 +256,7 @@ export function createActivityObservabilityInterceptors(dependencies: Omit observeActivityExecution(context.info, input.args, () => next(input), { ...dependencies, + ...(dependencies.spans ? { parentTraceparent: readTraceHeader(input.headers)?.traceparent ?? null } : {}), cancelled: () => context.cancellationSignal.aborted }) } diff --git a/apps/worker/src/memory-maintenance-worker.ts b/apps/worker/src/memory-maintenance-worker.ts index e12046033..71e91bbbb 100644 --- a/apps/worker/src/memory-maintenance-worker.ts +++ b/apps/worker/src/memory-maintenance-worker.ts @@ -8,7 +8,7 @@ import { } from "./memory-maintenance-config.js"; import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js"; import { otlpSpanExporterFromEnv } from "@space/runtime"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { workerTracingOptions } from "./worker-tracing.js"; import { reloadWhenServicesCommitChanges } from "./deployed-code-reload.js"; const service = "space-memory-maintenance-worker"; @@ -30,7 +30,7 @@ if (!memoryMaintenanceEnabled(process.env)) { workflowsPath, activities, ...memoryMaintenanceWorkerOptions, - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + ...workerTracingOptions(spans) }); console.log(JSON.stringify({ service, diff --git a/apps/worker/src/memory-mutation-worker.ts b/apps/worker/src/memory-mutation-worker.ts index c2eadf657..29d45b010 100644 --- a/apps/worker/src/memory-mutation-worker.ts +++ b/apps/worker/src/memory-mutation-worker.ts @@ -4,7 +4,7 @@ import { executePersistedMemoryChangeSet } from "./memory-mutation-activity.js"; import { MEMORY_MUTATION_TASK_QUEUE, memoryMutationsEnabled, memoryMutationWorkerOptions } from "./memory-mutation-config.js"; import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js"; import { otlpSpanExporterFromEnv } from "@space/runtime"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { workerTracingOptions } from "./worker-tracing.js"; import { reloadWhenServicesCommitChanges } from "./deployed-code-reload.js"; const service = "space-memory-mutation-worker"; @@ -23,7 +23,7 @@ if (!memoryMutationsEnabled(process.env)) { workflowsPath: fileURLToPath(new URL("./memory-mutation-workflows.js", import.meta.url)), activities: { executePersistedMemoryChangeSet }, ...memoryMutationWorkerOptions, - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + ...workerTracingOptions(spans) }); console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE })); const stopReloadWatch = reloadWhenServicesCommitChanges(worker, { log: (message) => console.log(JSON.stringify({ service, status: "reloading", message })) }); diff --git a/apps/worker/src/room-background-continuation.ts b/apps/worker/src/room-background-continuation.ts index e15c29a21..de690e459 100644 --- a/apps/worker/src/room-background-continuation.ts +++ b/apps/worker/src/room-background-continuation.ts @@ -122,10 +122,18 @@ maxMissionTokens: number | null = DEFAULT_ROOM_AGENT_MAX_MISSION_TOKENS): Promis ].join("\n"); const prefix = `${instructions}\n\nOperator request:\n${request}\n\nSettled action results (bounded; inspect for details):\n`; const prompt = prefix + continuationEvidence(actions, Math.max(0, 8000 - prefix.length)); + // A continuation is a new trace (trace:bg-) with an OTLP span link to the + // operator request it continues; it does not reuse that request's + // traceparent as its parent. Later generations keep linking to the same request. + const { traceparent: sourceRequestTraceparent, ...sourceTurn } = source.queueItem.turn; + const sourceTraceparent = sourceRequestTraceparent ?? source.queueItem.turn.roomAgentContinuation?.sourceTraceparent; const item = roomAgentSupervisorQueueItemSchema.parse({ missionId: input.missionId, turn: { - ...source.queueItem.turn, + ...sourceTurn, prompt, roomAgentOperatorRequest: request, - roomAgentContinuation: { sourceRunId: source.run.runId, generation }, + roomAgentContinuation: { + sourceRunId: source.run.runId, generation, + ...(sourceTraceparent ? { sourceTraceparent } : {}) + }, artifactIds: [], agentRunId: runId, agentUserMessageId: `agent_msg:bg-context-${input.continuationId}`, agentAssistantMessageId: `agent_msg:bg-response-${input.continuationId}`, diff --git a/apps/worker/src/trace-carrier.ts b/apps/worker/src/trace-carrier.ts new file mode 100644 index 000000000..1abb83f5a --- /dev/null +++ b/apps/worker/src/trace-carrier.ts @@ -0,0 +1,255 @@ +/** + * Trace context that travels between Temporal workflows and activities in a + * Temporal header (never in workflow or activity arguments), and the span + * attribute allowlist shared by workflow and activity spans. + * + * This module is pure: it is imported both by the workflow sandbox + * (workflow-tracing.ts) and by the worker process. It performs no I/O, reads no + * clock and uses no random source, so it is safe during workflow replay. + */ +import { defaultPayloadConverter, type Payload } from "@temporalio/common"; +import { formatTraceparent, isSafeErrorCode, parseTraceparent, type ParsedTraceparent } from "@space/contracts"; + +/** Temporal header that carries {@link TraceCarrier}. */ +export const SPACE_TRACE_HEADER = "space-trace"; + +export type TraceLinkType = "continue_as_new" | "background_continuation" | "parent_workflow"; + +export interface TraceLink { + traceparent: string; + type: TraceLinkType; + /** Temporal run id of the run that is continued, when known. */ + runId?: string; +} + +export interface TraceCarrier { + /** W3C traceparent whose parent id is the span the receiver must be a child of. */ + traceparent: string | null; + /** Links the receiver attaches to its span. */ + links?: TraceLink[]; +} + +const MAX_LINKS = 8; +const runIdPattern = /^[A-Za-z0-9._:-]{1,200}$/; +const linkTypes: ReadonlySet = new Set(["continue_as_new", "background_continuation", "parent_workflow"]); + +type Headers = Record; + +/** Returns headers with the carrier set; other headers are kept. */ +export function withTraceHeader(headers: Headers, carrier: TraceCarrier): Headers { + return { ...headers, [SPACE_TRACE_HEADER]: defaultPayloadConverter.toPayload(carrier) }; +} + +/** Reads and validates the carrier; anything malformed is ignored (null). */ +export function readTraceHeader(headers: Headers | undefined | null): TraceCarrier | null { + const payload = headers?.[SPACE_TRACE_HEADER]; + if (!payload) return null; + let value: unknown; + try { + value = defaultPayloadConverter.fromPayload(payload); + } catch { + return null; + } + if (!value || typeof value !== "object") return null; + const record = value as { traceparent?: unknown; links?: unknown }; + const parent = parseTraceparent(record.traceparent); + const links: TraceLink[] = []; + if (Array.isArray(record.links)) { + for (const raw of record.links.slice(0, MAX_LINKS)) { + const link = raw as { traceparent?: unknown; type?: unknown; runId?: unknown } | null; + const parsed = parseTraceparent(link?.traceparent); + if (!parsed || typeof link?.type !== "string" || !linkTypes.has(link.type)) continue; + links.push({ + traceparent: formatTraceparent(parsed.traceId, parsed.parentId, parsed.flags), + type: link.type as TraceLinkType, + ...(typeof link.runId === "string" && runIdPattern.test(link.runId) ? { runId: link.runId } : {}) + }); + } + } + return { + traceparent: parent ? formatTraceparent(parent.traceId, parent.parentId, parent.flags) : null, + ...(links.length ? { links } : {}) + }; +} + +// --------------------------------------------------------------------------- +// Deterministic ids. Workflow span ids are derived from Temporal ids, so the +// same run always gets the same span id: a replayed or retried workflow task +// produces identical ids (and the worker drops the duplicate), and the +// workflow's seeded random source is never consumed, so existing histories +// replay unchanged. + +const FNV_OFFSET = 0xcbf29ce484222325n; +const FNV_PRIME = 0x100000001b3n; +const MASK_64 = 0xffffffffffffffffn; + +function fnv1a64(seed: string): bigint { + let hash = FNV_OFFSET; + for (let index = 0; index < seed.length; index += 1) { + hash ^= BigInt(seed.charCodeAt(index)); + hash = (hash * FNV_PRIME) & MASK_64; + } + return hash; +} + +function hex64(seed: string): string { + const value = fnv1a64(seed).toString(16).padStart(16, "0"); + // All-zero ids are invalid in W3C Trace Context. + return /^0+$/.test(value) ? "0000000000000001" : value; +} + +/** 16 lower-case hex characters derived from the seed. */ +export function deterministicSpanId(seed: string): string { + return hex64(`span:${seed}`); +} + +/** 32 lower-case hex characters derived from the seed. */ +export function deterministicTraceId(seed: string): string { + return `${hex64(`trace-hi:${seed}`)}${hex64(`trace-lo:${seed}`)}`; +} + +// --------------------------------------------------------------------------- +// Attribute allowlist. Only these keys are ever exported from the workers, and +// only with safe values: identifiers (no spaces, slashes or other characters +// that prompts, URLs, file paths or error messages contain), finite integers, +// and enumerated codes. Everything else is dropped. + +const identifierPattern = /^[A-Za-z0-9:._-]{1,200}$/; +const typeNamePattern = /^[A-Za-z][A-Za-z0-9_]{0,119}$/; + +type AttributeKind = "id" | "type" | "count" | "code" | "enum"; + +export const SPAN_ATTRIBUTE_ALLOWLIST: Readonly> = { + "space.trace_id": "id", + "space.parent_trace_id": "id", + "space.workflow_id": "id", + "space.run_id": "id", + "space.parent_workflow_id": "id", + "space.continued_from_run_id": "id", + "space.room_id": "id", + "space.pane_id": "id", + "space.mission_id": "id", + "space.agent_run_id": "id", + "space.action_id": "id", + "space.goal_id": "id", + "space.activity_id": "id", + "space.workflow_type": "type", + "space.activity_type": "type", + "space.attempt": "count", + "space.duration_ms": "count", + "space.queue_ms": "count", + "space.continuation_generation": "count", + "space.outcome": "enum", + "space.reason_code": "code", + "space.category": "code", + "space.link_type": "enum" +}; + +const enumValues: Readonly>> = { + "space.outcome": new Set(["COMPLETED", "FAILED_RESULT", "FAILED", "CANCELLED", "CONTINUED_AS_NEW"]), + "space.link_type": linkTypes +}; + +function safeValue(key: string, kind: AttributeKind, value: unknown): string | number | null { + switch (kind) { + case "id": + return typeof value === "string" && identifierPattern.test(value) ? value : null; + case "type": + return typeof value === "string" && typeNamePattern.test(value) ? value : null; + case "count": + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null; + case "code": + return isSafeErrorCode(value) ? value : null; + case "enum": + return typeof value === "string" && enumValues[key]?.has(value) ? value : null; + } +} + +/** Keeps allowlisted keys with safe values; drops everything else. */ +export function allowlistSpanAttributes(attributes: Readonly>): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(attributes)) { + const kind = SPAN_ATTRIBUTE_ALLOWLIST[key]; + if (!kind || value === null || value === undefined) continue; + const safe = safeValue(key, kind, value); + if (safe !== null) result[key] = safe; + } + return result; +} + +// --------------------------------------------------------------------------- +// Correlation ids from workflow arguments (ids only; prompts and content are +// never read). Mirrors activityCorrelation in activity-observability.ts. + +const correlationKeys = { + traceId: "space.trace_id", + requestId: "space.parent_trace_id", + roomId: "space.room_id", + paneId: "space.pane_id", + missionId: "space.mission_id", + roomAgentMissionId: "space.mission_id", + agentRunId: "space.agent_run_id", + actionId: "space.action_id", + goalId: "space.goal_id" +} as const; +const nestedKeys = ["turn", "item", "bridge", "input", "turnInput"] as const; + +function collect(value: unknown, into: Record, depth: number): void { + if (!value || typeof value !== "object" || Array.isArray(value)) return; + const record = value as Record; + for (const [key, attribute] of Object.entries(correlationKeys)) { + if (into[attribute] === undefined && typeof record[key] === "string" && identifierPattern.test(record[key] as string)) { + into[attribute] = record[key]; + } + } + if (depth <= 0) return; + for (const key of nestedKeys) collect(record[key], into, depth - 1); +} + +/** Allowlisted correlation attributes found in workflow or activity arguments. */ +export function correlationAttributes(args: readonly unknown[]): Record { + const found: Record = {}; + for (const arg of args) collect(arg, found, 2); + if (found["space.parent_trace_id"] === found["space.trace_id"]) delete found["space.parent_trace_id"]; + return allowlistSpanAttributes(found); +} + +function findTraceparent(value: unknown, depth: number): ParsedTraceparent | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + const own = parseTraceparent(record.traceparent); + if (own) return own; + if (depth <= 0) return null; + for (const key of nestedKeys) { + const nested = findTraceparent(record[key], depth - 1); + if (nested) return nested; + } + return null; +} + +/** The originating request's traceparent carried in arguments (for example DummyTurnInput.traceparent). */ +export function traceparentFromArgs(args: readonly unknown[]): ParsedTraceparent | null { + for (const arg of args) { + const found = findTraceparent(arg, 2); + if (found) return found; + } + return null; +} + +/** The background continuation a turn continues, from roomAgentContinuation in its arguments. */ +export function continuationFromArgs(args: readonly unknown[]): { traceparent: string | null; sourceRunId: string | null; generation: number | null } | null { + for (const arg of args) { + for (const candidate of [arg, (arg as { turn?: unknown } | null)?.turn]) { + const continuation = (candidate as { roomAgentContinuation?: unknown } | null)?.roomAgentContinuation as + { sourceTraceparent?: unknown; sourceRunId?: unknown; generation?: unknown } | undefined; + if (!continuation || typeof continuation !== "object") continue; + const parsed = parseTraceparent(continuation.sourceTraceparent); + return { + traceparent: parsed ? formatTraceparent(parsed.traceId, parsed.parentId, parsed.flags) : null, + sourceRunId: typeof continuation.sourceRunId === "string" && identifierPattern.test(continuation.sourceRunId) ? continuation.sourceRunId : null, + generation: typeof continuation.generation === "number" && Number.isSafeInteger(continuation.generation) ? continuation.generation : null + }; + } + } + return null; +} diff --git a/apps/worker/src/worker-tracing.ts b/apps/worker/src/worker-tracing.ts new file mode 100644 index 000000000..b415e6f72 --- /dev/null +++ b/apps/worker/src/worker-tracing.ts @@ -0,0 +1,84 @@ +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import type { InjectedSinks, WorkerOptions } from "@temporalio/worker"; +import { parseTraceparent } from "@space/contracts"; +import type { ServerSpan, SpanExporter } from "@space/runtime"; +import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { allowlistSpanAttributes } from "./trace-carrier.js"; +import type { SpaceTracingSinks, WorkflowSpanRecord } from "./workflow-tracing.js"; + +/** + * Worker-process half of the OTLP tracing (see workflow-tracing.ts). One call + * gives every Temporal worker entrypoint the same interceptors and sinks: + * with `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` unset (`spans` null) only the + * existing activity observability interceptor is installed and no workflow + * interceptor or sink is registered. + */ + +/** The workflow interceptor module; the TypeScript source when there is no build (tests). */ +export function workflowTracingModulePath(): string { + const built = fileURLToPath(new URL("./workflow-tracing.js", import.meta.url)); + if (existsSync(built)) return built; + return built.replace(/\.js$/, ".ts"); +} + +/** Converts a sink record into an exporter span, keeping only allowlisted attributes. */ +export function workflowSpanToServerSpan(record: WorkflowSpanRecord): ServerSpan { + return { + traceId: record.traceId, + spanId: record.spanId, + parentSpanId: record.parentSpanId, + name: record.name, + startTimeUnixNano: BigInt(Math.trunc(record.startTimeMs)) * 1_000_000n, + endTimeUnixNano: BigInt(Math.trunc(Math.max(record.startTimeMs, record.endTimeMs))) * 1_000_000n, + attributes: allowlistSpanAttributes(record.attributes), + error: record.error, + kind: 1, + links: record.links.flatMap((link) => { + const parsed = parseTraceparent(link.traceparent); + return parsed ? [{ + traceId: parsed.traceId, + spanId: parsed.parentId, + attributes: allowlistSpanAttributes({ "space.link_type": link.type, "space.run_id": link.runId }) + }] : []; + }) + }; +} + +/** + * Sinks for the workflow spans. `callDuringReplay: false`: replaying a history + * never emits a span. A failed or timed-out workflow task can call a sink again + * when the task is retried; span ids are deterministic, so the repeat is + * dropped here. Recording never throws into the workflow. + */ +export function createWorkflowTracingSinks(spans: SpanExporter, recentLimit = 4096): InjectedSinks { + const recent = new Set(); + return { + spaceTracing: { + recordSpan: { + fn(_info, record) { + try { + const key = `${record.traceId}:${record.spanId}`; + if (recent.has(key)) return; + recent.add(key); + if (recent.size > recentLimit) recent.delete(recent.values().next().value!); + spans.record(workflowSpanToServerSpan(record)); + } catch { + // Tracing is best effort and must never affect workflow work. + } + }, + callDuringReplay: false + } + } + }; +} + +/** Interceptors and sinks for Worker.create, the same for all three worker processes. */ +export function workerTracingOptions(spans: SpanExporter | null): Pick { + const activity = [createActivityObservabilityInterceptors({ spans })]; + if (!spans) return { interceptors: { activity } }; + return { + interceptors: { activity, workflowModules: [workflowTracingModulePath()] }, + sinks: createWorkflowTracingSinks(spans) + }; +} diff --git a/apps/worker/src/worker.ts b/apps/worker/src/worker.ts index 83d23cbc6..788031869 100644 --- a/apps/worker/src/worker.ts +++ b/apps/worker/src/worker.ts @@ -6,7 +6,7 @@ import { StreamingCredentialStore } from "@space/streaming"; import { TelegramDeliveryWorker, TelegramSecretStore, otlpSpanExporterFromEnv } from "@space/runtime"; import { NativeConnection, Runtime, Worker } from "@temporalio/worker"; import * as activities from "./activities.js"; -import { createActivityObservabilityInterceptors } from "./activity-observability.js"; +import { workerTracingOptions } from "./worker-tracing.js"; import { resolveTemporalCoreMetricsBind, resolveWorkerMetricsListen, startWorkerMetricsServer } from "./worker-metrics.js"; import { workerLog } from "./worker-log.js"; import { sweepCodexCliCompletions } from "./codex-cli-completion.js"; @@ -43,9 +43,10 @@ const worker = await Worker.create({ taskQueue, workflowsPath, activities, - // Structured, correlated activity records and worker metrics. Activity-side - // only: the workflow sandbox and its replay behaviour are unchanged. - interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] } + // Structured, correlated activity records and worker metrics, plus OTLP + // workflow and activity-attempt spans when SPACE_OTEL_EXPORTER_OTLP_ENDPOINT + // is set (replay-safe: spans leave the workflow sandbox only through sinks). + ...workerTracingOptions(spans) }); // Optional loopback-only Prometheus endpoint for worker metrics. diff --git a/apps/worker/src/workflow-tracing.ts b/apps/worker/src/workflow-tracing.ts new file mode 100644 index 000000000..e6060f8e6 --- /dev/null +++ b/apps/worker/src/workflow-tracing.ts @@ -0,0 +1,285 @@ +/** + * Workflow-side OTLP tracing (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT). Registered as + * a Temporal workflow interceptor module only when export is enabled + * (worker-tracing.ts). + * + * Determinism: this code runs inside the workflow sandbox and does no I/O. Span + * ids are derived from Temporal ids (never from the workflow's seeded random + * source), times come from the workflow clock, and finished spans leave the + * sandbox only through the `spaceTracing` sink, which the worker registers with + * `callDuringReplay: false`. Replaying a history therefore emits nothing, and + * the trace context travels in Temporal headers, which are not part of the + * command sequence that replay checks. + * + * Tree: the API SERVER span (traceparent from the parent header, the workflow + * input or the memo) -> workflow span -> one span per activity attempt (emitted + * by the activity interceptor) and child workflow spans. Continue-as-new runs + * and background continuations carry an OTLP span link to what they continue. + */ +import { + ContinueAsNew, + isCancellation, + proxySinks, + workflowInfo, + type Sinks, + type WorkflowInterceptorsFactory +} from "@temporalio/workflow"; +import { formatTraceparent, isSafeErrorCode, parseTraceparent, type ParsedTraceparent } from "@space/contracts"; +import { + allowlistSpanAttributes, + continuationFromArgs, + correlationAttributes, + deterministicSpanId, + deterministicTraceId, + readTraceHeader, + traceparentFromArgs, + withTraceHeader, + type TraceCarrier, + type TraceLink +} from "./trace-carrier.js"; + +export type WorkflowSpanOutcome = "COMPLETED" | "FAILED_RESULT" | "FAILED" | "CANCELLED" | "CONTINUED_AS_NEW"; + +/** A finished workflow (or workflow scope) span, as handed to the sink. */ +export interface WorkflowSpanRecord { + traceId: string; + spanId: string; + parentSpanId: string | null; + name: string; + startTimeMs: number; + endTimeMs: number; + error: boolean; + attributes: Record; + links: TraceLink[]; +} + +export interface SpaceTracingSinks extends Sinks { + spaceTracing: { recordSpan(span: WorkflowSpanRecord): void }; +} + +interface TraceScope { + traceId: string; + spanId: string; + flags: string; +} + +interface WorkflowTraceState { + run: TraceScope | null; + /** The run's own parent, carried to a continue-as-new run. */ + runParent: ParsedTraceparent | null; + current: TraceScope | null; + scopes: number; +} + +// Kept on the workflow's global object: Temporal isolates globals per workflow +// execution, also when V8 contexts are reused. +const STATE_KEY = "__spaceWorkflowTraceState"; + +function traceState(): WorkflowTraceState | null { + return ((globalThis as Record)[STATE_KEY] as WorkflowTraceState | undefined) ?? null; +} + +function scopeTraceparent(scope: TraceScope): string { + return formatTraceparent(scope.traceId, scope.spanId, scope.flags); +} + +function failureCode(error: unknown): string | null { + let failure = error as { type?: unknown; name?: unknown; cause?: unknown } | null | undefined; + let code: string | null = null; + for (let depth = 0; failure && depth < 8; depth += 1, failure = failure.cause as typeof failure) { + if (isSafeErrorCode(failure.type)) code = failure.type; + else if (!code && failure.name === "TimeoutFailure") code = "WORKFLOW_ACTIVITY_TIMEOUT"; + } + return code; +} + +function outcomeOf(error: unknown): WorkflowSpanOutcome { + if (error instanceof ContinueAsNew) return "CONTINUED_AS_NEW"; + return isCancellation(error) ? "CANCELLED" : "FAILED"; +} + +function resultOutcome(result: unknown): WorkflowSpanOutcome { + const status = (result as { status?: unknown } | null)?.status; + if (status === "FAILED") return "FAILED_RESULT"; + return "COMPLETED"; +} + +function dedupeLinks(links: TraceLink[]): TraceLink[] { + const seen = new Set(); + return links.filter((link) => !seen.has(link.traceparent) && Boolean(seen.add(link.traceparent))).slice(0, 8); +} + +export const interceptors: WorkflowInterceptorsFactory = () => { + const sinks = proxySinks(); + const state: WorkflowTraceState = { run: null, runParent: null, current: null, scopes: 0 }; + (globalThis as Record)[STATE_KEY] = state; + + const carrierForChild = (args: readonly unknown[]): TraceCarrier => { + const scope = state.current!; + const own = traceparentFromArgs(args); + // A turn queued by a later API request carries that request's context; + // parent it there and link it to the workflow that started it. + if (own && own.traceId !== scope.traceId) { + return { + traceparent: formatTraceparent(own.traceId, own.parentId, own.flags), + links: [{ traceparent: scopeTraceparent(scope), type: "parent_workflow", runId: workflowInfo().runId }] + }; + } + return { traceparent: scopeTraceparent(scope) }; + }; + + return { + inbound: [{ + async execute(input, next) { + const info = workflowInfo(); + const header = readTraceHeader(input.headers); + const parent = header + ? parseTraceparent(header.traceparent) + : traceparentFromArgs(input.args) ?? parseTraceparent(info.memo?.traceparent); + const continuation = continuationFromArgs(input.args); + const links = dedupeLinks([ + ...(header?.links ?? []), + ...(continuation?.traceparent ? [{ + traceparent: continuation.traceparent, + type: "background_continuation" as const + }] : []) + ]); + const run: TraceScope = { + traceId: parent?.traceId ?? deterministicTraceId(`${info.workflowId}/${info.runId}`), + spanId: deterministicSpanId(`workflow/${info.runId}`), + flags: parent?.flags ?? "01" + }; + state.run = run; + state.runParent = parent; + state.current = run; + const startTimeMs = info.runStartTime.getTime(); + let outcome: WorkflowSpanOutcome = "COMPLETED"; + let reasonCode: string | null = null; + try { + const result = await next(input); + outcome = resultOutcome(result); + return result; + } catch (error) { + outcome = outcomeOf(error); + if (outcome === "FAILED") reasonCode = failureCode(error) ?? "WORKFLOW_FAILED"; + throw error; + } finally { + const continuedFrom = links.find((link) => link.type === "continue_as_new")?.runId; + sinks.spaceTracing.recordSpan({ + traceId: run.traceId, + spanId: run.spanId, + parentSpanId: parent?.parentId ?? null, + name: `workflow ${info.workflowType}`, + startTimeMs, + endTimeMs: Math.max(startTimeMs, Date.now()), + error: outcome === "FAILED" || outcome === "FAILED_RESULT", + attributes: allowlistSpanAttributes({ + ...correlationAttributes(input.args), + "space.workflow_type": info.workflowType, + "space.workflow_id": info.workflowId, + "space.run_id": info.runId, + "space.parent_workflow_id": info.parent?.workflowId, + "space.continued_from_run_id": continuedFrom, + "space.continuation_generation": continuation?.generation ?? undefined, + "space.attempt": info.attempt, + "space.outcome": outcome, + "space.reason_code": reasonCode, + "space.duration_ms": Math.max(0, Date.now() - startTimeMs) + }), + links + }); + } + } + }], + outbound: [{ + scheduleActivity(input, next) { + return next(state.current ? { ...input, headers: withTraceHeader(input.headers, { traceparent: scopeTraceparent(state.current) }) } : input); + }, + scheduleLocalActivity(input, next) { + return next(state.current ? { ...input, headers: withTraceHeader(input.headers, { traceparent: scopeTraceparent(state.current) }) } : input); + }, + startChildWorkflowExecution(input, next) { + return next(state.current ? { ...input, headers: withTraceHeader(input.headers, carrierForChild(input.options.args ?? [])) } : input); + }, + continueAsNew(input, next) { + if (!state.run) return next(input); + // The next run stays under this run's parent (same trace) and links to this run. + return next({ ...input, headers: withTraceHeader(input.headers, { + traceparent: state.runParent ? formatTraceparent(state.runParent.traceId, state.runParent.parentId, state.runParent.flags) : null, + links: [{ traceparent: scopeTraceparent(state.run), type: "continue_as_new", runId: workflowInfo().runId }] + }) }); + } + }] + }; +}; + +export interface WorkflowScopeOrigin { + /** Span name suffix, for example "mission". */ + name: string; + /** The originating request's traceparent; the scope starts a new trace without one. */ + traceparent?: string | null; + /** Arguments to read allowlisted correlation ids and continuation links from. */ + args: readonly unknown[]; +} + +/** + * Runs `execute` in its own span inside a long-lived workflow (one Room Agent + * supervisor mission turn), so the turn's activities and child workflow join + * the trace of the API request that queued it instead of the supervisor run. + * The scope span links to the supervisor run. Without tracing installed this + * only calls `execute`. + */ +export async function traceWorkflowScope(origin: WorkflowScopeOrigin, execute: () => Promise): Promise { + const state = traceState(); + if (!state?.run) return execute(); + const info = workflowInfo(); + const index = state.scopes++; + const parent = parseTraceparent(origin.traceparent); + const continuation = continuationFromArgs(origin.args); + const scope: TraceScope = { + traceId: parent?.traceId ?? deterministicTraceId(`${info.workflowId}/${info.runId}/scope/${index}`), + spanId: deterministicSpanId(`workflow/${info.runId}/scope/${index}`), + flags: parent?.flags ?? state.run.flags + }; + const links = dedupeLinks([ + { traceparent: scopeTraceparent(state.run), type: "parent_workflow", runId: info.runId }, + ...(continuation?.traceparent ? [{ traceparent: continuation.traceparent, type: "background_continuation" as const }] : []) + ]); + const sinks = proxySinks(); + const previous = state.current; + state.current = scope; + const startTimeMs = Date.now(); + let outcome: WorkflowSpanOutcome = "COMPLETED"; + let reasonCode: string | null = null; + try { + const result = await execute(); + outcome = resultOutcome(result); + return result; + } catch (error) { + outcome = outcomeOf(error); + if (outcome === "FAILED") reasonCode = failureCode(error) ?? "WORKFLOW_FAILED"; + throw error; + } finally { + state.current = previous; + sinks.spaceTracing.recordSpan({ + traceId: scope.traceId, + spanId: scope.spanId, + parentSpanId: parent?.parentId ?? null, + name: `workflow ${info.workflowType} ${origin.name}`, + startTimeMs, + endTimeMs: Math.max(startTimeMs, Date.now()), + error: outcome === "FAILED" || outcome === "FAILED_RESULT", + attributes: allowlistSpanAttributes({ + ...correlationAttributes(origin.args), + "space.workflow_type": info.workflowType, + "space.workflow_id": info.workflowId, + "space.run_id": info.runId, + "space.continuation_generation": continuation?.generation ?? undefined, + "space.outcome": outcome, + "space.reason_code": reasonCode, + "space.duration_ms": Math.max(0, Date.now() - startTimeMs) + }), + links + }); + } +} diff --git a/apps/worker/src/workflows.ts b/apps/worker/src/workflows.ts index 5f3a7d4cd..1ea445c36 100644 --- a/apps/worker/src/workflows.ts +++ b/apps/worker/src/workflows.ts @@ -39,6 +39,7 @@ import { } from "./room-supervisor-state.js"; import { isCliChatTurnProviderId, isNativeChatTurn, NATIVE_CHAT_TURN_ACTIVITY_TIMEOUT } from "./turn-runtime-policy.js"; import { runDotGoalLoop } from "./dot-goal-loop.js"; +import { traceWorkflowScope } from "./workflow-tracing.js"; const { recordDummyTurnStarted, recordDummyTurnCompleted } = proxyActivities({ startToCloseTimeout: "1 minute", @@ -317,39 +318,43 @@ async function continuingRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervis scope = new CancellationScope({ cancellable: true }); stopReason = "Stopped by operator."; let paused = false; - try { - await scope.run(async () => { - const state = await supervisorActivities.checkRoomAgentSupervisorTurn(item); - if (state === "PAUSED") { paused = true; return; } - if (state === "SKIP") { - await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason: "Turn is no longer active or was superseded." }); - return; - } - const result = await executeChild(codexAppServerTurnWorkflow, { - args: [item.turn], workflowId: buildCodexAppServerTurnWorkflowId(item.turn), - cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED + // One span per mission turn in the trace of the request that queued it + // (a no-op without OTLP export; it does no I/O and issues no commands). + await traceWorkflowScope({ name: "mission", traceparent: item.turn.traceparent, args: [item] }, async () => { + try { + await scope!.run(async () => { + const state = await supervisorActivities.checkRoomAgentSupervisorTurn(item); + if (state === "PAUSED") { paused = true; return; } + if (state === "SKIP") { + await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason: "Turn is no longer active or was superseded." }); + return; + } + const result = await executeChild(codexAppServerTurnWorkflow, { + args: [item.turn], workflowId: buildCodexAppServerTurnWorkflowId(item.turn), + cancellationType: ChildWorkflowCancellationType.WAIT_CANCELLATION_COMPLETED + }); + const completion = roomAgentMissionCompletion(result); + if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { + monitoring.add(item.missionId); + } else if (completion.status === "COMPLETED" && monitoring.has(item.missionId) && !item.turn.roomAgentContinuation) { + // A FIND/layout follow-up can finish while earlier pane tasks continue. + } else if (!roomAgentMissionHasContinuation(pending, item.missionId)) { + monitoring.delete(item.missionId); + await supervisorActivities.finishRoomAgentSupervisorTurn({ item, ...completion }); + } }); - const completion = roomAgentMissionCompletion(result); - if (result.roomAgentOutcome?.status === "VERIFIED" && result.roomAgentOutcome.pendingActionIds?.length) { - monitoring.add(item.missionId); - } else if (completion.status === "COMPLETED" && monitoring.has(item.missionId) && !item.turn.roomAgentContinuation) { - // A FIND/layout follow-up can finish while earlier pane tasks continue. - } else if (!roomAgentMissionHasContinuation(pending, item.missionId)) { - monitoring.delete(item.missionId); - await supervisorActivities.finishRoomAgentSupervisorTurn({ item, ...completion }); - } - }); - } catch (error) { - await CancellationScope.nonCancellable(async () => { - const reason = isCancellation(error) ? stopReason : "Room Agent supervisor turn failed."; - const errorCode = isCancellation(error) ? undefined : supervisorFailureCode(error); - await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason, errorCode }); - if (!roomAgentMissionHasContinuation(pending, item.missionId)) { - monitoring.delete(item.missionId); - await supervisorActivities.finishRoomAgentSupervisorTurn({ item, status: errorCode ? "FAILED" : "INTERRUPTED", statusReason: reason, errorCode }); - } - }); - } finally { active = null; scope = null; processedCount++; } + } catch (error) { + await CancellationScope.nonCancellable(async () => { + const reason = isCancellation(error) ? stopReason : "Room Agent supervisor turn failed."; + const errorCode = isCancellation(error) ? undefined : supervisorFailureCode(error); + await supervisorActivities.discardRoomAgentSupervisorTurn({ item, reason, errorCode }); + if (!roomAgentMissionHasContinuation(pending, item.missionId)) { + monitoring.delete(item.missionId); + await supervisorActivities.finishRoomAgentSupervisorTurn({ item, status: errorCode ? "FAILED" : "INTERRUPTED", statusReason: reason, errorCode }); + } + }); + } finally { active = null; scope = null; processedCount++; } + }); if (paused) { pending.push(item); const revision = signalRevision; diff --git a/apps/worker/tests/activity-observability.test.ts b/apps/worker/tests/activity-observability.test.ts index 654bf7962..bbe7d07ac 100644 --- a/apps/worker/tests/activity-observability.test.ts +++ b/apps/worker/tests/activity-observability.test.ts @@ -182,11 +182,15 @@ describe("worker metrics endpoint", () => { }); }); -it("registers the activity interceptor on every Temporal worker, including both memory workers", async () => { +it("registers the activity interceptor and OTLP tracing on every Temporal worker, including both memory workers", async () => { const { readFile } = await import("node:fs/promises"); for (const entry of ["worker.ts", "memory-maintenance-worker.ts", "memory-mutation-worker.ts"]) { const source = await readFile(new URL(`../src/${entry}`, import.meta.url), "utf8"); - expect(source, entry).toMatch(/interceptors: \{ activity: \[createActivityObservabilityInterceptors\(\{ spans \}\)\] \}/); + // workerTracingOptions installs createActivityObservabilityInterceptors({ spans }) and, + // with export on, the workflow interceptor module and its sink (worker-tracing.ts). + expect(source, entry).toMatch(/const spans = otlpSpanExporterFromEnv\(process\.env, /); + expect(source, entry).toMatch(/\.\.\.workerTracingOptions\(spans\)/); + expect(source, entry).toMatch(/await spans\?\.close\(\)/); } }); diff --git a/apps/worker/tests/otlp-workflow-tracing.integration.test.ts b/apps/worker/tests/otlp-workflow-tracing.integration.test.ts new file mode 100644 index 000000000..8f5af5ec6 --- /dev/null +++ b/apps/worker/tests/otlp-workflow-tracing.integration.test.ts @@ -0,0 +1,317 @@ +// One connected OTLP trace from an API request through Temporal workflows, +// child workflows and every activity attempt, across the three worker +// processes' task queues. Runs real production workflows with stub activities +// on the @temporalio/testing time-skipping server and a local fake collector. +// No Docker, database or AI provider is used. +import { existsSync, readdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { ApplicationFailure } from "@temporalio/common"; +import { TestWorkflowEnvironment } from "@temporalio/testing"; +import { DefaultLogger, Runtime, Worker, type WorkerOptions } from "@temporalio/worker"; +import { formatTraceparent, buildCodexAppServerTurnWorkflowId, buildRoomAgentSupervisorWorkflowId } from "@space/contracts"; +import { createOtlpSpanExporter, type ServerSpan, type SpanExporter } from "@space/runtime"; +import { MEMORY_MAINTENANCE_TASK_QUEUE } from "../src/memory-maintenance-config.js"; +import { MEMORY_MUTATION_TASK_QUEUE } from "../src/memory-mutation-config.js"; +import { SPAN_ATTRIBUTE_ALLOWLIST } from "../src/trace-carrier.js"; +import { workerTracingOptions } from "../src/worker-tracing.js"; +import { startFakeOtlpCollector, type CollectedSpan, type FakeOtlpCollector } from "./support/fake-otlp-collector.js"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, "../../.."); +const src = (name: string) => path.join(here, "../src", name); +const MAIN_QUEUE = "space-tracing-test"; + +// Inside the workflow bundle, @space/* resolves to this checkout's sources. +const bundlerOptions: WorkerOptions["bundlerOptions"] = { + webpackConfigHook: (config) => { + const alias: Record = {}; + for (const name of readdirSync(path.join(repoRoot, "packages"))) { + const entry = path.join(repoRoot, "packages", name, "src", "index.ts"); + if (existsSync(entry)) alias[`@space/${name}$`] = entry; + } + return { ...config, resolve: { ...config.resolve, alias: { ...(config.resolve?.alias as object), ...alias } } }; + } +}; + +// Sensitive-looking text placed in every input and error. None of it may be exported. +const PRIVATE = "secret prompt text https://private.example/a?token=sk-live-1 /home/operator/notes.txt"; +const hex = (n: number, seed: string) => seed.repeat(n).slice(0, n); +const apiTraceId = (seed: string) => hex(32, seed); +const apiSpanId = (seed: string) => hex(16, seed); + +function turn(seed: string, overrides: Record = {}) { + return { + roomId: "room:tracing-01", paneId: "pane:tracing-01", prompt: PRIVATE, artifactIds: [], + providerId: "codex", traceId: `req:tracing-${seed}`, agentRunId: `agent_run:tracing-${seed}`, + permissionMode: null, collaborationMode: "default", + traceparent: formatTraceparent(apiTraceId(seed), apiSpanId(seed)), + ...overrides + }; +} + +const memoFor = (seed: string) => ({ spaceTraceId: `req:tracing-${seed}`, traceparent: formatTraceparent(apiTraceId(seed), apiSpanId(seed)), w3cTraceId: apiTraceId(seed) }); + +let env: TestWorkflowEnvironment; +let collector: FakeOtlpCollector; +const exporters: SpanExporter[] = []; +const workers: Worker[] = []; +const running: Promise[] = []; +const calls = { turnAttempts: 0, dotSteps: 0, finished: [] as string[] }; + +function exporterFor(service: string, endpoint: string, extra: Partial[0]> = {}) { + const exporter = createOtlpSpanExporter({ endpoint, serviceName: service, flushIntervalMs: 50, ...extra }); + exporters.push(exporter); + return exporter; +} + +async function startWorker(options: Omit) { + const worker = await Worker.create({ connection: env.nativeConnection, namespace: env.namespace ?? "default", bundlerOptions, ...options }); + workers.push(worker); + running.push(worker.run()); + return worker; +} + +const mainActivities = { + async runCodexAppServerTurn() { + calls.turnAttempts += 1; + if (calls.turnAttempts === 1) { + // A retryable failure whose message carries private text: only its stable type may be exported. + throw ApplicationFailure.retryable(`upstream failed for ${PRIVATE}`, "CODEX_APP_SERVER_UNAVAILABLE"); + } + return { status: "COMPLETED", message: "done", roomAgentOutcome: { status: "VERIFIED", statusReason: "Verified." } }; + }, + async checkRoomAgentSupervisorTurn() { return "READY"; }, + async discardRoomAgentSupervisorTurn() { return undefined; }, + async finishRoomAgentSupervisorTurn(input: { item: { turn: { agentRunId: string } } }) { calls.finished.push(input.item.turn.agentRunId); }, + async prepareRoomAgentBackgroundContinuation() { return { state: "DONE" }; }, + async runRoomAgentBackgroundAction() { return { status: "COMPLETED" }; }, + async dotGoalStep() { + calls.dotSteps += 1; + // 300 waiting steps end the first run (continue-as-new); the next run finishes. + return calls.dotSteps <= 300 ? { state: "WAITING", runId: null, goalStatus: "RUNNING" } : { state: "DONE", runId: null, goalStatus: "COMPLETED" }; + }, + async dotGoalObserve() { return { run: "COMPLETED", goalStatus: "RUNNING" }; } +}; + +const maintenanceActivities = { + async refreshMemoryGraphSnapshot() { return { sourceHash: "hash:tracing", status: "REFRESHED" }; }, + async refreshAllMonthsMemoryGraphSnapshot() { return undefined; }, + async reconcileStaleSpaceAgentSessions() { return undefined; }, + async createScheduledMemoryAudit() { return { id: "memory_run:tracing-1", workflowId: "space-memory-consolidation:tracing-1" }; }, + async prepareMemoryConsolidation() { return { run: { id: "memory_run:tracing-1" }, changeSetId: "memory_change:tracing-1" }; }, + async finalizeMemoryRepair() { return { id: "memory_run:tracing-1", status: "COMPLETED" }; } +}; + +const mutationActivities = { + async executePersistedMemoryChangeSet() { return { status: "APPLIED", changeSetId: "memory_change:tracing-1" }; } +}; + +async function spansWhere(predicate: (spans: CollectedSpan[]) => boolean, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + await Promise.all(exporters.map((exporter) => exporter.flush())); + if (predicate(collector.spans)) return collector.spans; + if (Date.now() > deadline) throw new Error(`Spans did not arrive: ${JSON.stringify(collector.spans.map((s) => s.name))}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +const inTrace = (traceId: string) => collector.spans.filter((span) => span.traceId === traceId); +const named = (spans: CollectedSpan[], name: string) => spans.filter((span) => span.name === name); + +beforeAll(async () => { + try { Runtime.install({ logger: new DefaultLogger("WARN") }); } catch { /* already installed in this process */ } + collector = await startFakeOtlpCollector(); + env = await TestWorkflowEnvironment.createTimeSkipping(); + // The three worker processes, each with its own exporter (service name) as in production. + await startWorker({ taskQueue: MAIN_QUEUE, workflowsPath: src("workflows.ts"), activities: mainActivities, + ...workerTracingOptions(exporterFor("space-worker", collector.endpoint)) }); + await startWorker({ taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE, workflowsPath: src("memory-maintenance-workflows.ts"), activities: maintenanceActivities, + ...workerTracingOptions(exporterFor("space-memory-maintenance-worker", collector.endpoint)) }); + await startWorker({ taskQueue: MEMORY_MUTATION_TASK_QUEUE, workflowsPath: src("memory-mutation-workflows.ts"), activities: mutationActivities, + ...workerTracingOptions(exporterFor("space-memory-mutation-worker", collector.endpoint)) }); +}, 180_000); + +afterAll(async () => { + for (const worker of workers) worker.shutdown(); + await Promise.allSettled(running); + await Promise.allSettled(exporters.map((exporter) => exporter.close())); + await env?.teardown(); + await collector?.close(); +}, 60_000); + +describe("OTLP worker and workflow spans", () => { + it("connects the API span, the turn workflow and every activity attempt in one trace", async () => { + const input = turn("a"); + const result = await env.client.workflow.execute("codexAppServerTurnWorkflow", { + taskQueue: MAIN_QUEUE, workflowId: buildCodexAppServerTurnWorkflowId(input), args: [input], memo: memoFor("a") + }); + expect(result).toMatchObject({ status: "COMPLETED" }); + await spansWhere(() => named(inTrace(apiTraceId("a")), "activity runCodexAppServerTurn").length === 2 && named(inTrace(apiTraceId("a")), "workflow codexAppServerTurnWorkflow").length === 1); + const trace = inTrace(apiTraceId("a")); + const [workflow] = named(trace, "workflow codexAppServerTurnWorkflow"); + expect(workflow).toMatchObject({ service: "space-worker", kind: 1, parentSpanId: apiSpanId("a"), status: 0 }); + expect(workflow!.attributes).toMatchObject({ "space.workflow_type": "codexAppServerTurnWorkflow", "space.outcome": "COMPLETED", "space.trace_id": "req:tracing-a", "space.room_id": "room:tracing-01" }); + const attempts = named(trace, "activity runCodexAppServerTurn").sort((a, b) => Number(a.attributes["space.attempt"]) - Number(b.attributes["space.attempt"])); + expect(attempts.map((span) => span.parentSpanId)).toEqual([workflow!.spanId, workflow!.spanId]); + expect(attempts[0]!.attributes).toMatchObject({ "space.attempt": 1, "space.outcome": "FAILED", "space.reason_code": "CODEX_APP_SERVER_UNAVAILABLE", "space.category": "UPSTREAM_UNAVAILABLE" }); + expect(attempts[0]!.status).toBe(2); + expect(attempts[1]!.attributes).toMatchObject({ "space.attempt": 2, "space.outcome": "COMPLETED" }); + expect(new Set(attempts.map((span) => span.spanId)).size).toBe(2); + }, 60_000); + + it("parents a Room Agent action workflow to the request span named in its memo", async () => { + await env.client.workflow.execute("roomAgentActionWorkflow", { + taskQueue: MAIN_QUEUE, workflowId: "space-room-action:action_tracing-1", + args: [{ actionId: "action_tracing-1", traceId: "req:tracing-9", bridge: { roomId: "room:tracing-01", missionId: "mission:tracing-1", prompt: PRIVATE, backgroundExecution: true } }], + memo: memoFor("9") + }); + await spansWhere(() => named(inTrace(apiTraceId("9")), "activity runRoomAgentBackgroundAction").length === 1 && named(inTrace(apiTraceId("9")), "workflow roomAgentActionWorkflow").length === 1); + const trace = inTrace(apiTraceId("9")); + const [workflow] = named(trace, "workflow roomAgentActionWorkflow"); + const [activity] = named(trace, "activity runRoomAgentBackgroundAction"); + expect(workflow).toMatchObject({ service: "space-worker", parentSpanId: apiSpanId("9"), status: 0 }); + expect(workflow!.attributes).toMatchObject({ "space.workflow_type": "roomAgentActionWorkflow", "space.outcome": "COMPLETED" }); + expect(activity).toMatchObject({ parentSpanId: workflow!.spanId }); + expect(activity!.attributes).toMatchObject({ "space.attempt": 1, "space.outcome": "COMPLETED" }); + }, 60_000); + + it("nests child workflows under their parent across the maintenance and mutation workers", async () => { + await env.client.workflow.execute("memoryMaintenanceWorkflow", { + taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE, workflowId: "space-memory-maintenance:tracing", args: [{ traceId: "req:tracing-m" }], memo: memoFor("b") + }); + await spansWhere(() => named(inTrace(apiTraceId("b")), "activity executePersistedMemoryChangeSet").length === 1 && named(inTrace(apiTraceId("b")), "workflow memoryMaintenanceWorkflow").length === 1); + const trace = inTrace(apiTraceId("b")); + const [maintenance] = named(trace, "workflow memoryMaintenanceWorkflow"); + const [consolidation] = named(trace, "workflow memoryConsolidationWorkflow"); + expect(maintenance).toMatchObject({ parentSpanId: apiSpanId("b"), service: "space-memory-maintenance-worker" }); + expect(consolidation).toMatchObject({ parentSpanId: maintenance!.spanId }); + expect(consolidation!.attributes["space.parent_workflow_id"]).toBe("space-memory-maintenance:tracing"); + for (const name of ["refreshMemoryGraphSnapshot", "refreshAllMonthsMemoryGraphSnapshot", "reconcileStaleSpaceAgentSessions", "createScheduledMemoryAudit"]) { + expect(named(trace, `activity ${name}`)[0]?.parentSpanId).toBe(maintenance!.spanId); + } + expect(named(trace, "activity prepareMemoryConsolidation")[0]?.parentSpanId).toBe(consolidation!.spanId); + expect(named(trace, "activity finalizeMemoryRepair")[0]?.parentSpanId).toBe(consolidation!.spanId); + // Scheduled by the consolidation workflow, executed by the mutation worker. + expect(named(trace, "activity executePersistedMemoryChangeSet")[0]).toMatchObject({ parentSpanId: consolidation!.spanId, service: "space-memory-mutation-worker" }); + }, 60_000); + + it("traces each Room Agent mission turn in its request's trace and links background continuations", async () => { + const roomId = "room:tracing-01"; + const workflowId = buildRoomAgentSupervisorWorkflowId(roomId); + const operator = { missionId: "mission:tracing-1", turn: turn("c", { roomAgentMissionId: "mission:tracing-1", agentRunId: "agent_run:tracing-op" }) }; + const handle = await env.client.workflow.signalWithStart("roomAgentSupervisorWorkflow", { + taskQueue: MAIN_QUEUE, workflowId, args: [{ roomId, pending: [], processedCount: 0 }], + signal: "enqueueRoomAgentMission", signalArgs: [operator] + }); + // A background continuation carries no traceparent of its own, only the link source. + const continuationTurn: Record = turn("d", { + roomAgentMissionId: "mission:tracing-1", agentRunId: "agent_run:bg-tracing-1", traceId: "trace:bg-tracing-1", + roomAgentContinuation: { sourceRunId: "agent_run:tracing-op", generation: 1, sourceTraceparent: formatTraceparent(apiTraceId("c"), apiSpanId("c")) } + }); + delete continuationTurn.traceparent; + await spansWhere(() => calls.finished.includes("agent_run:tracing-op") && named(inTrace(apiTraceId("c")), "workflow roomAgentSupervisorWorkflow mission").length === 1); + await handle.signal("enqueueRoomAgentMission", { missionId: "mission:tracing-1", turn: continuationTurn }); + const continuationSpans = () => collector.spans.filter((span) => span.name === "workflow roomAgentSupervisorWorkflow mission" && span.attributes["space.agent_run_id"] === "agent_run:bg-tracing-1"); + await spansWhere(() => continuationSpans().length === 1 && collector.spans.some((span) => span.traceId === continuationSpans()[0]!.traceId && span.name === "activity runCodexAppServerTurn")); + await handle.terminate("test finished"); + + const request = inTrace(apiTraceId("c")); + const [mission] = named(request, "workflow roomAgentSupervisorWorkflow mission"); + expect(mission).toMatchObject({ parentSpanId: apiSpanId("c") }); + expect(mission!.attributes).toMatchObject({ "space.mission_id": "mission:tracing-1", "space.workflow_id": workflowId }); + const [child] = named(request, "workflow codexAppServerTurnWorkflow"); + expect(child!.parentSpanId).toBe(mission!.spanId); + expect(named(request, "activity checkRoomAgentSupervisorTurn")[0]?.parentSpanId).toBe(mission!.spanId); + expect(named(request, "activity finishRoomAgentSupervisorTurn")[0]?.parentSpanId).toBe(mission!.spanId); + expect(named(request, "activity runCodexAppServerTurn")[0]?.parentSpanId).toBe(child!.spanId); + // The mission scope links to the long-lived supervisor run. + expect(mission!.links).toEqual([expect.objectContaining({ attributes: expect.objectContaining({ "space.link_type": "parent_workflow" }) })]); + + // trace:bg-: a new trace whose spans link back to the request it continues. + const [continuation] = continuationSpans(); + expect(continuation!.traceId).not.toBe(apiTraceId("c")); + expect(continuation!.parentSpanId).toBeNull(); + expect(continuation!.attributes["space.continuation_generation"]).toBe(1); + expect(continuation!.links).toContainEqual({ traceId: apiTraceId("c"), spanId: apiSpanId("c"), attributes: { "space.link_type": "background_continuation" } }); + const continued = inTrace(continuation!.traceId); + const [continuedTurn] = named(continued, "workflow codexAppServerTurnWorkflow"); + expect(continuedTurn!.parentSpanId).toBe(continuation!.spanId); + expect(continuedTurn!.links).toContainEqual({ traceId: apiTraceId("c"), spanId: apiSpanId("c"), attributes: { "space.link_type": "background_continuation" } }); + }, 60_000); + + it("links a continue-as-new run to the run it continues", async () => { + const result = await env.client.workflow.execute("dotGoalWorkflow", { + taskQueue: MAIN_QUEUE, workflowId: "space-dot-goal:tracing", args: [{ goalId: "dot_goal:tracing-1" }], memo: memoFor("e") + }); + expect(result).toEqual({ goalStatus: "COMPLETED" }); + await spansWhere(() => named(inTrace(apiTraceId("e")), "workflow dotGoalWorkflow").length === 2); + const runs = named(inTrace(apiTraceId("e")), "workflow dotGoalWorkflow"); + const first = runs.find((span) => span.attributes["space.outcome"] === "CONTINUED_AS_NEW")!; + const second = runs.find((span) => span.attributes["space.outcome"] === "COMPLETED")!; + expect(first.parentSpanId).toBe(apiSpanId("e")); + expect(second.parentSpanId).toBe(apiSpanId("e")); + expect(second.attributes["space.continued_from_run_id"]).toBe(first.attributes["space.run_id"]); + expect(second.links).toEqual([{ traceId: first.traceId, spanId: first.spanId, + attributes: { "space.link_type": "continue_as_new", "space.run_id": first.attributes["space.run_id"] } }]); + expect(named(inTrace(apiTraceId("e")), "activity dotGoalStep").filter((span) => span.parentSpanId === second.spanId)).toHaveLength(1); + }, 120_000); + + it("emits no spans when a recorded history is replayed", async () => { + const replayed: ServerSpan[] = []; + const counting: SpanExporter = { record: (span) => { replayed.push(span); }, flush: async () => {}, close: async () => {}, stats: () => ({ exported: 0, dropped: 0, failed: 0 }) }; + const traced = workerTracingOptions(counting); + const cases: Array<[string, string]> = [ + [buildCodexAppServerTurnWorkflowId(turn("a")), "workflows.ts"], + ["space-memory-maintenance:tracing", "memory-maintenance-workflows.ts"], + ["space-memory-consolidation:tracing-1", "memory-maintenance-workflows.ts"] + ]; + for (const [workflowId, file] of cases) { + const history = await env.client.workflow.getHandle(workflowId).fetchHistory(); + // With tracing installed: replay succeeds (deterministic) and records nothing. + await Worker.runReplayHistory({ workflowsPath: src(file), bundlerOptions, interceptors: traced.interceptors, sinks: traced.sinks }, history); + // Without tracing: the same history replays too, so export can be switched on or off at any time. + await Worker.runReplayHistory({ workflowsPath: src(file), bundlerOptions }, history); + } + expect(replayed).toEqual([]); + }, 180_000); + + it("exports only allowlisted attributes and never private text", () => { + expect(collector.spans.length).toBeGreaterThan(10); + const allowed = new Set(Object.keys(SPAN_ATTRIBUTE_ALLOWLIST)); + for (const span of collector.spans) { + for (const key of Object.keys(span.attributes)) expect(allowed.has(key), `${span.name}: ${key}`).toBe(true); + for (const link of span.links) for (const key of Object.keys(link.attributes)) expect(allowed.has(key)).toBe(true); + } + const exported = JSON.stringify(collector.spans); + for (const fragment of ["secret prompt", "https://", "private.example", "/home/", "token=", "sk-live", "upstream failed"]) { + expect(exported).not.toContain(fragment); + } + }); + + it("does not delay or fail workflow work when the collector hangs", async () => { + const hanging = await startFakeOtlpCollector({ hang: true }); + const exporter = createOtlpSpanExporter({ endpoint: hanging.endpoint, serviceName: "space-worker", flushIntervalMs: 20, timeoutMs: 200, maxQueue: 2 }); + const queue = "space-tracing-hanging"; + const worker = await Worker.create({ connection: env.nativeConnection, namespace: env.namespace ?? "default", bundlerOptions, taskQueue: queue, + workflowsPath: src("workflows.ts"), activities: mainActivities, ...workerTracingOptions(exporter) }); + calls.turnAttempts = 1; // succeed on the first attempt + try { + const input = turn("f"); + const started = Date.now(); + const result = await worker.runUntil(env.client.workflow.execute("codexAppServerTurnWorkflow", { + taskQueue: queue, workflowId: buildCodexAppServerTurnWorkflowId(input), args: [input], memo: memoFor("f") + })); + expect(result).toMatchObject({ status: "COMPLETED" }); + expect(Date.now() - started).toBeLessThan(10_000); + await exporter.close(); + // The POSTs timed out and were counted; nothing was thrown into the workflow. + expect(exporter.stats().failed + exporter.stats().dropped).toBeGreaterThan(0); + expect(exporter.stats().exported).toBe(0); + } finally { + await hanging.close(); + } + }, 60_000); +}); diff --git a/apps/worker/tests/room-background-continuation.test.ts b/apps/worker/tests/room-background-continuation.test.ts index 127ccb649..fe292e380 100644 --- a/apps/worker/tests/room-background-continuation.test.ts +++ b/apps/worker/tests/room-background-continuation.test.ts @@ -35,6 +35,21 @@ describe("durable background continuation", () => { expect(prepared.item.turn.prompt.length).toBeLessThanOrEqual(8000); }); + it("starts a new trace for the continuation and keeps a link to the request it continues", async () => { + const source = `00-${"a".repeat(32)}-${"b".repeat(16)}-01`; + const c = continuationFixture("OPEN_TYPES", source); + const prepared = await prepareBackgroundContinuation(c.input, c.store); + if (prepared.state !== "READY") throw Error("Missing continuation"); + expect(prepared.item.turn.traceId).toBe("trace:bg-test-1"); + expect(prepared.item.turn.traceparent).toBeUndefined(); + expect(prepared.item.turn.roomAgentContinuation).toEqual({ sourceRunId: "agent_run:initial", generation: 1, sourceTraceparent: source }); + // Without a traced source the continuation carries no link. + const plain = continuationFixture(); + const untraced = await prepareBackgroundContinuation(plain.input, plain.store); + if (untraced.state !== "READY") throw Error("Missing continuation"); + expect(untraced.item.turn.roomAgentContinuation).toEqual({ sourceRunId: "agent_run:initial", generation: 1 }); + }); + it("retains totals beyond the detail window and emits intact bounded JSON", () => { const c = continuationFixture(); const actions = Array.from({ length: 80 }, (_, index) => ({ ...c.action, actionId: `action:${index}`, diff --git a/apps/worker/tests/room-continuation-fixture.ts b/apps/worker/tests/room-continuation-fixture.ts index 340e964e0..d8f90f252 100644 --- a/apps/worker/tests/room-continuation-fixture.ts +++ b/apps/worker/tests/room-continuation-fixture.ts @@ -1,7 +1,7 @@ import { buildCodexAppServerTurnWorkflowId, roomAgentSupervisorQueueItemSchema } from "@space/contracts"; import { InMemorySpaceStore } from "@space/runtime"; -export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" = "OPEN_TYPES") { +export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" = "OPEN_TYPES", traceparent?: string) { const store = new InMemorySpaceStore(); const room = store.createRoom({ name: "Continuation regression", initialPaneCount: 0 }); const pane = store.getOrCreateRoomAgentPane(room.id); @@ -14,7 +14,8 @@ export function continuationFixture(actionType: "OPEN_TYPES" | "CONFIGURE_PANE" modelId: "gpt-5.6-sol", reasoningEffort: "high", agentSessionId: session.sessionId, agentRunId: `agent_run:${suffix}`, agentUserMessageId: `agent_msg:user-${suffix}`, agentAssistantMessageId: `agent_msg:assistant-${suffix}`, roomAgentMissionId: "mission:continuation", - traceId: `trace:continuation-${suffix}`, selectedToolIds: ["room:inspect", "room:open_types", "room:send"] + traceId: `trace:continuation-${suffix}`, selectedToolIds: ["room:inspect", "room:open_types", "room:send"], + ...(traceparent ? { traceparent } : {}) } }); return store.enqueueRoomAgentMission({ requestId: `request:${suffix}`, clientRequestId: `client-${suffix}`, content: prompt, supervisorWorkflowId: `room-supervisor:${room.id}`, childWorkflowId: buildCodexAppServerTurnWorkflowId(item.turn), diff --git a/apps/worker/tests/support/fake-otlp-collector.ts b/apps/worker/tests/support/fake-otlp-collector.ts new file mode 100644 index 000000000..95a443aef --- /dev/null +++ b/apps/worker/tests/support/fake-otlp-collector.ts @@ -0,0 +1,80 @@ +// A local OTLP/HTTP JSON collector for tests: records every span posted to +// /v1/traces. No Docker, no network beyond loopback. +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +export interface CollectedSpan { + service: string; + traceId: string; + spanId: string; + parentSpanId: string | null; + name: string; + kind: number; + startTimeUnixNano: string; + endTimeUnixNano: string; + status: number; + attributes: Record; + links: Array<{ traceId: string; spanId: string; attributes: Record }>; +} + +type OtlpAttribute = { key: string; value: { stringValue?: string; intValue?: string } }; + +function attributes(list: OtlpAttribute[] | undefined): Record { + const result: Record = {}; + for (const { key, value } of list ?? []) result[key] = value.intValue !== undefined ? Number(value.intValue) : value.stringValue ?? ""; + return result; +} + +export interface FakeOtlpCollector { + endpoint: string; + spans: CollectedSpan[]; + requests: number; + close(): Promise; +} + +export async function startFakeOtlpCollector(options: { hang?: boolean } = {}): Promise { + const collector: FakeOtlpCollector = { endpoint: "", spans: [], requests: 0, close: async () => {} }; + const server: Server = createServer((request, response) => { + collector.requests += 1; + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + if (options.hang) return; // never answers: the exporter must time out + if (request.method !== "POST" || request.url !== "/v1/traces") { + response.writeHead(404).end(); + return; + } + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")); + for (const resourceSpans of body.resourceSpans ?? []) { + const service = attributes(resourceSpans.resource?.attributes)["service.name"] as string; + for (const scope of resourceSpans.scopeSpans ?? []) { + for (const span of scope.spans ?? []) { + collector.spans.push({ + service, + traceId: span.traceId, + spanId: span.spanId, + parentSpanId: span.parentSpanId ?? null, + name: span.name, + kind: span.kind, + startTimeUnixNano: span.startTimeUnixNano, + endTimeUnixNano: span.endTimeUnixNano, + status: span.status?.code ?? 0, + attributes: attributes(span.attributes), + links: (span.links ?? []).map((link: { traceId: string; spanId: string; attributes?: OtlpAttribute[] }) => ({ + traceId: link.traceId, spanId: link.spanId, attributes: attributes(link.attributes) + })) + }); + } + } + } + response.writeHead(200, { "content-type": "application/json" }).end("{}"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + collector.endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + collector.close = () => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }); + return collector; +} diff --git a/apps/worker/tests/workflow-tracing.test.ts b/apps/worker/tests/workflow-tracing.test.ts new file mode 100644 index 000000000..f1b2f61f4 --- /dev/null +++ b/apps/worker/tests/workflow-tracing.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { defaultPayloadConverter } from "@temporalio/common"; +import type { SpanExporter, ServerSpan } from "@space/runtime"; +import { + SPACE_TRACE_HEADER, + SPAN_ATTRIBUTE_ALLOWLIST, + allowlistSpanAttributes, + continuationFromArgs, + correlationAttributes, + deterministicSpanId, + deterministicTraceId, + readTraceHeader, + traceparentFromArgs, + withTraceHeader +} from "../src/trace-carrier.js"; +import { createWorkflowTracingSinks, workerTracingOptions, workflowSpanToServerSpan } from "../src/worker-tracing.js"; +import type { WorkflowSpanRecord } from "../src/workflow-tracing.js"; + +const traceparent = `00-${"a".repeat(32)}-${"b".repeat(16)}-01`; +const PRIVATE = ["Summarize my secret plan", "https://example.com/x?token=1", "/home/operator/.ssh/id_rsa", "Error: ENOENT open '/var/lib/x'"]; + +function recorder() { + const spans: ServerSpan[] = []; + const exporter: SpanExporter = { record: (span) => { spans.push(span); }, flush: async () => {}, close: async () => {}, stats: () => ({ exported: 0, dropped: 0, failed: 0 }) }; + return { spans, exporter }; +} + +const record = (overrides: Partial = {}): WorkflowSpanRecord => ({ + traceId: "a".repeat(32), spanId: "c".repeat(16), parentSpanId: "b".repeat(16), name: "workflow codexAppServerTurnWorkflow", + startTimeMs: 1_000, endTimeMs: 1_500, error: false, attributes: { "space.workflow_type": "codexAppServerTurnWorkflow" }, links: [], ...overrides +}); + +describe("span attribute allowlist", () => { + it("keeps only allowlisted keys with safe values", () => { + const attributes = allowlistSpanAttributes({ + "space.workflow_type": "codexAppServerTurnWorkflow", + "space.room_id": "room:01", + "space.attempt": 2, + "space.outcome": "FAILED", + "space.reason_code": "CODEX_APP_SERVER_TIMEOUT", + "space.category": "TIMEOUT", + prompt: PRIVATE[0], + "space.error_message": PRIVATE[3], + "space.pane_id": PRIVATE[2], + "space.workflow_id": PRIVATE[1], + "space.reason_code_bad": "x", + "space.outcome_bad": "maybe", + "space.duration_ms": -1 + }); + expect(attributes).toEqual({ + "space.workflow_type": "codexAppServerTurnWorkflow", "space.room_id": "room:01", "space.attempt": 2, + "space.outcome": "FAILED", "space.reason_code": "CODEX_APP_SERVER_TIMEOUT", "space.category": "TIMEOUT" + }); + }); + + it("rejects free text, URLs, paths and messages for every allowlisted key", () => { + for (const key of Object.keys(SPAN_ATTRIBUTE_ALLOWLIST)) { + for (const value of PRIVATE) expect(allowlistSpanAttributes({ [key]: value })).toEqual({}); + } + }); + + it("reads only identifiers from workflow arguments", () => { + expect(correlationAttributes([{ prompt: PRIVATE[0], roomId: "room:01", turn: { paneId: "pane:01", traceId: "req:abcdefgh", developerContext: PRIVATE[1] } }])) + .toEqual({ "space.room_id": "room:01", "space.pane_id": "pane:01", "space.trace_id": "req:abcdefgh" }); + expect(traceparentFromArgs([{ turn: { traceparent } }])?.parentId).toBe("b".repeat(16)); + expect(continuationFromArgs([{ turn: { roomAgentContinuation: { sourceRunId: "agent_run:1", generation: 2, sourceTraceparent: traceparent } } }])) + .toEqual({ traceparent, sourceRunId: "agent_run:1", generation: 2 }); + }); +}); + +describe("trace header", () => { + it("round-trips the carrier and ignores malformed values", () => { + const headers = withTraceHeader({ other: defaultPayloadConverter.toPayload("kept") }, { + traceparent, links: [{ traceparent, type: "continue_as_new", runId: "run-1" }] + }); + expect(Object.keys(headers).sort()).toEqual(["other", SPACE_TRACE_HEADER]); + expect(readTraceHeader(headers)).toEqual({ traceparent, links: [{ traceparent, type: "continue_as_new", runId: "run-1" }] }); + expect(readTraceHeader({ [SPACE_TRACE_HEADER]: defaultPayloadConverter.toPayload({ traceparent: PRIVATE[0], links: [{ traceparent: "x", type: "y" }] }) })) + .toEqual({ traceparent: null }); + expect(readTraceHeader({})).toBeNull(); + }); + + it("derives stable W3C ids without a random source", () => { + expect(deterministicSpanId("workflow/run-1")).toBe(deterministicSpanId("workflow/run-1")); + expect(deterministicSpanId("workflow/run-1")).not.toBe(deterministicSpanId("workflow/run-2")); + expect(deterministicSpanId("x")).toMatch(/^[0-9a-f]{16}$/); + expect(deterministicTraceId("x")).toMatch(/^[0-9a-f]{32}$/); + }); +}); + +describe("worker tracing options", () => { + it("installs no workflow interceptor or sink when export is off", () => { + const options = workerTracingOptions(null); + expect(options.sinks).toBeUndefined(); + expect(options.interceptors?.workflowModules).toBeUndefined(); + expect(options.interceptors?.activity).toHaveLength(1); + }); + + it("registers the workflow module and a replay-suppressed sink when export is on", () => { + const options = workerTracingOptions(recorder().exporter); + expect(options.interceptors?.workflowModules?.[0]).toMatch(/workflow-tracing\.(js|ts)$/); + expect(options.sinks?.spaceTracing?.recordSpan?.callDuringReplay).toBe(false); + }); + + it("records each workflow span once, even when a retried workflow task repeats it", () => { + const { spans, exporter } = recorder(); + const sinks = createWorkflowTracingSinks(exporter); + const fn = sinks.spaceTracing!.recordSpan!.fn as (info: unknown, span: WorkflowSpanRecord) => void; + fn({}, record()); + fn({}, record()); + fn({}, record({ spanId: "d".repeat(16) })); + expect(spans.map((span) => span.spanId)).toEqual(["c".repeat(16), "d".repeat(16)]); + }); + + it("never throws into the workflow when recording fails", () => { + const sinks = createWorkflowTracingSinks({ ...recorder().exporter, record: () => { throw new Error("collector down"); } }); + const fn = sinks.spaceTracing!.recordSpan!.fn as (info: unknown, span: WorkflowSpanRecord) => void; + expect(() => fn({}, record())).not.toThrow(); + }); + + it("converts sink records into INTERNAL spans with allowlisted attributes and links", () => { + const span = workflowSpanToServerSpan(record({ + attributes: { "space.workflow_type": "dotGoalWorkflow", prompt: PRIVATE[0] }, + links: [{ traceparent, type: "continue_as_new", runId: "run-1" }] + })); + expect(span).toMatchObject({ kind: 1, startTimeUnixNano: 1_000_000_000n, endTimeUnixNano: 1_500_000_000n, attributes: { "space.workflow_type": "dotGoalWorkflow" } }); + expect(span.links).toEqual([{ traceId: "a".repeat(32), spanId: "b".repeat(16), attributes: { "space.link_type": "continue_as_new", "space.run_id": "run-1" } }]); + }); +}); diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index e432af4c6..cf117c21f 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -52,3 +52,23 @@ background work, and is hidden in the web app. Public installs upgraded from 1.1.x that used Job Copilot, the games, channels, Telegram or the desktop routes set the modules they keep, for example `SPACE_OPTIONAL_MODULES=job-copilot,games`. + +## OTLP trace export + +`SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` names an OTLP/HTTP collector base URL, for +example `http://127.0.0.1:4318`; spans are posted as JSON to `/v1/traces`. +It is off when unset or when the value is not a plain `http:`/`https:` URL +(credentials, query strings and fragments are rejected). The same variable is +read by the API (`space-api`) and by every Temporal worker process: +`space-worker`, `space-memory-maintenance-worker` and +`space-memory-mutation-worker` (the `service.name` of their spans). Set it for +each service that should export. With it unset a worker registers no workflow +interceptor and no sink, and its activity records are unchanged. + +Export is best effort and needs no OpenTelemetry dependency +(`packages/runtime/src/otlp-span-exporter.ts`, shared by the API and the +workers): a bounded queue (4096 spans) drops and counts spans instead of +growing, each POST has a 3 s timeout, and a slow or failing collector never +delays or fails a request, workflow or activity. Every recorded span is exported +(no sampling). What the spans contain is described in +[observability](quality/observability.md#traces-w3c-trace-context). diff --git a/docs/quality/observability.md b/docs/quality/observability.md index d5cd7080e..01fbe0272 100644 --- a/docs/quality/observability.md +++ b/docs/quality/observability.md @@ -217,11 +217,69 @@ Without the endpoint, every activity still writes `activity.started` and The trace context is local by default. An optional OTLP/HTTP (JSON) exporter sends one SERVER span per API request when `SPACE_OTEL_EXPORTER_OTLP_ENDPOINT` is set (no -OpenTelemetry dependency; `packages/runtime/src/otlp-span-exporter.ts`). Since -2026-10-16 the worker and both memory workers use the same exporter (services -`space-worker`, `space-memory-maintenance-worker`, `space-memory-mutation-worker`): -one INTERNAL span per activity whose input carries a `traceparent`, as a child of -that span, with the activity type, outcome and attempt only. +OpenTelemetry dependency; `packages/runtime/src/otlp-span-exporter.ts`). The worker +and both memory workers use the same exporter module (services `space-worker`, +`space-memory-maintenance-worker`, `space-memory-mutation-worker`), so one API +request that starts Temporal work produces **one connected trace**: + +```text +API SERVER span space-api +└── workflow (INTERNAL) worker + ├── activity one span per attempt, attempt 1..n + └── workflow child workflows nest under their parent + └── activity also when another worker runs it +``` + +- **Workflow spans** are recorded by a workflow interceptor module + (`apps/worker/src/workflow-tracing.ts`). The parent is the span id of the + propagated `traceparent`: from the parent workflow's Temporal header, else the + workflow input's `traceparent`, else the memo's. The API passes its own SERVER + span id there (`traceparentForTraceId`), so the workflow span hangs directly + under the exported request span. A workflow without any context starts its + own trace. Covered: `codexAppServerTurnWorkflow`, `roomAgentActionWorkflow`, + `dotGoalWorkflow` (the API now puts its `traceparent` in the memo), + `memoryMaintenanceWorkflow` → `memoryConsolidationWorkflow`, + `memoryMutationWorkflow` and memory consolidations started by the API (memo + added when the starting request is known). +- **Room Agent missions.** `roomAgentSupervisorWorkflow` runs for a long time + and serves many requests, so each mission turn it processes gets its own span + (`workflow roomAgentSupervisorWorkflow mission`) in the trace of the request + that queued the turn. The turn's supervisor activities and its + `codexAppServerTurnWorkflow` child are children of that span, and the span + links to the supervisor run (`space.link_type=parent_workflow`). +- **Activity spans:** one INTERNAL span per activity *attempt* (a retried + activity has one span per attempt), child of the scheduling workflow's span, + which travels in the `space-trace` Temporal header, never in activity + arguments. Activities scheduled by a workflow without tracing fall back to the + input's `traceparent` as before. +- **Span links.** A continue-as-new run keeps the parent of the run it + continues and links to that run's span (`space.link_type=continue_as_new`, + `space.continued_from_run_id`). A background continuation (`trace:bg-`) + starts a new trace and links to the request it continues + (`space.link_type=background_continuation`): `prepareBackgroundContinuation` + stores that request's `traceparent` as `roomAgentContinuation.sourceTraceparent` + (an optional contract field) instead of reusing it as the parent. +- **Attributes come from an allowlist** (`SPAN_ATTRIBUTE_ALLOWLIST` in + `apps/worker/src/trace-carrier.ts`): ids (Space trace, workflow, run, parent + workflow, room, pane, mission, agent run, action, goal, activity), workflow + and activity type, attempt, outcome, the existing `reasonCode` and `category`, + durations and the continuation generation. String values must be identifiers + (`[A-Za-z0-9:._-]`), so prompts, model output, file paths, URLs, error + messages and any user text are never exported, even if a caller tried. +- **Determinism.** Workflow code does no I/O for tracing. Span ids are derived + from Temporal ids (never from the workflow's random source), times come from + the workflow clock, and spans leave the sandbox only through the + `spaceTracing` sink, registered with `callDuringReplay: false`; replaying a + history emits nothing. Headers are not part of the command sequence replay + checks, so histories recorded with or without tracing replay with either + setting. A workflow task that fails and is retried can call the sink again; + the worker drops the repeated span id. +- **Bounded and best effort.** The exporter's queue, drop counter and POST + timeout apply to worker spans too; a hanging collector never delays workflow + or activity work (`otlp-workflow-tracing.integration.test.ts`). A shared + exporter fix: a flush on an empty queue (the idle interval tick) left a + settled promise marked as in flight, after which no further spans were sent; + flush now returns early on an empty queue. - **API:** a valid inbound `traceparent` is continued (same trace ID, new span). An invalid or missing header gets a new trace ID. Every API log line carries `w3cTraceId` (Fastify `childLoggerFactory`). @@ -233,9 +291,9 @@ that span, with the activity type, outcome and attempt only. - **Worker:** worker logs read the trace ID from the activity input. Internal room-action calls from the worker send a child `traceparent`, so the internal API request joins the same trace. -- **Workflow code is unchanged.** Correlation is done in the API starter - (input/memo) and in an activity interceptor. The workflow bundle only gains the - optional contract field. +- **Workflow code:** the only change is the mission span around each + supervisor turn (`traceWorkflowScope`, a no-op without export); everything else + is in interceptors registered by `apps/worker/src/worker-tracing.ts`. ### Replay compatibility @@ -292,6 +350,9 @@ No credential or transcript export was added. | `apps/api/src/__tests__/files-search-errors.test.ts` | `EMPTY_RESULT`/`OK` and each failure code | | `apps/api/tests/files-search-routes.test.ts` | The HTTP error envelope | | `apps/worker/tests/activity-observability.test.ts` | Interceptor records, outcomes, categories, metrics, loopback endpoint | +| `apps/worker/tests/otlp-workflow-tracing.integration.test.ts` | Real workflows on the `@temporalio/testing` time-skipping server with stub activities and a fake OTLP collector: trace id continuity and parent/child structure (API → workflow → each activity attempt, child workflows across the maintenance and mutation workers, supervisor mission spans), continue-as-new and background-continuation links, replay emits no spans, attribute allowlist, hanging collector | +| `apps/worker/tests/workflow-tracing.test.ts` | Allowlist, `space-trace` header, deterministic ids, disabled path, sink de-duplication | +| `packages/runtime/tests/otlp-span-exporter-shared.test.ts` | Span links, flush after an idle tick, bounded queue with a slow collector | | `apps/api/tests/observability-fault-injection.test.ts` | Four injected executor faults and one Temporal start failure, end to end | ## Not covered @@ -341,9 +402,14 @@ No credential or transcript export was added. the lookup also returns the Space-native agent runs (`space_agent_runs`) of the trace's workflows, joined by workflow id, with their status and error code, and names a failed run's code as the cause when no event carries one. -- **Hosts emit no spans.** Since 2026-10-16 the OTLP exporter sends API SERVER - spans and worker activity spans (the worker and both memory workers); the CLI - and Browser Hosts still join traces through log lines only. +- **Span export covers the API, workflows and activities; hosts emit no + spans.** The OTLP exporter sends API SERVER spans and, from the worker and both + memory workers, workflow, mission and activity-attempt spans in one trace (see + Traces above). The CLI and Browser Hosts still join traces through log lines + only. A workflow span is exported when its run (or mission turn) ends, so a + running or terminated workflow's span is missing while its activity spans are + already visible. Workflows started outside an API request (for example the + memory maintenance schedule) start their own trace. - **Recovery counters are partial.** Recovery counters exist for plan model retries, CLI terminal reconnects and worker activity retries only. - **Worker endpoints are not scraped yet.** Stable's `deploy/space-worker.service` diff --git a/package-lock.json b/package-lock.json index 96d7fd636..5f8537f26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -187,11 +187,13 @@ "@space/streaming": "0.2.5", "@temporalio/activity": "1.24.0", "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", "@temporalio/worker": "1.24.0", "@temporalio/workflow": "1.24.0", "undici": "8.12.3" }, "devDependencies": { + "@temporalio/testing": "1.24.0", "typescript": "^5.9.3", "vitest": "^4.1.12" } @@ -3674,6 +3676,25 @@ "node": ">=12.0.0" } }, + "node_modules/@temporalio/testing": { + "version": "1.24.0", + "resolved": "https://registry.npmjs.org/@temporalio/testing/-/testing-1.24.0.tgz", + "integrity": "sha512-XKCuvgb7smKR3oXy5nDgwYSFX/hmg9zTJ2rZQQsVwJzFkga1TbnWDiuT5YMU0INKjoLhnM6c5YA/uLKlp8sdfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@temporalio/activity": "1.24.0", + "@temporalio/client": "1.24.0", + "@temporalio/common": "1.24.0", + "@temporalio/core-bridge": "1.24.0", + "@temporalio/proto": "1.24.0", + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0" + }, + "engines": { + "node": ">= 20.3.0" + } + }, "node_modules/@temporalio/worker": { "version": "1.24.0", "resolved": "https://registry.npmjs.org/@temporalio/worker/-/worker-1.24.0.tgz", @@ -11641,12 +11662,12 @@ } }, "packages/run-spaceapp": { - "version": "1.0.35", + "version": "1.1.9", "cpu": [ "x64", "arm64" ], - "license": "Apache-2.0", + "license": "MIT", "os": [ "darwin", "linux", diff --git a/packages/contracts/src/schemas.ts b/packages/contracts/src/schemas.ts index 7fb3c217a..fde7c8676 100644 --- a/packages/contracts/src/schemas.ts +++ b/packages/contracts/src/schemas.ts @@ -6909,7 +6909,10 @@ export const dummyTurnInputSchema = z.object({ roomAgentOperatorRequest: z.string().trim().min(1).max(4000).optional(), roomAgentContinuation: z.object({ sourceRunId: idSchema, - generation: z.number().int().min(1).max(16) + generation: z.number().int().min(1).max(16), + // Optional W3C trace context of the turn this continuation continues; the + // continuation starts its own trace and exports an OTLP span link to it. + sourceTraceparent: traceparentSchema.optional() }).optional(), agentUserMessageId: idSchema.optional(), agentAssistantMessageId: idSchema.optional(), diff --git a/packages/runtime/src/otlp-span-exporter.ts b/packages/runtime/src/otlp-span-exporter.ts index 86a0edf05..0a35bf190 100644 --- a/packages/runtime/src/otlp-span-exporter.ts +++ b/packages/runtime/src/otlp-span-exporter.ts @@ -1,5 +1,6 @@ /** - * Optional OTLP/HTTP (JSON) export of API server spans and worker activity spans. Off unless + * Optional OTLP/HTTP (JSON) export of API server spans and worker workflow and + * activity spans. The API and all three Temporal workers share this module. Off unless * SPACE_OTEL_EXPORTER_OTLP_ENDPOINT names a collector base URL (spans go to * `/v1/traces`). No dependency, no collector required: Space logs stay * joinable by W3C trace id without it. @@ -11,6 +12,13 @@ * request. */ +/** An OTLP span link: a causal reference to a span in this or another trace. */ +export interface SpanLink { + traceId: string; + spanId: string; + attributes?: Readonly>; +} + export interface ServerSpan { traceId: string; spanId: string; @@ -22,6 +30,11 @@ export interface ServerSpan { error: boolean; /** OTLP span kind; SERVER (2) unless given. Worker activities use INTERNAL (1). */ kind?: number; + /** + * Optional OTLP span links (continue-as-new runs, background continuations). + * Omitted from the request when absent, so API spans are serialized as before. + */ + links?: readonly SpanLink[]; } export interface SpanExporterStats { @@ -85,6 +98,11 @@ export function otlpTraceRequest(spans: readonly ServerSpan[], serviceName: stri startTimeUnixNano: span.startTimeUnixNano.toString(), endTimeUnixNano: span.endTimeUnixNano.toString(), attributes: Object.entries(span.attributes).map(([key, value]) => ({ key, value: attributeValue(value) })), + ...(span.links?.length ? { links: span.links.map(link => ({ + traceId: link.traceId, + spanId: link.spanId, + attributes: Object.entries(link.attributes ?? {}).map(([key, value]) => ({ key, value: attributeValue(value) })) + })) } : {}), // STATUS_CODE_ERROR or STATUS_CODE_UNSET status: { code: span.error ? 2 : 0 } })) @@ -121,15 +139,20 @@ export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanEx } }; const flush = (): Promise => { - if (!inFlight) { - inFlight = (async () => { - try { - while (queue.length) await flushOnce(); - } finally { - inFlight = null; - } - })(); - } + if (inFlight) return inFlight; + // An empty queue must not leave a settled promise behind as "in flight": + // the async body would clear inFlight before it was assigned, and every + // later flush (timer, full batch, close) would then be a no-op. + if (!queue.length) return Promise.resolve(); + // The queue is not empty, so the body awaits before its finally block and + // inFlight is always assigned before it is cleared. + inFlight = (async () => { + try { + while (queue.length) await flushOnce(); + } finally { + inFlight = null; + } + })(); return inFlight; }; const timer = setInterval(() => void flush(), options.flushIntervalMs ?? 5000); @@ -138,6 +161,9 @@ export function createOtlpSpanExporter(options: OtlpSpanExporterOptions): SpanEx return { record(span) { if (!traceIdPattern.test(span.traceId) || !spanIdPattern.test(span.spanId)) return; + if (span.links?.some(link => !traceIdPattern.test(link.traceId) || !spanIdPattern.test(link.spanId))) { + span = { ...span, links: span.links.filter(link => traceIdPattern.test(link.traceId) && spanIdPattern.test(link.spanId)) }; + } if (queue.length >= maxQueue) { stats.dropped += 1; return; diff --git a/packages/runtime/tests/otlp-span-exporter-shared.test.ts b/packages/runtime/tests/otlp-span-exporter-shared.test.ts new file mode 100644 index 000000000..bd6916146 --- /dev/null +++ b/packages/runtime/tests/otlp-span-exporter-shared.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { createOtlpSpanExporter, otlpTraceRequest, type ServerSpan } from "../src/otlp-span-exporter.js"; + +const span = (overrides: Partial = {}): ServerSpan => ({ + traceId: "a".repeat(32), spanId: "b".repeat(16), parentSpanId: null, name: "workflow example", + startTimeUnixNano: 1n, endTimeUnixNano: 2n, attributes: { "space.attempt": 1 }, error: false, kind: 1, ...overrides +}); + +describe("shared OTLP span exporter (worker additions)", () => { + it("serializes span links and leaves spans without links unchanged", () => { + const linked = otlpTraceRequest([span({ links: [{ traceId: "c".repeat(32), spanId: "d".repeat(16), attributes: { "space.link_type": "continue_as_new" } }] })], "space-worker"); + expect(linked.resourceSpans[0]!.scopeSpans[0]!.spans[0]).toMatchObject({ + links: [{ traceId: "c".repeat(32), spanId: "d".repeat(16), attributes: [{ key: "space.link_type", value: { stringValue: "continue_as_new" } }] }] + }); + const plain = otlpTraceRequest([span()], "space-api").resourceSpans[0]!.scopeSpans[0]!.spans[0]!; + expect("links" in plain).toBe(false); + }); + + it("drops invalid links but keeps the span", async () => { + const bodies: string[] = []; + const exporter = createOtlpSpanExporter({ endpoint: "http://collector.test", serviceName: "space-worker", flushIntervalMs: 60_000, + fetch: async (_url, init) => { bodies.push(init.body); return { ok: true }; } }); + exporter.record(span({ links: [{ traceId: "not-hex", spanId: "d".repeat(16) }, { traceId: "c".repeat(32), spanId: "d".repeat(16) }] })); + await exporter.close(); + const sent = JSON.parse(bodies[0]!).resourceSpans[0].scopeSpans[0].spans[0]; + expect(sent.links).toEqual([{ traceId: "c".repeat(32), spanId: "d".repeat(16), attributes: [] }]); + }); + + it("keeps exporting after a flush with an empty queue (idle timer tick)", async () => { + let posts = 0; + const exporter = createOtlpSpanExporter({ endpoint: "http://collector.test", serviceName: "space-worker", flushIntervalMs: 60_000, + fetch: async () => { posts += 1; return { ok: true }; } }); + await exporter.flush(); // what the interval timer does while nothing was recorded + exporter.record(span()); + await exporter.flush(); + exporter.record(span({ spanId: "e".repeat(16) })); + await exporter.close(); + expect(posts).toBe(2); + expect(exporter.stats()).toEqual({ exported: 2, dropped: 0, failed: 0 }); + }); + + it("never blocks record() on a slow collector and counts drops beyond the bound", async () => { + let release: () => void = () => {}; + const exporter = createOtlpSpanExporter({ endpoint: "http://collector.test", serviceName: "space-worker", flushIntervalMs: 60_000, + batchSize: 1, maxQueue: 2, timeoutMs: 50, + fetch: (_url, init) => new Promise((resolve, reject) => { + release = () => resolve({ ok: true }); + init.signal.addEventListener("abort", () => reject(new Error("aborted"))); + }) }); + const started = Date.now(); + for (let index = 0; index < 10; index += 1) exporter.record(span({ spanId: index.toString(16).padStart(16, "1") })); + expect(Date.now() - started).toBeLessThan(50); + expect(exporter.stats().dropped).toBeGreaterThan(0); + await exporter.close(); + release(); + const stats = exporter.stats(); + expect(stats.exported + stats.failed + stats.dropped).toBe(10); + }); +});