jdjioe5-cpu · Oct 29, 2026
diff --git a/apps/worker/package.json b/apps/worker/package.json
@@ -18,16 +18,17 @@
     "@space/db": "0.2.5",
     "@space/memory-graph": "0.2.5",
     "@space/opencode-control": "0.2.5",
+    "@space/runtime": "0.2.5",
     "@space/streaming": "0.2.5",
     "@temporalio/activity": "1.24.0",
     "@temporalio/client": "1.24.0",
     "@temporalio/worker": "1.24.0",
     "@temporalio/workflow": "1.24.0",
-    "undici": "8.10.2"
+    "undici": "8.12.3"
   },
   "devDependencies": {
     "typescript": "^5.9.3",
-    "vitest": "^4.1.11"
+    "vitest": "^4.1.12"
   },
-  "license": "BUSL-1.1"
+  "license": "MIT"
 }
diff --git a/apps/worker/src/activity-observability.ts b/apps/worker/src/activity-observability.ts
@@ -10,8 +10,10 @@ import {
   traceIdFromTraceparent,
   type FailureCategory
 } from "@space/contracts";
+import type { SpanExporter } from "@space/runtime";
 import { logIdentifier, workerLog, type WorkerLogValue } from "./worker-log.js";
 import { workerMetrics, type WorkerMetrics } from "./worker-metrics.js";
+import { allowlistSpanAttributes, runIdToSpanId, type OtlpSpan, type OtlpSpanLink } from "./otlp-attributes.js";
 
 /**
  * One structured start/finish record and metrics per activity execution, with
@@ -131,9 +133,18 @@ function isFailedResult(value: unknown): boolean {
 }
 
 export interface ObserveActivityDependencies {
+  spanExporter?: SpanExporter | null;
   metrics?: WorkerMetrics;
   now?: () => number;
   cancelled?: () => boolean;
+  /** OTLP export (SPACE_OTEL_EXPORTER_OTLP_ENDPOINT): one span per activity whose input carries a traceparent. */
+  spans?: SpanExporter | null;
+}
+
+function activityTraceparent(args: readonly unknown[]) {
+  const found: Partial<Record<CorrelationKey, string>> = {};
+  for (const arg of args) readCorrelation(arg, found, 2);
+  return parseTraceparent(found.traceparent);
 }
 
 export async function observeActivityExecution<T>(
@@ -185,17 +196,41 @@ export async function observeActivityExecution<T>(
       queueMs,
       rssBytes: process.memoryUsage().rss
     });
+    const parent = dependencies.spans ? activityTraceparent(args) : null;
+    if (parent) {
+      dependencies.spans!.record({
+        traceId: parent.traceId,
+        spanId: randomBytes(8).toString("hex"),
+        parentSpanId: parent.parentId,
+        name: `activity ${activityType}`,
+        startTimeUnixNano: BigInt(startedAt) * 1_000_000n,
+        endTimeUnixNano: BigInt(startedAt + durationMs) * 1_000_000n,
+        attributes: { "space.activity_type": activityType, "space.outcome": outcome, "space.attempt": info.attempt },
+        error: outcome === "FAILED" || outcome === "FAILED_RESULT",
+        kind: 1
+      });
+    }
   };
 
   try {
     const result = await observations.run(observation, execute);
-    if (isFailedResult(result)) finish("FAILED_RESULT", observation.reasonCode ?? "ACTIVITY_RESULT_FAILED", null);
-    else finish("COMPLETED", null, null);
+    if (isFailedResult(result)) {
+      const code = observation.reasonCode ?? "ACTIVITY_RESULT_FAILED";
+      finish("FAILED_RESULT", code, null);
+      if (dependencies.spanExporter) exportActivitySpanOtlp(dependencies, info, args, activityType, startedAt, now() - startedAt, queueMs, "FAILED_RESULT", categorizeFailureCode(code), code, null);
+    } else {
+      finish("COMPLETED", null, null);
+      if (dependencies.spanExporter) exportActivitySpanOtlp(dependencies, info, args, activityType, startedAt, now() - startedAt, queueMs, "COMPLETED", null, null, null);
+    }
     return result;
   } catch (error) {
     const cancelled = dependencies.cancelled?.() === true || /Cancel/i.test((error as { name?: string } | null)?.name ?? "");
     const errorName = logIdentifier((error as { name?: unknown } | null)?.name);
-    finish(cancelled ? "CANCELLED" : "FAILED", observation.reasonCode ?? errorCodeOf(error), errorName);
+    const code = observation.reasonCode ?? errorCodeOf(error);
+    const outcome: ActivityOutcome = cancelled ? "CANCELLED" : "FAILED";
+    const cat = cancelled ? "CANCELLED" : categorizeFailureCode(code);
+    finish(outcome, code, errorName);
+    if (dependencies.spanExporter) exportActivitySpanOtlp(dependencies, info, args, activityType, startedAt, now() - startedAt, queueMs, outcome, cat, code, errorName);
     throw error;
   }
 }
@@ -211,3 +246,99 @@ export function createActivityObservabilityInterceptors(dependencies: Omit<Obser
     }
   });
 }
+
+
+function exportActivitySpanOtlp(
+  dependencies: ObserveActivityDependencies,
+  info: ActivityInfo,
+  args: readonly unknown[],
+  activityType: string,
+  startedAt: number,
+  durationMs: number,
+  queueMs: number | null,
+  outcome: ActivityOutcome,
+  category: FailureCategory | null,
+  reasonCode: string | null,
+  errorName: string | null
+) {
+  const exporter = dependencies.spanExporter;
+  if (!exporter) return;
+  try {
+    const correlation = activityCorrelation(args);
+    const parent = activityTraceparent(args);
+    let traceId: string | null = parent?.traceId ?? null;
+    const temporalRunId = info.workflowExecution?.runId ?? null;
+    let parentSpanId: string | null = temporalRunId ? runIdToSpanId(temporalRunId) : (parent?.parentId ?? null);
+
+    if (!traceId && correlation.traceId && /^[0-9a-f]{32}$/i.test(correlation.traceId)) {
+      traceId = correlation.traceId.toLowerCase();
+      if (temporalRunId && !parentSpanId) {
+        parentSpanId = runIdToSpanId(temporalRunId);
+      }
+    }
+    if (!traceId) return;
+
+    let spanId = randomBytes(8).toString("hex");
+    while (/^0+$/.test(spanId)) spanId = randomBytes(8).toString("hex");
+
+    const rawAttrs: Record<string, unknown> = {
+      "service.name": "space-worker",
+      "temporal.activity_type": activityType,
+      "temporal.activity_id": info.activityId,
+      "temporal.attempt": info.attempt,
+      "temporal.workflow_type": info.workflowType ?? null,
+      "temporal.workflow_id": info.workflowExecution?.workflowId ?? null,
+      "temporal.run_id": temporalRunId,
+      "temporal.status": outcome === "COMPLETED" ? "COMPLETED" : "FAILED",
+      "temporal.outcome": outcome,
+      "temporal.duration_ms": durationMs,
+      "temporal.queue_ms": queueMs,
+      "temporal.reason_code": reasonCode,
+      "temporal.failure_category": category,
+      "temporal.error_name": errorName,
+      "space.room_id": correlation.roomId,
+      "space.pane_id": correlation.paneId,
+      "space.mission_id": correlation.missionId,
+      "space.turn_id": correlation.turnId,
+      "space.action_id": correlation.actionId,
+      "space.agent_run_id": correlation.agentRunId,
+      "space.agent_session_id": correlation.agentSessionId,
+      "space.trace_id": correlation.traceId,
+      "space.parent_trace_id": correlation.parentTraceId,
+      "space.activity_type": activityType,
+      "space.outcome": outcome,
+      "space.attempt": info.attempt
+    };
+    const attributes = allowlistSpanAttributes(rawAttrs);
+
+    const links: OtlpSpanLink[] = [];
+    for (const arg of args) {
+      if (!arg || typeof arg !== "object") continue;
+      const r = arg as Record<string, unknown>;
+      const turn = (r.turn ?? (r.item as Record<string, unknown>)?.turn) as Record<string, unknown> | undefined;
+      const cont = (r.roomAgentContinuation ?? turn?.roomAgentContinuation) as { sourceRunId?: string } | undefined;
+      if (cont?.sourceRunId) {
+        links.push({
+          traceId,
+          spanId: runIdToSpanId(cont.sourceRunId),
+          attributes: { "link.reason": "room_background_continuation" }
+        });
+        break;
+      }
+    }
+
+    const span: OtlpSpan = {
+      traceId,
+      spanId,
+      parentSpanId,
+      name: `activity ${activityType}`,
+      kind: 1,
+      startTimeUnixNano: BigInt(startedAt) * 1_000_000n,
+      endTimeUnixNano: BigInt(startedAt + durationMs) * 1_000_000n,
+      attributes,
+      error: outcome === "FAILED" || outcome === "FAILED_RESULT",
+      links: links.length > 0 ? links : undefined
+    };
+    exporter.record(span);
+  } catch {}
+}
diff --git a/apps/worker/src/memory-maintenance-worker.ts b/apps/worker/src/memory-maintenance-worker.ts
@@ -1,4 +1,5 @@
 import { fileURLToPath } from "node:url";
+import { createOtlpWorkflowSinks } from "./otlp-workflow-sink.js";
 import { NativeConnection, Worker } from "@temporalio/worker";
 import * as activities from "./memory-maintenance-activities.js";
 import {
@@ -7,6 +8,9 @@ import {
   memoryMaintenanceWorkerOptions
 } from "./memory-maintenance-config.js";
 import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js";
+import { otlpSpanExporterFromEnv } from "@space/runtime";
+import { createActivityObservabilityInterceptors } from "./activity-observability.js";
+import { reloadWhenServicesCommitChanges } from "./deployed-code-reload.js";
 
 const service = "space-memory-maintenance-worker";
 
@@ -19,13 +23,16 @@ if (!memoryMaintenanceEnabled(process.env)) {
 
   try {
     const connection = await NativeConnection.connect({ address });
+    const spans = otlpSpanExporterFromEnv(process.env, service);
     const worker = await Worker.create({
       connection,
       namespace,
       taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE,
+      sinks: createOtlpWorkflowSinks(spans),
       workflowsPath,
       activities,
-      ...memoryMaintenanceWorkerOptions
+      ...memoryMaintenanceWorkerOptions,
+      interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] }
     });
     console.log(JSON.stringify({
       service,
@@ -35,7 +42,10 @@ if (!memoryMaintenanceEnabled(process.env)) {
       taskQueue: MEMORY_MAINTENANCE_TASK_QUEUE,
       activityConcurrency: memoryMaintenanceWorkerOptions.maxConcurrentActivityTaskExecutions
     }));
+    const stopReloadWatch = reloadWhenServicesCommitChanges(worker, { log: (message) => console.log(JSON.stringify({ service, status: "reloading", message })) });
     await worker.run();
+    stopReloadWatch();
+    await spans?.close();
   } catch (error) {
     console.error(JSON.stringify({
       service,

diff --git a/apps/worker/src/memory-mutation-worker.ts b/apps/worker/src/memory-mutation-worker.ts
@@ -1,8 +1,12 @@
 import { fileURLToPath } from "node:url";
+import { createOtlpWorkflowSinks } from "./otlp-workflow-sink.js";
 import { NativeConnection, Worker } from "@temporalio/worker";
 import { executePersistedMemoryChangeSet } from "./memory-mutation-activity.js";
 import { MEMORY_MUTATION_TASK_QUEUE, memoryMutationsEnabled, memoryMutationWorkerOptions } from "./memory-mutation-config.js";
 import { DEFAULT_TEMPORAL_ADDRESS, DEFAULT_TEMPORAL_NAMESPACE } from "./constants.js";
+import { otlpSpanExporterFromEnv } from "@space/runtime";
+import { createActivityObservabilityInterceptors } from "./activity-observability.js";
+import { reloadWhenServicesCommitChanges } from "./deployed-code-reload.js";
 
 const service = "space-memory-mutation-worker";
 
@@ -12,14 +16,23 @@ if (!memoryMutationsEnabled(process.env)) {
   const address = process.env.SPACE_TEMPORAL_ADDRESS ?? DEFAULT_TEMPORAL_ADDRESS;
   const namespace = process.env.SPACE_TEMPORAL_NAMESPACE ?? DEFAULT_TEMPORAL_NAMESPACE;
   const connection = await NativeConnection.connect({ address });
+  const spans = otlpSpanExporterFromEnv(process.env, service);
   const worker = await Worker.create({
     connection,
     namespace,
     taskQueue: MEMORY_MUTATION_TASK_QUEUE,
+    sinks: createOtlpWorkflowSinks(spans),
     workflowsPath: fileURLToPath(new URL("./memory-mutation-workflows.js", import.meta.url)),
     activities: { executePersistedMemoryChangeSet },
-    ...memoryMutationWorkerOptions
+    ...memoryMutationWorkerOptions,
+    interceptors: { activity: [createActivityObservabilityInterceptors({ spans })] }
   });
   console.log(JSON.stringify({ service, status: "running", address, namespace, taskQueue: MEMORY_MUTATION_TASK_QUEUE }));
-  await worker.run();
+  const stopReloadWatch = reloadWhenServicesCommitChanges(worker, { log: (message) => console.log(JSON.stringify({ service, status: "reloading", message })) });
+  try {
+    await worker.run();
+  } finally {
+    stopReloadWatch();
+    await spans?.close();
+  }
 }
diff --git a/apps/worker/src/otlp-attributes.ts b/apps/worker/src/otlp-attributes.ts
@@ -0,0 +1,102 @@
+import { isSafeErrorCode } from "@space/contracts";
+import { logIdentifier } from "./worker-log.js";
+
+/**
+ * Strict allowlist of attributes that may be exported in OTLP spans.
+ *
+ * Privacy & Security Policy (Issue #61):
+ * Spans MUST carry only IDs, types, attempt counts, status/outcome, safe error
+ * codes/categories, and durations.
+ * Under NO circumstances may prompts, operator instructions, assistant responses,
+ * message contents, tool payloads, query strings, URLs, file paths, or credentials
+ * be emitted in attributes.
+ */
+
+const ALLOWED_ATTRIBUTE_KEYS = new Set([
+  // Resource / Service
+  "service.name",
+  // Standard HTTP / Route (API)
+  "http.request.method",
+  "http.route",
+  "http.response.status_code",
+  "space.request_id",
+  // Correlation IDs
+  "space.trace_id",
+  "space.w3c_trace_id",
+  "space.parent_trace_id",
+  "space.room_id",
+  "space.pane_id",
+  "space.mission_id",
+  "space.turn_id",
+  "space.action_id",
+  "space.goal_id",
+  "space.dot_id",
+  "space.agent_run_id",
+  "space.agent_session_id",
+  "space.cli_task_id",
+  "space.revision_id",
+  // Temporal Workflow / Activity identifiers & execution state
+  "temporal.workflow_id",
+  "temporal.workflow_type",
+  "temporal.run_id",
+  "temporal.parent_run_id",
+  "temporal.parent_workflow_id",
+  "temporal.activity_id",
+  "temporal.activity_type",
+  "temporal.attempt",
+  "temporal.status",
+  "temporal.outcome",
+  "temporal.duration_ms",
+  "temporal.queue_ms",
+  // Error / Classification
+  "temporal.reason_code",
+  "temporal.failure_category",
+  "temporal.error_code",
+  "temporal.error_name",
+  // Link metadata
+  "link.reason"
+]);
+
+/**
+ * Validates and filters attributes against the strict allowlist.
+ * Free-text strings, paths, URLs, and unknown keys are strictly dropped.
+ */
+export function allowlistSpanAttributes(
+  raw: Readonly<Record<string, unknown>>
+): Record<string, string | number> {
+  const sanitized: Record<string, string | number> = {};
+
+  for (const [key, value] of Object.entries(raw)) {
+    if (!ALLOWED_ATTRIBUTE_KEYS.has(key) || value === null || value === undefined) {
+      continue;
+    }
+
+    if (typeof value === "number") {
+      if (Number.isFinite(value)) {
+        sanitized[key] = value;
+      }
+      continue;
+    }
+
+    if (typeof value === "string") {
+      // Must pass identifier or safe code validation
+      const safeIdent = logIdentifier(value);
+      if (safeIdent !== null) {
+        sanitized[key] = safeIdent;
+      } else if (isSafeErrorCode(value)) {
+        sanitized[key] = value;
+      }
+    }
+  }
+
+  return sanitized;
+}
+
+/**
+ * Derives a deterministic 16-character hex W3C Span ID from a 128-bit Temporal UUID runId.
+ */
+export function runIdToSpanId(runId: string | null | undefined): string {
+  if (!runId) return "0000000000000000";
+  const hex = runId.replace(/[^0-9a-fA-F]/g, "").toLowerCase();
+  return hex.length >= 16 ? hex.slice(0, 16) : hex.padStart(16, "0");
+}