fix(workspaces): wait for the Docker bootstrap before using a woken workspace

services/api/src/workspaces/docker.ts

@@ -24,6 +24,9 @@ const KIND = "workspace-v2";
 export class DockerWorkspaceRuntime implements WorkspaceRuntime {
   readonly provider = "docker" as const;
 
+  /** Workspace container name → the container start already proven ready. */
+  private readonly readyStarts = new Map<string, string>();
+
   constructor(private readonly docker = new Docker()) {}
 
   async create(input: CreateWorkspace): Promise<WorkspaceHandle> {
@@ -35,7 +38,7 @@ export class DockerWorkspaceRuntime implements WorkspaceRuntime {
     if (existing) {
       this.assertOwned(existing, input.id, names.volume);
       if (!existing.State?.Running) await this.docker.getContainer(existing.Id).start();
-      await this.waitUntilReady(this.docker.getContainer(existing.Id));
+      await this.ensureReady(names.container, this.docker.getContainer(existing.Id));
       return this.handle(input, existing.Id, names);
     }
 
@@ -79,7 +82,7 @@ export class DockerWorkspaceRuntime implements WorkspaceRuntime {
     });
     try {
       await container.start();
-      await this.waitUntilReady(container);
+      await this.ensureReady(names.container, container);
     } catch (error) {
       await container.remove({ force: true, v: false }).catch(() => undefined);
       throw error;
@@ -92,7 +95,9 @@ export class DockerWorkspaceRuntime implements WorkspaceRuntime {
     const existing = await this.inspectContainer(names.container);
     if (!existing) return this.create(workspace);
     this.assertOwned(existing, workspace.id, names.volume);
-    if (!existing.State?.Running) await this.docker.getContainer(existing.Id).start();
+    const container = this.docker.getContainer(existing.Id);
+    if (!existing.State?.Running) await container.start();
+    await this.ensureReady(names.container, container);
     return this.handle(workspace, existing.Id, names);
   }
 
@@ -244,6 +249,7 @@ export class DockerWorkspaceRuntime implements WorkspaceRuntime {
 
   async destroy(workspace: WorkspaceLocator): Promise<void> {
     const names = this.assertLocator(workspace);
+    this.readyStarts.delete(names.container);
     const current = await this.inspectContainer(names.container);
     if (current) {
       this.assertOwned(current, workspace.id, names.volume);
@@ -402,6 +408,26 @@ export class DockerWorkspaceRuntime implements WorkspaceRuntime {
     await killer.start({}).catch(() => undefined);
   }
 
+  /**
+   * A started container is not yet a usable workspace: the bootstrap removes its
+   * readiness marker on every start, then brings up dockerd, the socket
+   * permissions and the preview gateways. Readiness only ever moves forward
+   * within one start, so each start is proven once and then remembered.
+   */
+  private async ensureReady(containerName: string, container: Docker.Container) {
+    const state = await container.inspect();
+    if (!state.State.Running) {
+      throw new WorkspaceRuntimeError(
+        "workspace_initialize_failed",
+        `workspace bootstrap exited with status ${state.State.ExitCode}`,
+      );
+    }
+    const start = `${state.Id}:${state.State.StartedAt}`;
+    if (this.readyStarts.get(containerName) === start) return;
+    await this.waitUntilReady(container);
+    this.readyStarts.set(containerName, start);
+  }
+
   private async waitUntilReady(container: Docker.Container) {
     const deadline = Date.now() + 180_000;
     while (Date.now() < deadline) {
@@ -461,7 +487,15 @@ function workspaceBootstrapCommand(input: CreateWorkspace) {
     "rm -f /workspace/.facility/runtime-ready",
     "mkdir -p /workspace/.facility/home /workspace/.facility/claude /workspace/.facility/codex /workspace/.facility/docker",
     "chown -R node:node /workspace",
-    "rm -f /var/run/docker.sock",
+    // A stopped container keeps its writable layer, so the previous start's
+    // runtime state is still on disk when it is woken. dockerd refuses to boot
+    // while /var/run/docker.pid exists, and its exec root holds containerd's
+    // stale socket and pidfile too — the daemon would wait for a peer that
+    // stopped with the container. Nothing is running yet, because this script
+    // is the container's own entrypoint, so all of it is stale by construction.
+    // Compute replacement never hit this: it discards the layer. Suspend and
+    // wake reuse it, which is the path this clears.
+    "rm -rf /var/run/docker /var/run/docker.pid /var/run/docker.sock",
     "dockerd --host=unix:///var/run/docker.sock --data-root=/workspace/.facility/docker --storage-driver=vfs >/workspace/.facility/dockerd.log 2>&1 &",
     "attempt=0; until docker info >/dev/null 2>&1; do attempt=$((attempt + 1)); test $attempt -lt 120; sleep 1; done",
     "chown root:node /var/run/docker.sock",

services/api/test/workspace-docker.test.ts

@@ -0,0 +1,181 @@
+import { createHash } from "node:crypto";
+import type Docker from "dockerode";
+import { describe, expect, it, vi } from "vitest";
+import { DockerWorkspaceRuntime } from "../src/workspaces/docker.js";
+import type { WorkspaceLocator } from "../src/workspaces/runtime.js";
+
+const workspaceId = "ws_0123456789abcdef";
+const suffix = createHash("sha256").update(workspaceId).digest("hex").slice(0, 24);
+const workspace: WorkspaceLocator = {
+  id: workspaceId,
+  image: "facility-runner:test",
+  externalRef: `facility-ws-${suffix}`,
+  volumeRef: `facility-ws-volume-${suffix}`,
+};
+
+/**
+ * A container that boots the way the workspace bootstrap does: `start` returns
+ * immediately, and the readiness marker only appears after `readyAfterProbes`
+ * probes. Every probe is recorded so a test can prove one was made — or wasn't.
+ */
+function fakeDocker(options: {
+  running: boolean;
+  readyAfterProbes: number;
+  bootstrapExits?: boolean;
+}) {
+  const probes: string[][] = [];
+  const state = {
+    running: options.running,
+    startedAt: "2026-09-07T09:00:00.000000000Z",
+    remaining: options.readyAfterProbes,
+  };
+  const start = vi.fn(async () => {
+    // A bootstrap that dies on start leaves the container stopped again.
+    state.running = options.bootstrapExits !== true;
+    state.startedAt = "2026-09-07T10:00:00.000000000Z";
+  });
+  const container = {
+    id: "container-1",
+    inspect: async () => ({
+      Id: "container-1",
+      State: { Running: state.running, StartedAt: state.startedAt, ExitCode: 0 },
+      Config: {
+        Labels: { "facility.workspace.id": workspaceId, "facility.workload.kind": "workspace-v2" },
+      },
+      Mounts: [{ Destination: "/workspace", Name: workspace.volumeRef }],
+    }),
+    start,
+    exec: async (options: { Cmd: string[] }) => {
+      probes.push(options.Cmd);
+      const ready = state.remaining <= 0;
+      state.remaining -= 1;
+      return {
+        start: async () => undefined,
+        inspect: async () => ({ Running: false, ExitCode: ready ? 0 : 1 }),
+      };
+    },
+  };
+  const docker = { getContainer: vi.fn(() => container) } as unknown as Docker;
+  return { docker, container, start, probes };
+}
+
+describe("Docker workspace readiness", () => {
+  it("waits for the bootstrap before handing back a woken workspace", async () => {
+    const { docker, start, probes } = fakeDocker({ running: false, readyAfterProbes: 2 });
+    const runtime = new DockerWorkspaceRuntime(docker);
+
+    await expect(runtime.wake(workspace)).resolves.toMatchObject({
+      computeRef: "container-1",
+      state: "running",
+    });
+
+    expect(start).toHaveBeenCalledOnce();
+    // Two failing probes plus the one that finally observed the marker.
+    expect(probes).toHaveLength(3);
+    expect(probes.at(-1)?.at(-1)).toContain("/workspace/.facility/runtime-ready");
+  });
+
+  it("waits for a container someone else started and is still booting", async () => {
+    const { docker, start, probes } = fakeDocker({ running: true, readyAfterProbes: 1 });
+    const runtime = new DockerWorkspaceRuntime(docker);
+
+    await runtime.wake(workspace);
+
+    expect(start).not.toHaveBeenCalled();
+    expect(probes).toHaveLength(2);
+  });
+
+  it("proves each container start once and reuses the result on later wakes", async () => {
+    const { docker, probes } = fakeDocker({ running: true, readyAfterProbes: 0 });
+    const runtime = new DockerWorkspaceRuntime(docker);
+
+    await runtime.wake(workspace);
+    await runtime.wake(workspace);
+    await runtime.wake(workspace);
+
+    expect(probes).toHaveLength(1);
+  });
+
+  it("refuses a workspace whose bootstrap exited instead of reporting it ready", async () => {
+    const { docker } = fakeDocker({
+      running: false,
+      readyAfterProbes: 0,
+      bootstrapExits: true,
+    });
+
+    await expect(new DockerWorkspaceRuntime(docker).wake(workspace)).rejects.toMatchObject({
+      code: "workspace_initialize_failed",
+    });
+  });
+});
+
+/** Records the container definition `create` builds, without a Docker daemon. */
+function recordingDocker() {
+  const created: Array<Record<string, unknown>> = [];
+  const notFound = Object.assign(new Error("no such container"), { statusCode: 404 });
+  const container = {
+    id: "container-new",
+    start: async () => undefined,
+    remove: async () => undefined,
+    inspect: async () => ({
+      Id: "container-new",
+      State: { Running: true, StartedAt: "2026-09-15T10:00:00.000000000Z", ExitCode: 0 },
+      NetworkSettings: { Ports: {} },
+    }),
+    exec: async () => ({
+      start: async () => undefined,
+      inspect: async () => ({ Running: false, ExitCode: 0 }),
+    }),
+  };
+  const docker = {
+    getContainer: () => ({
+      ...container,
+      inspect: async () => {
+        throw notFound;
+      },
+    }),
+    getVolume: () => ({
+      inspect: async () => {
+        throw notFound;
+      },
+    }),
+    getNetwork: () => ({
+      inspect: async () => {
+        throw notFound;
+      },
+    }),
+    getImage: () => ({ inspect: async () => ({}) }),
+    createNetwork: async () => ({}),
+    createVolume: async () => ({}),
+    createContainer: async (options: Record<string, unknown>) => {
+      created.push(options);
+      return container;
+    },
+  } as unknown as Docker;
+  return { docker, created };
+}
+
+describe("Docker workspace bootstrap", () => {
+  it("clears the previous start's daemon runtime state before starting dockerd", async () => {
+    const { docker, created } = recordingDocker();
+
+    await new DockerWorkspaceRuntime(docker).create({
+      id: workspaceId,
+      image: "facility-runner:test",
+    });
+
+    const script = String((created[0]?.Cmd as string[])[0]);
+    const lines = script.split("\n");
+    const cleanup = lines.findIndex((line) => line.includes("/var/run/docker.pid"));
+    const daemon = lines.findIndex((line) => line.startsWith("dockerd "));
+
+    // A stopped container keeps its writable layer, so dockerd finds the pidfile
+    // its previous start wrote and refuses to boot. Removing it after the daemon
+    // line would be useless, which is why the order is asserted and not just the
+    // presence of the command.
+    expect(cleanup).toBeGreaterThanOrEqual(0);
+    expect(daemon).toBeGreaterThan(cleanup);
+    expect(lines[cleanup]).toContain("/var/run/docker");
+    expect(lines[cleanup]).toContain("/var/run/docker.sock");
+  });
+});

services/api/test/workspace-runtime.integration.test.ts

@@ -13,6 +13,55 @@ import {
 const enabled = process.env.FACILITY_E2E_DOCKER === "1";
 
 describe.skipIf(!enabled)("DockerWorkspaceRuntime integration", () => {
+  it("brings the nested daemon back after a suspend and wake of the same container", async () => {
+    // The suite's other case replaces compute, which discards the container's
+    // writable layer and so never meets the state a stop leaves behind. Suspend
+    // and wake reuse that layer: dockerd finds the pidfile its previous start
+    // wrote and refuses to boot, the readiness probe never passes, and the
+    // bootstrap exits. Nothing short of stopping and starting the real
+    // container observes it.
+    const id = `ws_${randomBytes(12).toString("hex")}`;
+    const runtime = new DockerWorkspaceRuntime(new Docker());
+    const created = await runtime.create({
+      id,
+      image: process.env.FACILITY_WORKSPACE_TEST_IMAGE ?? "facility-runner:serialized",
+      environment: { FACILITY_PREVIEW_GATEWAY_TOKEN: randomBytes(32).toString("base64url") },
+    });
+    const workspace = created as WorkspaceLocator;
+    try {
+      const before = await runtime.exec(workspace, {
+        command: "sh",
+        args: ["-lc", "printf marker > repo-state && docker info --format '{{.Driver}}'"],
+      });
+      expect(before, before.stderr).toMatchObject({ exitCode: 0, stdout: "vfs\n" });
+
+      await runtime.suspend(workspace);
+      await expect(runtime.inspect(workspace)).resolves.toMatchObject({ state: "sleeping" });
+
+      const resumed = await runtime.wake(workspace);
+      // The same compute, not a replacement: that is the whole point of the case.
+      expect(resumed.computeRef).toBe(created.computeRef);
+
+      const after = await runtime.exec(workspace, {
+        command: "sh",
+        args: ["-lc", "printf '%s|' \"$(cat repo-state)\"; docker info --format '{{.Driver}}'"],
+      });
+      expect(after, after.stderr).toMatchObject({ exitCode: 0, stdout: "marker|vfs\n" });
+
+      // A second cycle, because the first wake writes a pidfile of its own and
+      // a fix that only cleans the original create's state would pass once.
+      await runtime.suspend(workspace);
+      await runtime.wake(workspace);
+      const twice = await runtime.exec(workspace, {
+        command: "docker",
+        args: ["info", "--format", "{{.Driver}}"],
+      });
+      expect(twice, twice.stderr).toMatchObject({ exitCode: 0, stdout: "vfs\n" });
+    } finally {
+      await runtime.destroy(workspace);
+    }
+  }, 300_000);
+
   it("reattaches the same named volume after its compute is removed", async () => {
     const id = `ws_${randomBytes(12).toString("hex")}`;
     const gatewayToken = randomBytes(32).toString("base64url");