fix(api): coordinate workspace deletion with queued and running turns

apps/docs/docs/guides/operate-story.md

@@ -150,7 +150,10 @@ story done and suspends compute. Branch protection and required review remain th
 - **Archive** retains the same data but removes the story from the active workflow.
 - **Restore** makes an archived story active again and wakes compute when work requires it.
 - **Delete workspace** permanently removes the durable workspace. It requires explicit
-  confirmation and a matching idempotency key.
+  confirmation and a matching idempotency key. A running turn blocks deletion with a 409
+  response; wait for it to finish, then retry with a new idempotency key. Deletion cancels queued turns and prevents
+  pending messages or new requests from starting more work. If provider cleanup fails, retry
+  deletion; the workspace remains closed to new work until cleanup succeeds.
 
 Archive a completed story when it may still be useful. Delete only after the pull request, commits,
 artifacts, and any uncommitted files have been preserved elsewhere. See the [lifecycle

services/api/src/stories/service.ts

@@ -169,6 +169,8 @@ export class StoryWorkspaceService {
           "this story's workspace was explicitly deleted; start a new story identity",
         );
       }
+      await lockStory(tx, input.orgId, input.projectId, story.id);
+      await this.assertWorkspaceNotDeleting(tx, input.orgId, input.projectId, story.id);
       await recordParticipation(tx, story, input.actor);
 
       let conversation = (
@@ -243,52 +245,69 @@ export class StoryWorkspaceService {
       return { story, conversation, workspace };
     });
 
-    let handle: WorkspaceHandle;
-    const runtimeOperation =
-      aggregate.workspace.state === "creating" || !aggregate.workspace.externalRef
-        ? "create"
-        : "wake";
-    const runtimeStartedAt = performance.now();
-    try {
-      handle =
-        runtimeOperation === "create"
-          ? await this.runtime.create({
-              id: aggregate.workspace.id,
-              ...workspaceInput(aggregate.workspace),
-            })
-          : await this.runtime.wake(locatorFromRow(aggregate.workspace));
-      await this.db
-        .update(workspaces)
-        .set({
-          externalRef: handle.externalRef,
-          volumeRef: handle.volumeRef,
-          state: "running",
-          error: null,
-          lastActivityAt: new Date(),
-          updatedAt: new Date(),
-        })
-        .where(
-          and(
-            eq(workspaces.orgId, input.orgId),
-            eq(workspaces.projectId, input.projectId),
-            eq(workspaces.id, aggregate.workspace.id),
-          ),
-        );
-      await appendWorkspaceEvent(this.db, aggregate.workspace.id, input.orgId, "workspace.ready", {
-        provider: this.runtime.provider,
-        computeRef: handle.computeRef,
-        operation: runtimeOperation,
-        durationMs: Math.round(performance.now() - runtimeStartedAt),
-      });
-    } catch (error) {
-      await this.markRuntimeFailure(aggregate.story.id, aggregate.workspace.id, input, error);
-      throw new StoryServiceError(
-        "workspace_start_failed",
-        error instanceof Error ? error.message : "workspace failed to start",
-        503,
-        { cause: error },
+    const failure = await this.db.transaction(async (rawTx) => {
+      const tx = rawTx as unknown as FacilityDb;
+      // Admission commits before provider I/O so a failed first start remains visible.
+      // Reacquire deletion's lock and re-read state before touching external resources.
+      await lockStory(tx, input.orgId, input.projectId, aggregate.story.id);
+      await this.assertWorkspaceNotDeleting(tx, input.orgId, input.projectId, aggregate.story.id);
+      const workspace = await this.activeWorkspace(
+        input.orgId,
+        input.projectId,
+        aggregate.story.id,
+        false,
+        tx,
       );
-    }
+      if (!workspace)
+        throw new StoryServiceError("story_workspace_deleted", "workspace is unavailable", 409);
+      try {
+        const runtimeOperation =
+          workspace.state === "creating" || !workspace.externalRef ? "create" : "wake";
+        const runtimeStartedAt = performance.now();
+        const handle: WorkspaceHandle =
+          runtimeOperation === "create"
+            ? await this.runtime.create({
+                id: workspace.id,
+                ...workspaceInput(workspace),
+              })
+            : await this.runtime.wake(locatorFromRow(workspace));
+        await tx
+          .update(workspaces)
+          .set({
+            externalRef: handle.externalRef,
+            volumeRef: handle.volumeRef,
+            state: "running",
+            error: null,
+            lastActivityAt: new Date(),
+            updatedAt: new Date(),
+          })
+          .where(
+            and(
+              eq(workspaces.orgId, input.orgId),
+              eq(workspaces.projectId, input.projectId),
+              eq(workspaces.id, workspace.id),
+            ),
+          );
+        await appendWorkspaceEvent(tx, workspace.id, input.orgId, "workspace.ready", {
+          provider: this.runtime.provider,
+          computeRef: handle.computeRef,
+          operation: runtimeOperation,
+          durationMs: Math.round(performance.now() - runtimeStartedAt),
+        });
+
+        return undefined;
+      } catch (error) {
+        // Commit failure state before releasing the lifecycle lock as well.
+        await this.markRuntimeFailure(tx, aggregate.story.id, workspace.id, input, error);
+        return new StoryServiceError(
+          "workspace_start_failed",
+          error instanceof Error ? error.message : "workspace failed to start",
+          503,
+          { cause: error },
+        );
+      }
+    });
+    if (failure) throw failure;
 
     const queued = await this.queueMessage({
       orgId: input.orgId,
@@ -335,6 +354,7 @@ export class StoryWorkspaceService {
       if (story.deletedAt) {
         throw new StoryServiceError("story_workspace_deleted", "story workspace was deleted");
       }
+      await this.assertWorkspaceNotDeleting(tx, input.orgId, input.projectId, input.storyId);
       const conversation = (
         await tx
           .select()
@@ -719,6 +739,16 @@ export class StoryWorkspaceService {
       const tx = rawTx as unknown as FacilityDb;
       await lockStory(tx, input.orgId, input.projectId, input.storyId);
       const story = await scopedStory(tx, input.orgId, input.projectId, input.storyId);
+      // A late queue callback must not create a retry notice for deleted work.
+      const workspace = await this.activeWorkspace(
+        input.orgId,
+        input.projectId,
+        input.storyId,
+        true,
+        tx,
+      );
+      if (story.deletedAt || workspace?.state === "deleting" || workspace?.state === "destroyed")
+        return undefined;
       await tx
         .update(stories)
         .set({
@@ -961,6 +991,14 @@ export class StoryWorkspaceService {
     const turn = await this.db.transaction(async (rawTx) => {
       const tx = rawTx as unknown as FacilityDb;
       await lockStory(tx, input.orgId, input.projectId, input.storyId);
+      const workspace = await this.activeWorkspace(
+        input.orgId,
+        input.projectId,
+        input.storyId,
+        true,
+        tx,
+      );
+      if (!workspace || ["deleting", "destroyed"].includes(workspace.state)) return undefined;
       const active = (
         await tx
           .select()
@@ -1595,14 +1633,20 @@ export class StoryWorkspaceService {
   }
 
   async suspend(orgId: string, projectId: string, storyId: string) {
-    const workspace = await this.activeWorkspace(orgId, projectId, storyId);
-    if (!workspace || workspace.state === "destroyed") return this.get(orgId, projectId, storyId);
-    await this.runtime.suspend(locatorFromRow(workspace));
-    await this.db
-      .update(workspaces)
-      .set({ state: "sleeping", updatedAt: new Date() })
-      .where(and(eq(workspaces.orgId, orgId), eq(workspaces.id, workspace.id)));
-    await appendWorkspaceEvent(this.db, workspace.id, orgId, "workspace.suspended", {});
+    await this.db.transaction(async (rawTx) => {
+      const tx = rawTx as unknown as FacilityDb;
+      // Keep provider suspension and its state writeback under deletion's lock.
+      // Re-read after locking so deleting/destroyed workspaces remain untouched.
+      await lockStory(tx, orgId, projectId, storyId);
+      const workspace = await this.activeWorkspace(orgId, projectId, storyId, false, tx);
+      if (!workspace) return;
+      await this.runtime.suspend(locatorFromRow(workspace));
+      await tx
+        .update(workspaces)
+        .set({ state: "sleeping", updatedAt: new Date() })
+        .where(and(eq(workspaces.orgId, orgId), eq(workspaces.id, workspace.id)));
+      await appendWorkspaceEvent(tx, workspace.id, orgId, "workspace.suspended", {});
+    });
     return this.get(orgId, projectId, storyId);
   }
 
@@ -1867,27 +1911,73 @@ export class StoryWorkspaceService {
         400,
       );
     }
-    const workspace = await this.activeWorkspace(input.orgId, input.projectId, input.storyId, true);
-    if (!workspace || workspace.state === "destroyed") {
-      return this.get(input.orgId, input.projectId, input.storyId);
-    }
-    await this.db.transaction(async (rawTx) => {
+    const workspace = await this.db.transaction(async (rawTx) => {
       const tx = rawTx as unknown as FacilityDb;
       await lockStory(tx, input.orgId, input.projectId, input.storyId);
+      const workspace = await this.activeWorkspace(
+        input.orgId,
+        input.projectId,
+        input.storyId,
+        true,
+        tx,
+      );
+      if (!workspace || workspace.state === "destroyed") return undefined;
+      // Lock the same rows dispatch claims with UPDATE. Whichever operation wins
+      // determines whether deletion cancels a queued turn or refuses a running one.
+      const active = await tx
+        .select()
+        .from(turns)
+        .where(
+          and(
+            eq(turns.orgId, input.orgId),
+            eq(turns.projectId, input.projectId),
+            eq(turns.storyId, input.storyId),
+            inArray(turns.state, ["queued", "running"]),
+          ),
+        )
+        .for("update");
+      if (active.some((turn) => turn.state === "running")) {
+        throw new StoryServiceError(
+          "workspace_busy",
+          "wait for the running turn to finish before deleting the workspace",
+          409,
+        );
+      }
+      const now = new Date();
+      for (const turn of active) {
+        await tx
+          .update(turns)
+          .set({ state: "canceled", endedAt: now, retryAfter: null, error: null, updatedAt: now })
+          .where(eq(turns.id, turn.id));
+        await appendTurnEvent(tx, {
+          orgId: input.orgId,
+          projectId: input.projectId,
+          storyId: input.storyId,
+          turnId: turn.id,
+          type: "turn.canceled",
+          data: { actor: input.actor, reason: "workspace_deleted" },
+        });
+      }
+      await tx
+        .update(stories)
+        .set({ activeAgentName: null, updatedAt: now })
+        .where(eq(stories.id, input.storyId));
       await tx
         .update(workspaces)
-        .set({ state: "deleting", updatedAt: new Date() })
+        .set({ state: "deleting", updatedAt: now })
         .where(and(eq(workspaces.orgId, input.orgId), eq(workspaces.id, workspace.id)));
       await tx
         .update(previewSessions)
-        .set({ revokedAt: new Date(), updatedAt: new Date() })
+        .set({ revokedAt: now, updatedAt: now })
         .where(
           and(
             eq(previewSessions.orgId, input.orgId),
             eq(previewSessions.workspaceId, workspace.id),
           ),
         );
+      return workspace;
     });
+    if (!workspace) return this.get(input.orgId, input.projectId, input.storyId);
 
     // This is intentionally the only lifecycle path that destroys provider
     // storage. Merge, archive, suspend, reconciliation, and schedule handling
@@ -1912,6 +2002,22 @@ export class StoryWorkspaceService {
     return this.get(input.orgId, input.projectId, input.storyId);
   }
 
+  private async assertWorkspaceNotDeleting(
+    db: FacilityDb,
+    orgId: string,
+    projectId: string,
+    storyId: string,
+  ) {
+    const workspace = await this.activeWorkspace(orgId, projectId, storyId, true, db);
+    if (workspace && ["deleting", "destroyed"].includes(workspace.state)) {
+      throw new StoryServiceError(
+        "story_workspace_deleted",
+        "story workspace is being deleted or was deleted",
+        409,
+      );
+    }
+  }
+
   private async activeWorkspace(
     orgId: string,
     projectId: string,
@@ -1941,33 +2047,31 @@ export class StoryWorkspaceService {
   }
 
   private async markRuntimeFailure(
+    tx: FacilityDb,
     storyId: string,
     workspaceId: string,
     input: StartStoryInput,
     error: unknown,
   ) {
     const detail = error instanceof Error ? error.message : String(error);
-    await this.db.transaction(async (rawTx) => {
-      const tx = rawTx as unknown as FacilityDb;
-      await tx
-        .update(workspaces)
-        .set({ state: "error", error: detail, updatedAt: new Date() })
-        .where(and(eq(workspaces.orgId, input.orgId), eq(workspaces.id, workspaceId)));
-      await tx
-        .update(stories)
-        .set({ status: "attention", updatedAt: new Date() })
-        .where(and(eq(stories.orgId, input.orgId), eq(stories.id, storyId)));
-      await tx.insert(attentionItems).values({
-        id: newId("attn"),
-        orgId: input.orgId,
-        projectId: input.projectId,
-        storyId,
-        kind: "runtime_error",
-        title: "Workspace could not start",
-        detail,
-      });
+    await tx
+      .update(workspaces)
+      .set({ state: "error", error: detail, updatedAt: new Date() })
+      .where(and(eq(workspaces.orgId, input.orgId), eq(workspaces.id, workspaceId)));
+    await tx
+      .update(stories)
+      .set({ status: "attention", updatedAt: new Date() })
+      .where(and(eq(stories.orgId, input.orgId), eq(stories.id, storyId)));
+    await tx.insert(attentionItems).values({
+      id: newId("attn"),
+      orgId: input.orgId,
+      projectId: input.projectId,
+      storyId,
+      kind: "runtime_error",
+      title: "Workspace could not start",
+      detail,
     });
-    await appendWorkspaceEvent(this.db, workspaceId, input.orgId, "workspace.provider_error", {
+    await appendWorkspaceEvent(tx, workspaceId, input.orgId, "workspace.provider_error", {
       operation: "create",
       error: detail.slice(0, 8_000),
     });

services/api/src/turns/dispatcher.ts

@@ -560,6 +560,7 @@ export class TurnDispatcher {
     storyId: string;
     turnId: string;
   }) {
+    if (!(await this.activeWorkspace(input.orgId, input.projectId, input.storyId))) return;
     const next = await this.nextQueuedMessage(input.orgId, input.projectId, input.storyId);
     if (!next?.requestedAgentName) return;
     try {
@@ -579,8 +580,10 @@ export class TurnDispatcher {
         },
       });
     } catch (error) {
+      // Deletion may have committed while catalog resolution was in flight.
+      if (!(await this.activeWorkspace(input.orgId, input.projectId, input.storyId))) return;
       const detail = error instanceof Error ? error.message : String(error);
-      await this.storiesService.flagAttention({
+      const attention = await this.storiesService.flagAttention({
         orgId: input.orgId,
         projectId: input.projectId,
         storyId: input.storyId,
@@ -589,6 +592,7 @@ export class TurnDispatcher {
         title: `Could not dispatch queued ${next.requestedAgentName} turn`,
         detail,
       });
+      if (!attention) return;
       await appendTurnEvent(this.db, {
         orgId: input.orgId,
         projectId: input.projectId,