feat(programs): programs own their orchestrator flows and prompts

.claude/skills/wizard-development/SKILL.md

@@ -20,7 +20,8 @@ infrastructure should consume those boundaries.
 | Concern                                               | Owner                                                                                                                                                                           |
 | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | Framework detection, context, env conventions         | [FrameworkConfig](../../../src/programs/program-contract/framework-config.ts) and [framework configs](../../../src/services/frameworks/)                                                                       |
-| Integration instructions and orchestrator flows/tasks | [context-mill](https://github.com/PostHog/context-mill)                                                                                                                         |
+| Integration instructions (skills)                     | [context-mill](https://github.com/PostHog/context-mill)                                                                                                                         |
+| Orchestrator task graphs and task prompts             | Each program's `flow.ts` and `agent/`; see [orchestrator flows](../../../src/programs/README.md#orchestrator-flows)                                                              |
 | Programs: detection, runs, prerequisites and outcomes | One folder per program in [programs](../../../src/programs/), entered through its `index.ts`                                                                                                  |
 | Tools: commands that run no agent                     | [tools](../../../src/tools/), entered through `@tools`, with their screens in [TUI tools](../../../src/tui/tools/)                                                                            |
 | Program screens, flows and decks                      | [TUI programs](../../../src/tui/programs/)                                                                                                                                                    |

package.json

@@ -149,7 +149,7 @@
     "test:coverage": "pnpm build && vitest run --coverage",
     "test:e2e": "pnpm build && ./e2e-tests/run.sh",
     "test:e2e-record": "export RECORD_FIXTURES=true && pnpm build && ./e2e-tests/run.sh",
-    "try": "tsx --tsconfig tsconfig.base.json bin.ts",
+    "try": "tsx --tsconfig tsconfig.base.json --import ./scripts/markdown-loader.no-jest.mjs bin.ts",
     "dev": "pnpm build && pnpm link --global && pnpm build:watch",
     "test:watch": "vitest",
     "prepare": "husky",

scripts/markdown-hooks.no-jest.mjs

@@ -0,0 +1,14 @@
+// The load hook markdown-loader.no-jest.mjs registers: a `.md` module exports the file's text.
+import { readFile } from 'node:fs/promises';
+
+export async function load(url, context, nextLoad) {
+  if (!url.startsWith('file:') || !new URL(url).pathname.endsWith('.md')) {
+    return nextLoad(url, context);
+  }
+  const text = await readFile(new URL(url), 'utf8');
+  return {
+    format: 'module',
+    source: `export default ${JSON.stringify(text)};`,
+    shortCircuit: true,
+  };
+}

scripts/markdown-loader.no-jest.mjs

@@ -0,0 +1,4 @@
+// `pnpm try` runs from source, so it loads each `.md` import as the file's text, as the build does.
+import { register } from 'node:module';
+
+register('./markdown-hooks.no-jest.mjs', import.meta.url);

src/agent/__tests__/bundled-prompts.test.ts

@@ -0,0 +1,100 @@
+import fs from 'fs';
+import path from 'path';
+import { describe, expect, it, vi } from 'vitest';
+import {
+  assembleSeedPrompt,
+  assembleTaskPrompt,
+  bundledRegistry,
+  loadAgentRegistry,
+  type OrchestratorPromptContext,
+} from '../agent-prompt-loader';
+
+const ROOT = path.resolve(__dirname, '../../..');
+const RECORDED = path.join(ROOT, 'fixtures', 'context-mill');
+const BASE_URL = 'https://context-mill.test';
+
+// The fetch path reads the recorded release: the menu, then each prompt by its download URL's flow and id.
+vi.mock('@shared/fetch-retry', () => ({
+  fetchWithRetry: (url: string) => {
+    const menu = JSON.parse(
+      fs.readFileSync(path.join(RECORDED, 'agent-menu.json'), 'utf8'),
+    ) as { agents: { id: string; flow: string; downloadUrl: string }[] };
+    const file =
+      url === `${BASE_URL}/agent-menu.json`
+        ? path.join(RECORDED, 'agent-menu.json')
+        : (() => {
+            const entry = menu.agents.find((a) => a.downloadUrl === url);
+            if (!entry) throw new Error(`unexpected fetch ${url}`);
+            return path.join(RECORDED, 'agents', entry.flow, `${entry.id}.md`);
+          })();
+    const text = fs.readFileSync(file, 'utf8');
+    return Promise.resolve({ text: () => Promise.resolve(text) });
+  },
+}));
+
+const FLOWS: Array<[string, string]> = [
+  ['error-tracking', 'error-tracking'],
+  ['feature-flags', 'feature-flags'],
+  ['metrics', 'metrics'],
+  ['integration-v2', 'posthog-integration'],
+  ['replay-vision', 'replay-vision'],
+];
+
+// Run-specific values, normalized so only the prompt text can differ.
+const CONTEXT: OrchestratorPromptContext = {
+  projectId: 1,
+  projectApiKey: 'phc_test',
+  host: {
+    region: 'us',
+    apiHost: 'https://us.i.posthog.com',
+    appHost: 'https://us.posthog.com',
+    assetHost: 'https://us-assets.i.posthog.com',
+    mcpUrl: 'https://mcp.posthog.com/mcp',
+  },
+  examplePath: '.posthog-wizard-cache/reference/EXAMPLE.md',
+  commandmentsPath: '.posthog-wizard-cache/reference/COMMANDMENTS.md',
+  facts: [],
+};
+
+/** The value, or a failed test naming what was missing. */
+function must<T>(value: T | undefined, what: string): T {
+  if (value === undefined) throw new Error(`missing ${what}`);
+  return value;
+}
+
+function programPrompts(program: string): Record<string, string> {
+  const dir = path.join(ROOT, 'src', 'programs', program, 'agent');
+  return Object.fromEntries(
+    fs
+      .readdirSync(dir)
+      .filter((file) => file.endsWith('.md'))
+      .map((file) => [
+        file.replace(/\.md$/, ''),
+        fs.readFileSync(path.join(dir, file), 'utf8'),
+      ]),
+  );
+}
+
+describe.each(FLOWS)('bundled %s prompts', (flow, program) => {
+  it('assemble byte-identically to the fetched ones', async () => {
+    const fetched = await loadAgentRegistry(BASE_URL, flow);
+    const bundled = bundledRegistry(programPrompts(program), flow);
+
+    expect(bundled.types.sort()).toEqual(fetched.types.sort());
+    expect(bundled.sinkTypes).toEqual(fetched.sinkTypes);
+    expect(bundled.runnerSeededTypes).toEqual(fetched.runnerSeededTypes);
+    expect(
+      assembleSeedPrompt(CONTEXT, must(bundled.seed, 'bundled seed').body),
+    ).toBe(
+      assembleSeedPrompt(CONTEXT, must(fetched.seed, 'fetched seed').body),
+    );
+    for (const type of fetched.types) {
+      const a = must(bundled.get(type), `bundled ${type}`);
+      const b = must(fetched.get(type), `fetched ${type}`);
+      expect({ ...a, body: undefined }).toEqual({ ...b, body: undefined });
+      expect(assembleTaskPrompt(CONTEXT, a.body, a.skills)).toBe(
+        assembleTaskPrompt(CONTEXT, b.body, b.skills),
+      );
+    }
+  });
+});

src/agent/agent-prompt-loader.ts

@@ -459,6 +459,21 @@ async function fetchText(url: string): Promise<string> {
   return res.text();
 }
 
+/** The registry for one flow from prompts the wizard bundles, keyed by task type; parsed as the fetched ones are. */
+export function bundledRegistry(
+  prompts: Readonly<Record<string, string>>,
+  flow: string,
+  opts?: Parameters<typeof buildRegistry>[2],
+): AgentRegistry {
+  return buildRegistry(
+    Object.entries(prompts).map(([type, text]) =>
+      parseAgentPrompt(text, type, flow),
+    ),
+    flow,
+    opts,
+  );
+}
+
 /**
  * Fetch the agent menu and every agent prompt it lists, parse them, and build
  * the registry for one flow. Throws if the menu cannot be fetched — the

src/agent/runner/index.ts

@@ -70,6 +70,10 @@ export type {
 export { DEFAULT_BINDING } from './switchboard';
 export type { ProgramBinding } from './switchboard';
 export type { TaskOutcome } from './sequence/orchestrator/queue';
+export type {
+  TaskGraph,
+  TaskGraphNode,
+} from './sequence/orchestrator/task-graph';
 
 /**
  * Run a program's agent pipeline.

src/agent/runner/sequence/orchestrator/__tests__/task-graph.test.ts

@@ -0,0 +1,172 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+vi.mock('@utils/analytics', () => ({
+  analytics: { wizardCapture: vi.fn(), captureException: vi.fn() },
+}));
+
+import { QueueStore } from '@agent/runner/sequence/orchestrator/queue';
+import {
+  graphDiff,
+  graphsAgree,
+  typedEdges,
+  type TaskGraph,
+} from '@agent/runner/sequence/orchestrator/task-graph';
+
+function newStore(): QueueStore {
+  return new QueueStore(
+    fs.mkdtempSync(path.join(os.tmpdir(), 'task-graph-')),
+    'run',
+  );
+}
+
+const CHAIN: TaskGraph = {
+  sink: 'report',
+  excluded: [],
+  nodes: {
+    install: { dependsOn: [], picked: true },
+    init: { dependsOn: [], picked: true },
+    capture: { dependsOn: ['install', 'init'] },
+    report: { dependsOn: ['capture'] },
+  },
+};
+
+describe('typedEdges', () => {
+  it('splices an unqueued dependency into its own dependencies', () => {
+    const store = newStore();
+    const init = store.enqueue({ type: 'init' });
+    const capture = store.enqueue({ type: 'capture' });
+    const report = store.enqueue({ type: 'report' });
+    const edges = typedEdges(store, CHAIN);
+    expect(edges.get(capture.id)).toEqual([init.id]);
+    expect(edges.get(report.id)).toEqual([capture.id]);
+  });
+
+  it('pairs a fanned-out chain by its inputs', () => {
+    const graph: TaskGraph = {
+      sink: 'report',
+      excluded: [],
+      nodes: {
+        verify: { dependsOn: [], picked: true },
+        instrument: {
+          dependsOn: ['verify'],
+          picked: true,
+          matchInputs: ['service_name'],
+        },
+        report: { dependsOn: ['instrument'] },
+      },
+    };
+    const store = newStore();
+    const api = { service_name: 'api' };
+    const worker = { service_name: 'worker' };
+    const verifyApi = store.enqueue({ type: 'verify', inputs: api });
+    const verifyWorker = store.enqueue({ type: 'verify', inputs: worker });
+    const instrumentApi = store.enqueue({ type: 'instrument', inputs: api });
+    const instrumentWorker = store.enqueue({
+      type: 'instrument',
+      inputs: worker,
+    });
+    const report = store.enqueue({ type: 'report' });
+    const edges = typedEdges(store, graph);
+    expect(edges.get(instrumentApi.id)).toEqual([verifyApi.id]);
+    expect(edges.get(instrumentWorker.id)).toEqual([verifyWorker.id]);
+    expect(edges.get(report.id)).toEqual([
+      instrumentApi.id,
+      instrumentWorker.id,
+    ]);
+  });
+});
+
+describe('graphDiff', () => {
+  it('agrees with a planner graph that has redundant edges', () => {
+    const store = newStore();
+    const install = store.enqueue({ type: 'install' });
+    const init = store.enqueue({ type: 'init' });
+    const capture = store.enqueue({
+      type: 'capture',
+      dependsOn: [install.id, init.id],
+    });
+    store.enqueue({ type: 'report', dependsOn: [capture.id, install.id] });
+    expect(graphsAgree(graphDiff(store, CHAIN))).toBe(true);
+  });
+
+  it('catches a fanned-out task wired to the wrong sibling', () => {
+    const graph: TaskGraph = {
+      sink: 'report',
+      excluded: [],
+      nodes: {
+        verify: { dependsOn: [], picked: true },
+        instrument: {
+          dependsOn: ['verify'],
+          picked: true,
+          matchInputs: ['service_name'],
+        },
+        report: { dependsOn: ['instrument'] },
+      },
+    };
+    const store = newStore();
+    const api = { service_name: 'api' };
+    const worker = { service_name: 'worker' };
+    const verifyApi = store.enqueue({ type: 'verify', inputs: api });
+    store.enqueue({ type: 'verify', inputs: worker });
+    const instrumentWorker = store.enqueue({
+      type: 'instrument',
+      inputs: worker,
+      dependsOn: [verifyApi.id],
+    });
+    store.enqueue({ type: 'report', dependsOn: [instrumentWorker.id] });
+    expect(graphDiff(store, graph).edges).toContainEqual({
+      type: 'instrument',
+      missing: ['verify'],
+      extra: ['verify'],
+    });
+  });
+
+  it('names a dropped edge, a missing node and an extra one', () => {
+    const store = newStore();
+    store.enqueue({ type: 'install' });
+    store.enqueue({ type: 'report' });
+    store.enqueue({ type: 'invented' });
+    expect(graphDiff(store, CHAIN)).toEqual({
+      missingTypes: ['capture'],
+      extraTypes: ['invented'],
+      edges: [{ type: 'report', missing: ['install'], extra: [] }],
+    });
+  });
+});
+
+describe('QueueStore.replaceDependencies', () => {
+  it('rewires the queue to the typed graph', () => {
+    const store = newStore();
+    const install = store.enqueue({ type: 'install' });
+    const init = store.enqueue({ type: 'init' });
+    const capture = store.enqueue({ type: 'capture' });
+    const report = store.enqueue({ type: 'report' });
+    expect(store.replaceDependencies(typedEdges(store, CHAIN))).toEqual({
+      applied: true,
+    });
+    expect(store.get(capture.id)?.dependsOn).toEqual([install.id, init.id]);
+    expect(store.get(report.id)?.dependsOn).toEqual([capture.id]);
+    expect(graphsAgree(graphDiff(store, CHAIN))).toBe(true);
+  });
+
+  it('refuses a cycle and changes nothing', () => {
+    const store = newStore();
+    const a = store.enqueue({ type: 'a' });
+    const b = store.enqueue({ type: 'b', dependsOn: [a.id] });
+    const result = store.replaceDependencies(new Map([[a.id, [b.id]]]));
+    expect(result).toEqual({ applied: false, reason: 'cycle' });
+    expect(store.get(a.id)?.dependsOn).toEqual([]);
+  });
+
+  it('refuses to rewire a task that already started', () => {
+    const store = newStore();
+    const a = store.enqueue({ type: 'a' });
+    const b = store.enqueue({ type: 'b' });
+    store.start(b.id);
+    expect(store.replaceDependencies(new Map([[b.id, [a.id]]])).applied).toBe(
+      false,
+    );
+  });
+});