fix(framework): assemble prompts per harness and record what was sent (AI-1034)

Prompt assembly moves out of `run-eval.ts` (an entry script that runs
`main()` on import, so it cannot be unit-tested) into
`harness/system-prompt.ts`, keyed on `exp.agent.id`. Every block is now
ai-sdk-only — the task framing, the tool-surface addendum, the skills
listing — so a CLI harness assembles to `''` and the engine stages no
system prompt file. The two "end your turn with a short summary" sentences
are gone from both modes: stopping behaviour is part of what is measured.

`runOne` now returns the exact assembled `systemPrompt`, so it lands in
`results/<experiment>/<eval>.json` and what an agent was told is
verifiable from the artifacts. It was previously unrecorded for every CLI
harness. `export-results.ts` builds an explicit whitelist, so it does not
reach the published web data.

`apps/framework` gains a `test` script (`vitest run harness`), wired into
`check`, so the prompt-assembly tests have a runner.

Refs AI-1034, #164

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y5f9ELzsnwjydxea1h9EW7

Author: claude

apps/framework/harness/run-eval.ts

@@ -27,6 +27,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { buildSystemPrompt } from './system-prompt.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -245,9 +246,9 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet {
 }
 
 /**
- * Local-stack skill sources: resolve each skill name to its host directory so
- * the sandbox can install it with Vercel's `skills` CLI; the agent then
- * discovers each skill by reading its SKILL.md with its file tools. The
+ * Sandbox skill sources: resolve each skill name to its host directory so the
+ * sandbox can install it with Vercel's `skills` CLI, which places it in every
+ * CLI harness's native project scope for that harness to discover. The
  * `skills/` entries are symlinks into the agent-skills submodule; realpath them
  * so `docker cp` copies real files, not dangling links. Missing skills are
  * skipped with a warning.
@@ -306,31 +307,6 @@ function readSessionSeedArgs(ev: EvalManifest) {
   };
 }
 
-function basePromptFor(mode: EvalMode): string {
-  if (mode === 'local-stack') {
-    return (
-      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
-      'Use the provided tools to inspect and modify the workspace and run commands. ' +
-      'When you are done, end your turn with a short summary of what you did.'
-    );
-  }
-  return (
-    'You are an agent solving a Supabase eval task. ' +
-    'Use the provided tools to inspect and modify the project. ' +
-    'When you are done, end your turn with a short summary of what you did ' +
-    '(or for audit tasks, your findings).'
-  );
-}
-
-function buildSystemPrompt(
-  mode: EvalMode,
-  addendum?: string,
-  skillContext?: string
-): string {
-  const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean);
-  return blocks.join('\n\n');
-}
-
 /**
  * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with
  * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`,
@@ -360,6 +336,13 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    /**
+     * The exact system prompt handed to the agent (`''` when it got none). CLI
+     * harnesses receive theirs as a file in the sandbox scratch dir, outside the
+     * exported workspace, so recording it here is the only way to verify from a
+     * run artifact what the agent was actually told.
+     */
+    systemPrompt: string;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -391,6 +374,7 @@ async function runOne(
   let lastTranscript: TranscriptPart[] = [];
   let lastAgentReport = '';
   let lastStoppedReason = 'not_started';
+  let lastSystemPrompt = '';
 
   for (let attempt = 1; attempt <= RUNS; attempt += 1) {
     if (ev.mode === 'local-stack') {
@@ -423,6 +407,7 @@ async function runOne(
         : undefined;
       await using session = disposable(
         await exp.localStack.startSession({
+          agent: exp.agent.id,
           cliVersion: ev.metadata.cliVersion,
           localDir: ev.localDir,
           includeServices: ev.metadata.services,
@@ -443,8 +428,13 @@ async function runOne(
         })
       );
 
+      const systemPrompt = buildSystemPrompt(
+        exp.agent.id,
+        'local-stack',
+        session.promptAddendum
+      );
       const run = await exp.agent.run({
-        systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
+        systemPrompt,
         userPrompt: prompt,
         tools: session.tools,
         sandbox: session.sandbox,
@@ -455,6 +445,7 @@ async function runOne(
       lastTranscript = run.transcript;
       lastAgentReport = run.agentReport;
       lastStoppedReason = run.stoppedReason;
+      lastSystemPrompt = systemPrompt;
 
       // Export the agent's workspace to the host so scorers can run host
       // tooling (vite/vitest from the repo root) against the produced files
@@ -496,6 +487,7 @@ async function runOne(
           transcript: run.transcript,
           agentReport: run.agentReport,
           stoppedReason: run.stoppedReason,
+          systemPrompt,
         };
       }
       logRetryAttempt(expName, ev, attempt, last);
@@ -508,7 +500,12 @@ async function runOne(
     // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0).
     // An in-process agent runs host-side with no sandbox.
     await using cliSandbox = agentRunsInSandbox
-      ? disposable(await createBareSandbox({ skills: skillSources }))
+      ? disposable(
+          await createBareSandbox({
+            agent: exp.agent.id,
+            skills: skillSources,
+          })
+        )
       : undefined;
     await using session = disposable(
       await exp.runtime.startSession({
@@ -517,14 +514,16 @@ async function runOne(
       })
     );
 
-    // CLI agents read their installed skills from disk (the bare sandbox folds
-    // the discovery listing into its promptAddendum). In-process agents have
-    // no filesystem, so their skills are advertised in the prompt and pulled
-    // on demand via the load_skill tool.
+    // CLI agents discover their installed skills themselves — the skills CLI
+    // put them in every harness's native project scope, so each one advertises
+    // and loads them in its own words and the bare sandbox contributes nothing
+    // here. In-process agents have no filesystem, so their skills are advertised
+    // in the prompt and pulled on demand via the load_skill tool.
     const skillsPrompt = agentRunsInSandbox
       ? cliSandbox!.promptAddendum
       : buildToolsSkillsPrompt(toolsSkills);
     const systemPrompt = buildSystemPrompt(
+      exp.agent.id,
       'tools',
       session.promptAddendum,
       skillsPrompt
@@ -541,6 +540,7 @@ async function runOne(
     lastTranscript = run.transcript;
     lastAgentReport = run.agentReport;
     lastStoppedReason = run.stoppedReason;
+    lastSystemPrompt = systemPrompt;
     last = await (scorer as ToolScorer)({
       ...session.scoringContext,
       toolCalls: run.toolCalls,
@@ -562,6 +562,7 @@ async function runOne(
         transcript: run.transcript,
         agentReport: run.agentReport,
         stoppedReason: run.stoppedReason,
+        systemPrompt,
       };
     }
     logRetryAttempt(expName, ev, attempt, last);
@@ -576,6 +577,7 @@ async function runOne(
     transcript: lastTranscript,
     agentReport: lastAgentReport,
     stoppedReason: lastStoppedReason,
+    systemPrompt: lastSystemPrompt,
   };
 }
 

apps/framework/harness/system-prompt.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from 'vitest';
+import type { AgentHarnessId } from '@supabase-evals/core';
+import {
+  buildSkillsPrompt,
+  buildToolSurfaceAddendum,
+  type SkillEntry,
+} from '@supabase-evals/sandbox';
+import { buildSystemPrompt } from './system-prompt.js';
+import type { EvalMode } from './types.js';
+
+const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode'];
+const MODES: EvalMode[] = ['tools', 'local-stack'];
+
+describe('buildSystemPrompt', () => {
+  it('gives the ai-sdk agent task framing in both modes', () => {
+    // ai-sdk is the one harness with no system prompt of its own, so it's the
+    // one harness the framework has to supply one for.
+    for (const mode of MODES) {
+      expect(buildSystemPrompt('ai-sdk', mode)).toContain(
+        'Use the provided tools'
+      );
+    }
+  });
+
+  it('gives no framing of our own to any CLI harness', () => {
+    // CLI harnesses ship their own system prompt; we're measuring that.
+    for (const agent of CLI_AGENTS) {
+      for (const mode of MODES) {
+        expect(buildSystemPrompt(agent, mode)).toBe('');
+      }
+    }
+  });
+
+  it('passes a CLI harness only the runtime blocks, with no base prompt', () => {
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(agent, 'local-stack', 'Addendum.', 'Skills listing.')
+      ).toBe('Addendum.\n\nSkills listing.');
+    }
+  });
+
+  it('assembles to nothing at all for a CLI harness, even with skills', () => {
+    // The real block producers, not stand-ins: with skills installed, a CLI
+    // harness must still receive an entirely empty system prompt. Codex and
+    // OpenCode find the skills through their own project-scope discovery and
+    // describe them to the model themselves.
+    const skills: SkillEntry[] = [
+      {
+        name: 'supabase',
+        description: 'Use for Supabase tasks.',
+        dir: '.claude/skills/supabase',
+      },
+    ];
+    for (const agent of CLI_AGENTS) {
+      expect(
+        buildSystemPrompt(
+          agent,
+          'local-stack',
+          buildToolSurfaceAddendum(agent),
+          buildSkillsPrompt(agent, skills)
+        )
+      ).toBe('');
+    }
+    // ai-sdk has no such mechanism — it only learns about skills from us.
+    const aiSdk = buildSystemPrompt(
+      'ai-sdk',
+      'local-stack',
+      buildToolSurfaceAddendum('ai-sdk'),
+      buildSkillsPrompt('ai-sdk', skills)
+    );
+    expect(aiSdk).toContain('## Available skills');
+    expect(aiSdk).toContain('- supabase: Use for Supabase tasks.');
+  });
+
+  it('never tells any agent how to end its turn', () => {
+    // Stopping behaviour is part of what an eval measures, so the harness must
+    // not coach it (e.g. "end your turn with a short summary").
+    for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) {
+      for (const mode of MODES) {
+        const prompt = buildSystemPrompt(agent, mode);
+        expect(prompt).not.toMatch(/summary/i);
+        expect(prompt).not.toMatch(/end your turn/i);
+      }
+    }
+  });
+
+  it('drops empty blocks instead of leaving blank gaps', () => {
+    expect(
+      buildSystemPrompt('claude-code', 'tools', '', 'Skills listing.')
+    ).toBe('Skills listing.');
+    expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/);
+  });
+});

apps/framework/harness/system-prompt.ts

@@ -0,0 +1,52 @@
+/**
+ * System-prompt assembly, per agent harness.
+ *
+ * An eval measures out-of-the-box agent behaviour, so the harness injects as
+ * little prompt of its own as it can get away with: only the ai-sdk agent gets
+ * any base framing, because it is the only harness with no system prompt of its
+ * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI
+ * agents ship their own coding-agent prompt, tool guidance, and stopping
+ * behaviour — and codex/opencode have no system-prompt flag at all, so anything
+ * we pass them lands on the *user* prompt.
+ */
+
+import type { AgentHarnessId } from '@supabase-evals/core';
+import type { EvalMode } from './types.js';
+
+/**
+ * Base framing for the ai-sdk harness: what it can't infer on its own — that it
+ * has tools, and what they act on. Deliberately silent on how to finish a turn
+ * (no "end with a summary"): stopping behaviour is part of what's measured.
+ * Empty for every CLI harness.
+ */
+function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string {
+  if (agent !== 'ai-sdk') return '';
+  if (mode === 'local-stack') {
+    return (
+      'You are an agent solving a Supabase eval task in a Linux workspace. ' +
+      'Use the provided tools to inspect and modify the workspace and run commands.'
+    );
+  }
+  return (
+    'You are an agent solving a Supabase eval task. ' +
+    'Use the provided tools to inspect and modify the project.'
+  );
+}
+
+/**
+ * Assemble the system prompt handed to the agent. Every block is optional, and
+ * every one of them is ai-sdk-only (the base framing, the tool-surface addendum,
+ * the skills listing), so a CLI harness ends up with `''` — the CLI engine then
+ * stages no system-prompt file at all rather than an empty one.
+ */
+export function buildSystemPrompt(
+  agent: AgentHarnessId,
+  mode: EvalMode,
+  addendum?: string,
+  skillContext?: string
+): string {
+  const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter(
+    Boolean
+  );
+  return blocks.join('\n\n');
+}

apps/framework/package.json

@@ -4,11 +4,12 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test:framework",
+    "check": "pnpm typecheck && pnpm test && pnpm test:framework",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "typecheck": "tsc --noEmit",
+    "test": "vitest run harness",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",