diff --git a/README.md b/README.md index 124fb3f3..0e815e96 100644 --- a/README.md +++ b/README.md @@ -143,10 +143,10 @@ Skills come from [`supabase/agent-skills`](https://github.com/supabase/agent-ski To use a skill in an experiment, reference its directory name in the experiment's `skills` array. -Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem: +Skills are always loaded lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)) — a skill's full instructions are pulled on demand, never preloaded. How that happens depends on the harness: -- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change. -- **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. +- **CLI harnesses (Claude Code, Codex, OpenCode)** use their own built-in skills mechanism. Skills are installed into the sandbox workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), for each harness's own project scope: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI then discovers, advertises and loads the skills itself. The framework injects nothing — an agent's real-world skill-following behaviour is part of what an eval measures. +- **The in-process `ai-sdk` harness** has no such mechanism, so the framework supplies one. In local-stack mode it lists each skill's name+description in the system prompt and the agent reads `.claude/skills//SKILL.md` with its file tools. In tools mode there is no filesystem at all, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. ## Framework Checks diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index d84eb585..0da05658 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -29,6 +29,7 @@ import { } from '../lib/cli-args.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; +import { buildSystemPrompt } from './system-prompt.js'; import { buildDocsResult, buildSkillResult, @@ -252,9 +253,9 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet { } /** - * Local-stack skill sources: resolve each skill name to its host directory so - * the sandbox can install it with Vercel's `skills` CLI; the agent then - * discovers each skill by reading its SKILL.md with its file tools. The + * Sandbox skill sources: resolve each skill name to its host directory so the + * sandbox can install it with Vercel's `skills` CLI, which places it in every + * CLI harness's native project scope for that harness to discover. The * `skills/` entries are symlinks into the agent-skills submodule; realpath them * so `docker cp` copies real files, not dangling links. Missing skills are * skipped with a warning. @@ -313,31 +314,6 @@ function readSessionSeedArgs(ev: EvalManifest) { }; } -function basePromptFor(mode: EvalMode): string { - if (mode === 'local-stack') { - return ( - 'You are an agent solving a Supabase eval task in a Linux workspace. ' + - 'Use the provided tools to inspect and modify the workspace and run commands. ' + - 'When you are done, end your turn with a short summary of what you did.' - ); - } - return ( - 'You are an agent solving a Supabase eval task. ' + - 'Use the provided tools to inspect and modify the project. ' + - 'When you are done, end your turn with a short summary of what you did ' + - '(or for audit tasks, your findings).' - ); -} - -function buildSystemPrompt( - mode: EvalMode, - addendum?: string, - skillContext?: string -): string { - const blocks = [basePromptFor(mode), addendum, skillContext].filter(Boolean); - return blocks.join('\n\n'); -} - /** * Adapt a `{ close() }` resource to `AsyncDisposable` so it can be bound with * `await using` — cleanup then runs on scope exit (normal fall-through, `continue`, @@ -367,6 +343,13 @@ async function runOne( transcript: TranscriptPart[]; agentReport: string; stoppedReason: string; + /** + * The exact system prompt handed to the agent (`''` when it got none). CLI + * harnesses receive theirs as a file in the sandbox scratch dir, outside the + * exported workspace, so recording it here is the only way to verify from a + * run artifact what the agent was actually told. + */ + systemPrompt: string; } > { const prompt = parseEvalMarkdown( @@ -398,6 +381,7 @@ async function runOne( let lastTranscript: TranscriptPart[] = []; let lastAgentReport = ''; let lastStoppedReason = 'not_started'; + let lastSystemPrompt = ''; for (let attempt = 1; attempt <= RUNS; attempt += 1) { if (ev.mode === 'local-stack') { @@ -452,8 +436,13 @@ async function runOne( }) ); + const systemPrompt = buildSystemPrompt( + exp.agent.id, + 'local-stack', + session.promptAddendum + ); const run = await exp.agent.run({ - systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum), + systemPrompt, userPrompt: prompt, tools: session.tools, sandbox: session.sandbox, @@ -464,6 +453,7 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSystemPrompt = systemPrompt; // Export the agent's workspace to the host so scorers can run host // tooling (vite/vitest from the repo root) against the produced files @@ -505,6 +495,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, }; } logRetryAttempt(expName, ev, attempt, last); @@ -519,7 +510,6 @@ async function runOne( await using cliSandbox = agentRunsInSandbox ? disposable( await createBareSandbox({ - agent: exp.agent.id, skills: skillSources, mounts: supabaseMcpServerMounts(), }) @@ -532,14 +522,16 @@ async function runOne( }) ); - // CLI agents read their installed skills from disk (the bare sandbox folds - // the discovery listing into its promptAddendum). In-process agents have - // no filesystem, so their skills are advertised in the prompt and pulled - // on demand via the load_skill tool. + // CLI agents discover their installed skills themselves — the skills CLI + // put them in every harness's native project scope, so each one advertises + // and loads them in its own words and the bare sandbox contributes nothing + // here. In-process agents have no filesystem, so their skills are advertised + // in the prompt and pulled on demand via the load_skill tool. const skillsPrompt = agentRunsInSandbox - ? cliSandbox!.promptAddendum + ? undefined : buildToolsSkillsPrompt(toolsSkills); const systemPrompt = buildSystemPrompt( + exp.agent.id, 'tools', session.promptAddendum, skillsPrompt @@ -556,6 +548,7 @@ async function runOne( lastTranscript = run.transcript; lastAgentReport = run.agentReport; lastStoppedReason = run.stoppedReason; + lastSystemPrompt = systemPrompt; last = await (scorer as ToolScorer)({ ...session.scoringContext, toolCalls: run.toolCalls, @@ -577,6 +570,7 @@ async function runOne( transcript: run.transcript, agentReport: run.agentReport, stoppedReason: run.stoppedReason, + systemPrompt, }; } logRetryAttempt(expName, ev, attempt, last); @@ -591,6 +585,7 @@ async function runOne( transcript: lastTranscript, agentReport: lastAgentReport, stoppedReason: lastStoppedReason, + systemPrompt: lastSystemPrompt, }; } @@ -847,9 +842,16 @@ async function main() { } } -main() - .then(() => process.exit(0)) - .catch((e) => { - console.error(e); - process.exit(1); - }); +// Only when this file is the entry point. Importing it (a unit test reaching for +// one of its helpers) must not dispatch a run or call process.exit. +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main() + .then(() => process.exit(0)) + .catch((e) => { + console.error(e); + process.exit(1); + }); +} diff --git a/apps/framework/harness/system-prompt.test.ts b/apps/framework/harness/system-prompt.test.ts new file mode 100644 index 00000000..05babb63 --- /dev/null +++ b/apps/framework/harness/system-prompt.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; +import type { AgentHarnessId } from '@supabase-evals/core'; +import { + buildSkillsPrompt, + buildToolSurfaceAddendum, + type SkillEntry, +} from '@supabase-evals/sandbox'; +import { buildSystemPrompt } from './system-prompt.js'; +import type { EvalMode } from './types.js'; + +const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode']; +const MODES: EvalMode[] = ['tools', 'local-stack']; + +describe('buildSystemPrompt', () => { + it('gives the ai-sdk agent task framing in both modes', () => { + // ai-sdk is the one harness with no system prompt of its own, so it's the + // one harness the framework has to supply one for. + for (const mode of MODES) { + expect(buildSystemPrompt('ai-sdk', mode)).toContain( + 'Use the provided tools' + ); + } + }); + + it('gives no framing of our own to any CLI harness', () => { + // CLI harnesses ship their own system prompt; we're measuring that. + for (const agent of CLI_AGENTS) { + for (const mode of MODES) { + expect(buildSystemPrompt(agent, mode)).toBe(''); + } + } + }); + + it('refuses blocks a caller hands it for a CLI harness', () => { + // The producers gate their own output, so a non-empty block here means an + // experiment is misconfigured. Dropping it would be as silent as injecting + // it, and the block may be load-bearing: `executorMcpServer`'s addendum is + // the pause/resume protocol its tools require. + for (const agent of CLI_AGENTS) { + for (const mode of MODES) { + expect(() => buildSystemPrompt(agent, mode, 'Addendum.')).toThrow( + /must receive no system prompt/ + ); + expect(() => + buildSystemPrompt(agent, mode, undefined, 'Skills listing.') + ).toThrow(/must receive no system prompt/); + // Empty and absent blocks are the normal case, not a misconfiguration. + expect(buildSystemPrompt(agent, mode, '', '')).toBe(''); + } + } + }); + + it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => { + const base = buildSystemPrompt('ai-sdk', 'local-stack'); + expect( + buildSystemPrompt('ai-sdk', 'local-stack', 'Addendum.', 'Skills listing.') + ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`); + }); + + it('assembles to nothing at all for a CLI harness, even with skills', () => { + // The real block producers, not stand-ins: with skills installed, a CLI + // harness must still receive an entirely empty system prompt. Codex and + // OpenCode find the skills through their own project-scope discovery and + // describe them to the model themselves. + const skills: SkillEntry[] = [ + { + name: 'supabase', + description: 'Use for Supabase tasks.', + dir: '.claude/skills/supabase', + }, + ]; + for (const agent of CLI_AGENTS) { + expect( + buildSystemPrompt( + agent, + 'local-stack', + buildToolSurfaceAddendum(agent), + buildSkillsPrompt(agent, skills) + ) + ).toBe(''); + } + // ai-sdk has no such mechanism — it only learns about skills from us. + const aiSdk = buildSystemPrompt( + 'ai-sdk', + 'local-stack', + buildToolSurfaceAddendum('ai-sdk'), + buildSkillsPrompt('ai-sdk', skills) + ); + expect(aiSdk).toContain('## Available skills'); + expect(aiSdk).toContain('- supabase: Use for Supabase tasks.'); + }); + + it('never tells any agent how to end its turn', () => { + // Stopping behaviour is part of what an eval measures, so the harness must + // not coach it (e.g. "end your turn with a short summary"). + for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) { + for (const mode of MODES) { + const prompt = buildSystemPrompt(agent, mode); + expect(prompt).not.toMatch(/summary/i); + expect(prompt).not.toMatch(/end your turn/i); + } + } + }); + + it('drops empty blocks instead of leaving blank gaps', () => { + expect(buildSystemPrompt('ai-sdk', 'tools', '', 'Skills listing.')).toBe( + `${buildSystemPrompt('ai-sdk', 'tools')}\n\nSkills listing.` + ); + expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/); + }); +}); diff --git a/apps/framework/harness/system-prompt.ts b/apps/framework/harness/system-prompt.ts new file mode 100644 index 00000000..48dd03c7 --- /dev/null +++ b/apps/framework/harness/system-prompt.ts @@ -0,0 +1,76 @@ +/** + * System-prompt assembly, per agent harness. + * + * An eval measures out-of-the-box agent behaviour, so the harness injects as + * little prompt of its own as it can get away with: only the ai-sdk agent gets + * any base framing, because it is the only harness with no system prompt of its + * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI + * agents ship their own coding-agent prompt, tool guidance, and stopping + * behaviour — and codex/opencode have no system-prompt flag at all, so anything + * we pass them lands on the *user* prompt. + */ + +import type { AgentHarnessId } from '@supabase-evals/core'; +import type { EvalMode } from './types.js'; + +/** + * Base framing for the ai-sdk harness: what it can't infer on its own — that it + * has tools, and what they act on. Deliberately silent on how to finish a turn + * (no "end with a summary"): stopping behaviour is part of what's measured. + * Empty for every CLI harness. + */ +function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string { + if (agent !== 'ai-sdk') return ''; + if (mode === 'local-stack') { + return ( + 'You are an agent solving a Supabase eval task in a Linux workspace. ' + + 'Use the provided tools to inspect and modify the workspace and run commands.' + ); + } + return ( + 'You are an agent solving a Supabase eval task. ' + + 'Use the provided tools to inspect and modify the project.' + ); +} + +/** + * Assemble the system prompt handed to the agent. Every block is ai-sdk-only — + * the base framing, the tool-surface addendum, the skills listing — so a CLI + * harness ends up with `''`, and the CLI engine then stages no system-prompt + * file at all rather than an empty one. + * + * The callers' blocks are already gated by their producers, so a non-empty one + * arriving here means an experiment is misconfigured. Throw rather than drop it: + * dropping is as silent as injecting, and the block may be load-bearing. An MCP + * server carrying a `promptAddendum` is the live path in. Only + * `executorMcpServer` has one, and its text is the pause/resume protocol its + * tools require, not a tool description. A CLI harness paired with it would get + * the tools and none of the protocol, then stall on the first paused execution + * with a recorded prompt of `''` explaining nothing. + */ +export function buildSystemPrompt( + agent: AgentHarnessId, + mode: EvalMode, + addendum?: string, + skillContext?: string +): string { + if (agent !== 'ai-sdk') { + for (const [arg, block] of [ + ['addendum', addendum], + ['skillContext', skillContext], + ] as const) { + if (block) { + throw new Error( + `buildSystemPrompt got a non-empty ${arg} for '${agent}', which must receive ` + + 'no system prompt. Whichever runtime or MCP server produced it is not gated ' + + 'on the agent harness.' + ); + } + } + return ''; + } + const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter( + Boolean + ); + return blocks.join('\n\n'); +} diff --git a/apps/framework/package.json b/apps/framework/package.json index 5b282780..a78e9990 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,12 +4,13 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test:framework && pnpm test:vercel-runner", + "check": "pnpm typecheck && pnpm test && pnpm test:framework && pnpm test:vercel-runner", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts", "typecheck": "tsc --noEmit", + "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", diff --git a/packages/core/src/agents/claude-code/runner.test.ts b/packages/core/src/agents/claude-code/runner.test.ts index 07450363..28a42576 100644 --- a/packages/core/src/agents/claude-code/runner.test.ts +++ b/packages/core/src/agents/claude-code/runner.test.ts @@ -36,6 +36,46 @@ function streamJson(subtype: string, isError = false): string { ].join('\n'); } +/** The `claude` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand( + systemPromptPath: string | undefined +): Promise { + let runCommand = ''; + await claudeCodeRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes('/bin/claude')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'claude-sonnet-4-6', + apiKey: 'k', + systemPromptPath, + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('claudeCodeRunner.exec', () => { + it('appends the harness system prompt when there is one', async () => { + const command = await captureRunCommand('"$HOME/.eval/system-prompt.txt"'); + expect(command).toContain( + '--append-system-prompt-file "$HOME/.eval/system-prompt.txt"' + ); + }); + + it("omits the flag with no system prompt, leaving Claude Code's own intact", async () => { + const command = await captureRunCommand(undefined); + expect(command).not.toContain('--append-system-prompt-file'); + // The task itself is still piped in. + expect(command).toContain('cat "$HOME/.eval/user-prompt.txt"'); + }); +}); + describe('claudeCodeRunner.deriveStopReason', () => { const derive = claudeCodeRunner.deriveStopReason!; diff --git a/packages/core/src/agents/claude-code/runner.ts b/packages/core/src/agents/claude-code/runner.ts index 37129ca1..84bc1297 100644 --- a/packages/core/src/agents/claude-code/runner.ts +++ b/packages/core/src/agents/claude-code/runner.ts @@ -70,7 +70,10 @@ export const claudeCodeRunner: AgentRunner = { ...(reasoningEffort ? [`--effort ${shellQuote(reasoningEffort)}`] : []), // Append (not replace), from a file (no ARG_MAX/shell-expansion surface), // so Claude Code keeps its default coding-agent prompt + tool guidance. - `--append-system-prompt-file ${systemPromptPath}`, + // Omitted when there's nothing to append, leaving that default untouched. + ...(systemPromptPath + ? [`--append-system-prompt-file ${systemPromptPath}`] + : []), ...mcpFlags, // The sandbox is the isolation boundary, so skip permission prompts and // give the agent its full native toolset (same in both modes). diff --git a/packages/core/src/agents/codex/runner.test.ts b/packages/core/src/agents/codex/runner.test.ts new file mode 100644 index 00000000..ba2c5adc --- /dev/null +++ b/packages/core/src/agents/codex/runner.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; +import { codexRunner } from './runner.js'; + +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +/** The `codex exec` invocation from one exec, with a fake sandbox. */ +async function captureRunCommand( + systemPromptPath: string | undefined +): Promise { + let runCommand = ''; + await codexRunner.exec({ + sandbox: { + workspace: '/w', + exec: async (cmd) => { + if (cmd.includes(' exec ')) runCommand = cmd; + return ok; + }, + readFile: async () => '', + }, + model: 'gpt-5.4', + apiKey: 'k', + systemPromptPath, + userPromptPath: '"$HOME/.eval/user-prompt.txt"', + mcpServers: {}, + timeoutSec: 1, + }); + return runCommand; +} + +describe('codexRunner.exec', () => { + it('prepends the harness system prompt to the task when there is one', async () => { + // Codex has no system-prompt flag, so it lands on the user prompt. + const command = await captureRunCommand('"$HOME/.eval/system-prompt.txt"'); + expect(command).toContain( + `{ cat "$HOME/.eval/system-prompt.txt"; printf '\\n\\n'; cat "$HOME/.eval/user-prompt.txt"; }` + ); + }); + + it('sends the task alone with no system prompt (no leading blank block)', async () => { + const command = await captureRunCommand(undefined); + expect(command).not.toContain('system-prompt'); + expect(command).not.toContain("printf '\\n\\n'"); + expect(command.startsWith('cat "$HOME/.eval/user-prompt.txt" |')).toBe( + true + ); + }); +}); diff --git a/packages/core/src/agents/codex/runner.ts b/packages/core/src/agents/codex/runner.ts index f4e3837e..31ebd626 100644 --- a/packages/core/src/agents/codex/runner.ts +++ b/packages/core/src/agents/codex/runner.ts @@ -92,11 +92,15 @@ export const codexRunner: AgentRunner = { ].join(' '); // Codex has no system-prompt flag; prepend the system prompt to the task, - // both staged as files, fed on stdin. - const command = await sandbox.exec( - `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${codex} ${flags}`, - { timeoutMs: timeoutSec * 1000, env: { OPENAI_API_KEY: apiKey } } - ); + // both staged as files, fed on stdin. With no system prompt the task goes in + // alone — concatenating an empty one would open the prompt with a blank block. + const stdin = systemPromptPath + ? `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; }` + : `cat ${userPromptPath}`; + const command = await sandbox.exec(`${stdin} | ${codex} ${flags}`, { + timeoutMs: timeoutSec * 1000, + env: { OPENAI_API_KEY: apiKey }, + }); return { command, raw: command.stdout }; }, diff --git a/packages/core/src/agents/engine.test.ts b/packages/core/src/agents/engine.test.ts new file mode 100644 index 00000000..5b4a4c23 --- /dev/null +++ b/packages/core/src/agents/engine.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it } from 'vitest'; +import type { CommandResult } from '../index.js'; +import type { AgentTranscriptParser } from '../parsers/types.js'; +import { createCliAgent } from './engine.js'; +import { SYSTEM_PROMPT_PATH } from './shared.js'; +import type { AgentRunner } from './types.js'; + +const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + +const API_KEY_ENV_VAR = 'ENGINE_TEST_API_KEY'; + +/** A parser that reports one assistant message, so the engine stays quiet. */ +const parser: AgentTranscriptParser = { + parseTranscript: () => ({ + events: [{ type: 'message', role: 'assistant', content: 'done' }], + }), +}; + +/** + * Run a CLI agent against a fake sandbox, returning the `systemPromptPath` its + * runner was handed plus every command the engine ran in the sandbox. + */ +async function runWithSystemPrompt(systemPrompt: string): Promise<{ + systemPromptPath: string | undefined; + commands: string[]; +}> { + const commands: string[] = []; + let systemPromptPath: string | undefined; + const runner: AgentRunner = { + id: 'claude-code', + displayName: 'Fake CLI', + apiKeyEnvVar: API_KEY_ENV_VAR, + cliPackage: 'fake-cli', + defaultCliVersion: '1.0.0', + defaultModel: 'fake-model', + install: async () => undefined, + exec: async (args) => { + systemPromptPath = args.systemPromptPath; + return { command: ok, raw: '' }; + }, + }; + await createCliAgent(runner, parser, { model: 'fake-model' }).run({ + systemPrompt, + userPrompt: 'the task', + timeoutSec: 1, + sandbox: { + workspace: '/w', + exec: async (command) => { + commands.push(command); + return ok; + }, + readFile: async () => '', + }, + }); + return { systemPromptPath, commands }; +} + +describe('createCliAgent prompt staging', () => { + // The engine requires the runner's API key before it stages anything. + beforeEach(() => { + process.env[API_KEY_ENV_VAR] = 'k'; + }); + + it('stages a system prompt and hands its path to the runner', async () => { + const { systemPromptPath, commands } = await runWithSystemPrompt('Skills.'); + expect(systemPromptPath).toBe(SYSTEM_PROMPT_PATH); + expect(commands.some((c) => c.includes(SYSTEM_PROMPT_PATH))).toBe(true); + }); + + it('stages no file at all when the harness has no system prompt', async () => { + // The runner then omits its system-prompt plumbing, leaving the CLI's own + // prompt untouched instead of pointing it at an empty file. + const { systemPromptPath, commands } = await runWithSystemPrompt(''); + expect(systemPromptPath).toBeUndefined(); + expect(commands.some((c) => c.includes(SYSTEM_PROMPT_PATH))).toBe(false); + }); +}); diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index 41035a46..9749b91a 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -89,15 +89,25 @@ export function createCliAgent( await runner.install(sandbox, version, apiKey); // Stage the prompts into the sandbox scratch dir (outside the workspace). + // An empty system prompt is staged as no file at all — a CLI agent brings + // its own system prompt, and the harness only adds one when it has + // something real to say (e.g. an installed-skills listing). Runners then + // skip their system-prompt plumbing entirely rather than pointing a flag + // at an empty file or prepending a blank block to the user prompt. await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile(sandbox, SYSTEM_PROMPT_PATH, args.systemPrompt); + const systemPromptPath = args.systemPrompt + ? SYSTEM_PROMPT_PATH + : undefined; + if (systemPromptPath) { + await writeSandboxFile(sandbox, systemPromptPath, args.systemPrompt); + } await writeSandboxFile(sandbox, USER_PROMPT_PATH, args.userPrompt); const { command, raw } = await runner.exec({ sandbox, model: options.model, apiKey, - systemPromptPath: SYSTEM_PROMPT_PATH, + systemPromptPath, userPromptPath: USER_PROMPT_PATH, // Rewrite loopback hosts so in-container MCP servers can reach host-side // platform-lite; the runner writes them in its own config format. diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 13a5b3af..315b1b7b 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -86,7 +86,7 @@ describe('opencode runner', () => { /** Capture the `--model` flag, run env, and written config from one exec. */ async function captureExec( model: string, - opts: { mcp?: boolean } = {} + opts: { mcp?: boolean; systemPrompt?: boolean } = {} ): Promise<{ runCommand: string; runEnv: Record | undefined; @@ -113,7 +113,7 @@ async function captureExec( }, model, apiKey: 'gw-key', - systemPromptPath: '/s', + systemPromptPath: opts.systemPrompt === false ? undefined : '/s', userPromptPath: '/u', mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, timeoutSec: 1, @@ -139,4 +139,18 @@ describe('opencode runner exec routing', () => { expect(config?.mcp).toEqual({}); expect(runCommand).toContain('OPENCODE_CONFIG='); }); + + it('prepends the harness system prompt to the message when there is one', async () => { + // opencode has no system-prompt flag, so it lands on the user message. + const { runCommand } = await captureExec('moonshotai/kimi-k3'); + expect(runCommand).toContain(`"$(cat /s; printf '\\n\\n'; cat /u)"`); + }); + + it('sends the task alone with no system prompt (no leading blank block)', async () => { + const { runCommand } = await captureExec('moonshotai/kimi-k3', { + systemPrompt: false, + }); + expect(runCommand).toContain('"$(cat /u)"'); + expect(runCommand).not.toContain("printf '\\n\\n'"); + }); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index b7b437d9..051929bc 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -122,8 +122,11 @@ export function createOpencodeRunner( // opencode has no system-prompt flag, so prepend the system prompt to the // task; both are staged files, joined via command substitution into the - // single message argument. - const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + // single message argument. With no system prompt the task is the whole + // message — concatenating an empty one would open it with a blank block. + const message = systemPromptPath + ? `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"` + : `"$(cat ${userPromptPath})"`; await sandbox.exec(`mkdir -p ${SCRATCH}`); await writeSandboxFile( diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index baf91962..46d97674 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -54,8 +54,13 @@ export interface RunnerExecArgs { sandbox: AgentSandbox; model: M; apiKey: string; - /** Shell path to a file holding the system prompt (skills + task framing). */ - systemPromptPath: string; + /** + * Shell path to a file holding the system prompt (e.g. the installed-skills + * listing). Undefined when the harness has no system prompt for this agent — + * the runner must then leave the CLI's own prompt untouched: omit the flag, or + * for a CLI with no system-prompt flag, pass the user prompt on its own. + */ + systemPromptPath?: string; /** Shell path to a file holding the user prompt (the task). */ userPromptPath: string; /** MCP servers to expose, already loopback-rewritten. Empty when none. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4e69b3c8..8f672a65 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -371,6 +371,11 @@ export type LocalStackScorer = ( ) => Promise; export type AgentRunArgs = { + /** + * System prompt from the harness. Empty for a CLI agent the harness has + * nothing to add to — it keeps its own built-in prompt untouched, and no + * system prompt is staged into the sandbox at all. + */ systemPrompt: string; userPrompt: string; tools?: ToolSet; @@ -431,8 +436,10 @@ export type SandboxMount = { export type LocalStackSessionArgs = { /** - * The agent harness this session serves. The prompt addendum builders take it - * but do not branch on it yet; a follow-up change gates their output on it. + * The agent harness this session serves. Only `ai-sdk` calls the session's + * in-process `tools`, so only it gets the prompt addendum describing them — + * a CLI agent brings its own tools and would be told about tools it doesn't + * have. */ agent: AgentHarnessId; /** Supabase CLI version this scenario requires, overriding the runtime default. */ diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 47c4199c..4dcd8c26 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,23 +1,17 @@ import type { - AgentHarnessId, AgentSandbox, SandboxMount, SkillSource, } from '@supabase-evals/core'; import { createAgentEnvironment } from './agent-environment.js'; import { toAgentSandbox } from './local-stack-runtime.js'; -import { buildSkillsPrompt } from './skills.js'; export interface BareSandboxHandle { sandbox: AgentSandbox; - /** Skills-discovery text to fold into the agent's system prompt. */ - promptAddendum: string; close(): Promise; } export interface BareSandboxOptions { - /** Harness driving this sandbox. */ - agent: AgentHarnessId; /** Supabase CLI version baked into the sandbox image. */ cliVersion?: string; /** Skills to install into the sandbox. */ @@ -49,7 +43,6 @@ export async function createBareSandbox( }); return { sandbox: toAgentSandbox(env.sandbox), - promptAddendum: buildSkillsPrompt(options.agent, env.skills), close: env.close, }; } diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index c38d01a6..5afe609f 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -132,11 +132,17 @@ export function localStackRuntime( /** * Describes the session's tool surface: the binaries installed in the workspace * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. + * + * ai-sdk only. Those tools exist solely for `aiSdkAgent`, which the framework + * drives host-side; `createCliAgent` ignores `args.tools` entirely, so a CLI + * agent works the same workspace through its own built-in tools and this text + * would name tools it does not have. Empty string for every CLI agent. */ export function buildToolSurfaceAddendum( agent: AgentHarnessId, options: { skipCliInstall?: boolean } = {} ): string { + if (agent !== 'ai-sdk') return ''; let addendum = 'docker, psql, git, and curl are installed in the workspace. ' + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + diff --git a/packages/sandbox/src/skills.ts b/packages/sandbox/src/skills.ts index 105094da..dee601f0 100644 --- a/packages/sandbox/src/skills.ts +++ b/packages/sandbox/src/skills.ts @@ -13,11 +13,13 @@ * - `.claude/skills/` — Claude Code's project scope. * - `.agents/skills/` — Codex's and OpenCode's project scope. * - * Each CLI then discovers, advertises and loads the skills itself, in its own - * words (Codex injects a `` block, OpenCode exposes a - * `skill` tool). `buildSkillsPrompt` renders the harness's own discovery listing - * on top of that, for the in-process `ai-sdk` agent — which has no such - * mechanism — and for the CLI harnesses. + * Each CLI then advertises its own skills to the model, in its own words, with + * its own loading mechanism (Codex injects a `` block, + * OpenCode exposes a `skill` tool). The harness injects nothing: a synthetic + * "read this file with this tool" listing would both duplicate and contradict + * what the agent's own harness tells it. The one exception is the in-process + * `ai-sdk` agent, which has no such mechanism — `buildSkillsPrompt` renders the + * listing for it alone. */ import type { AgentHarnessId, SkillSource } from '@supabase-evals/core'; @@ -57,9 +59,9 @@ export const SKILLS_INSTALL_DIRS = [ ] as const; /** - * Claude Code's project scope, and the directory `buildSkillsPrompt`'s listing - * points at — it names one concrete path, read with the harness's own file - * tools. The CLI harnesses resolve their own paths natively. + * Claude Code's project scope, and the directory the `ai-sdk` discovery listing + * points at — that agent reads SKILL.md with the harness's own file tools, so it + * needs one concrete path. The CLI harnesses resolve their own paths. */ export const SKILLS_INSTALL_DIR = '.claude/skills'; @@ -73,7 +75,7 @@ export interface SkillEntry { /** * Workspace-relative directory of the installed skill in the `.claude/skills` * scope (`SKILLS_INSTALL_DIR`). The same tree exists under every entry of - * `SKILLS_INSTALL_DIRS`; this is the one `buildSkillsPrompt` cites. + * `SKILLS_INSTALL_DIRS`; this is the one the `ai-sdk` listing cites. */ dir: string; } @@ -113,12 +115,19 @@ export function frontmatterDescription(markdown: string): string { * Render the skills discovery listing: only names+descriptions enter the system * prompt, keeping context lean. When a task matches, the agent reads that * skill's SKILL.md with the existing file tools (progressive disclosure). - * Empty when no skills are installed. + * + * ai-sdk only. `files_read` is one of `buildLocalStackTools`' in-process tools, + * handed to the model by `aiSdkAgent` alone. Every CLI harness discovers the + * installed skills itself (see the module comment) and describes them to the + * model in its own words with its own loader, so injecting this would duplicate + * that listing and name a tool the agent does not have. Empty string for every + * CLI agent, and when no skills are installed. */ export function buildSkillsPrompt( agent: AgentHarnessId, skills: readonly SkillEntry[] ): string { + if (agent !== 'ai-sdk') return ''; if (skills.length === 0) return ''; return [ '## Available skills', @@ -159,7 +168,7 @@ export function buildSkillsAddCommand( * `/skills/` (the collection layout the CLI expects), then * `skills add` copies it into every project scope in `SKILLS_INSTALL_DIRS`, so * each CLI harness finds it through its own native discovery. Returns the - * installed registry (name+description+dir), used for the discovery listing. + * installed registry (name+description+dir), used for the `ai-sdk` listing. * A no-op that returns `[]` when no skills are requested. */ export async function installSkills( @@ -191,7 +200,7 @@ export async function installSkills( // Confirm each skill landed in *every* agent scope — a missing one means the // harness that reads it would silently see no skills at all — then read the - // description for the discovery listing. The name is the install directory. + // description for the ai-sdk listing. The name is the install directory. const entries: SkillEntry[] = []; for (const source of sources) { for (const installDir of SKILLS_INSTALL_DIRS) { diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index ba720794..bbba3a94 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -108,13 +108,13 @@ describe('buildSkillsPrompt', () => { expect(prompt).not.toContain('# Body'); }); - it('renders the same listing for every agent', () => { - // The listing is injected regardless of harness: each CLI also discovers - // the installed skills natively, so it hears about them twice. + it('is empty for every CLI agent — each discovers its own skills natively', () => { + // The skills CLI installs into .claude/skills and .agents/skills, which + // Claude Code, Codex and OpenCode each walk themselves; they then advertise + // the skills in their own words, with their own loader. Injecting our + // listing would duplicate theirs and name `files_read`, an ai-sdk-only tool. for (const agent of ['claude-code', 'codex', 'opencode'] as const) { - expect(buildSkillsPrompt(agent, entries)).toBe( - buildSkillsPrompt('ai-sdk', entries) - ); + expect(buildSkillsPrompt(agent, entries)).toBe(''); expect(buildSkillsPrompt(agent, [])).toBe(''); } }); @@ -228,13 +228,13 @@ describe('buildToolSurfaceAddendum', () => { expect(addendum).toContain('docker, psql, git, and curl'); }); - it('describes the same tool surface for every agent', () => { + it('is empty for every CLI agent — they never see these tools', () => { + // createCliAgent ignores `args.tools`, so a CLI agent works the workspace + // with its own built-in tools; naming ours would describe tools it lacks. for (const agent of ['claude-code', 'codex', 'opencode'] as const) { - expect(buildToolSurfaceAddendum(agent)).toBe( - buildToolSurfaceAddendum('ai-sdk') - ); + expect(buildToolSurfaceAddendum(agent)).toBe(''); expect(buildToolSurfaceAddendum(agent, { skipCliInstall: true })).toBe( - buildToolSurfaceAddendum('ai-sdk', { skipCliInstall: true }) + '' ); } });