fix(framework): remove the harness system prompt for CLI agents (AI-1034) #241
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| import { describe, expect, it } from 'vitest'; | ||
| import type { AgentHarnessId } from '@supabase-evals/core'; | ||
| import { | ||
| buildSkillsPrompt, | ||
| buildToolSurfaceAddendum, | ||
| type SkillEntry, | ||
| } from '@supabase-evals/sandbox'; | ||
| import { buildSystemPrompt } from './system-prompt.js'; | ||
| import type { EvalMode } from './types.js'; | ||
|
|
||
| const CLI_AGENTS: AgentHarnessId[] = ['claude-code', 'codex', 'opencode']; | ||
| const MODES: EvalMode[] = ['tools', 'local-stack']; | ||
|
|
||
| describe('buildSystemPrompt', () => { | ||
| it('gives the ai-sdk agent task framing in both modes', () => { | ||
| // ai-sdk is the one harness with no system prompt of its own, so it's the | ||
| // one harness the framework has to supply one for. | ||
| for (const mode of MODES) { | ||
| expect(buildSystemPrompt('ai-sdk', mode)).toContain( | ||
| 'Use the provided tools' | ||
| ); | ||
| } | ||
| }); | ||
|
|
||
| it('gives no framing of our own to any CLI harness', () => { | ||
| // CLI harnesses ship their own system prompt; we're measuring that. | ||
| for (const agent of CLI_AGENTS) { | ||
| for (const mode of MODES) { | ||
| expect(buildSystemPrompt(agent, mode)).toBe(''); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| it('refuses blocks a caller hands it for a CLI harness', () => { | ||
| // The producers gate their own output, so a non-empty block here means an | ||
| // experiment is misconfigured. Dropping it would be as silent as injecting | ||
| // it, and the block may be load-bearing: `executorMcpServer`'s addendum is | ||
| // the pause/resume protocol its tools require. | ||
| for (const agent of CLI_AGENTS) { | ||
| for (const mode of MODES) { | ||
| expect(() => buildSystemPrompt(agent, mode, 'Addendum.')).toThrow( | ||
| /must receive no system prompt/ | ||
| ); | ||
| expect(() => | ||
| buildSystemPrompt(agent, mode, undefined, 'Skills listing.') | ||
| ).toThrow(/must receive no system prompt/); | ||
| // Empty and absent blocks are the normal case, not a misconfiguration. | ||
| expect(buildSystemPrompt(agent, mode, '', '')).toBe(''); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| it('keeps the runtime blocks for ai-sdk, in order, after the base prompt', () => { | ||
| const base = buildSystemPrompt('ai-sdk', 'local-stack'); | ||
| expect( | ||
| buildSystemPrompt('ai-sdk', 'local-stack', 'Addendum.', 'Skills listing.') | ||
| ).toBe(`${base}\n\nAddendum.\n\nSkills listing.`); | ||
| }); | ||
|
|
||
| it('assembles to nothing at all for a CLI harness, even with skills', () => { | ||
| // The real block producers, not stand-ins: with skills installed, a CLI | ||
| // harness must still receive an entirely empty system prompt. Codex and | ||
| // OpenCode find the skills through their own project-scope discovery and | ||
| // describe them to the model themselves. | ||
| const skills: SkillEntry[] = [ | ||
| { | ||
| name: 'supabase', | ||
| description: 'Use for Supabase tasks.', | ||
| dir: '.claude/skills/supabase', | ||
| }, | ||
| ]; | ||
| for (const agent of CLI_AGENTS) { | ||
| expect( | ||
| buildSystemPrompt( | ||
| agent, | ||
| 'local-stack', | ||
| buildToolSurfaceAddendum(agent), | ||
| buildSkillsPrompt(agent, skills) | ||
| ) | ||
| ).toBe(''); | ||
| } | ||
| // ai-sdk has no such mechanism — it only learns about skills from us. | ||
| const aiSdk = buildSystemPrompt( | ||
| 'ai-sdk', | ||
| 'local-stack', | ||
| buildToolSurfaceAddendum('ai-sdk'), | ||
| buildSkillsPrompt('ai-sdk', skills) | ||
| ); | ||
| expect(aiSdk).toContain('## Available skills'); | ||
| expect(aiSdk).toContain('- supabase: Use for Supabase tasks.'); | ||
| }); | ||
|
|
||
| it('never tells any agent how to end its turn', () => { | ||
| // Stopping behaviour is part of what an eval measures, so the harness must | ||
| // not coach it (e.g. "end your turn with a short summary"). | ||
| for (const agent of [...CLI_AGENTS, 'ai-sdk' as const]) { | ||
| for (const mode of MODES) { | ||
| const prompt = buildSystemPrompt(agent, mode); | ||
| expect(prompt).not.toMatch(/summary/i); | ||
| expect(prompt).not.toMatch(/end your turn/i); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| it('drops empty blocks instead of leaving blank gaps', () => { | ||
| expect(buildSystemPrompt('ai-sdk', 'tools', '', 'Skills listing.')).toBe( | ||
| `${buildSystemPrompt('ai-sdk', 'tools')}\n\nSkills listing.` | ||
| ); | ||
| expect(buildSystemPrompt('ai-sdk', 'tools', '', '')).not.toMatch(/\n\n$/); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| /** | ||
| * System-prompt assembly, per agent harness. | ||
| * | ||
| * An eval measures out-of-the-box agent behaviour, so the harness injects as | ||
| * little prompt of its own as it can get away with: only the ai-sdk agent gets | ||
| * any base framing, because it is the only harness with no system prompt of its | ||
| * own (`aiSdkAgent` hands `systemPrompt` straight to the model's `system`). CLI | ||
| * agents ship their own coding-agent prompt, tool guidance, and stopping | ||
| * behaviour — and codex/opencode have no system-prompt flag at all, so anything | ||
| * we pass them lands on the *user* prompt. | ||
| */ | ||
|
|
||
| import type { AgentHarnessId } from '@supabase-evals/core'; | ||
| import type { EvalMode } from './types.js'; | ||
|
|
||
| /** | ||
| * Base framing for the ai-sdk harness: what it can't infer on its own — that it | ||
| * has tools, and what they act on. Deliberately silent on how to finish a turn | ||
| * (no "end with a summary"): stopping behaviour is part of what's measured. | ||
| * Empty for every CLI harness. | ||
| */ | ||
| function basePromptFor(agent: AgentHarnessId, mode: EvalMode): string { | ||
| if (agent !== 'ai-sdk') return ''; | ||
| if (mode === 'local-stack') { | ||
| return ( | ||
| 'You are an agent solving a Supabase eval task in a Linux workspace. ' + | ||
|
Collaborator
There was a problem hiding this comment. Could we also tie in the fix for #164 here? I.e. we shouldn't mention that this is a Supabase eval or that there is a "project" needing to be modified, as that could give an unfair advantage to the non terminal-agent experiments. I recognize it's a minor issue at this point, since only the AI SDK harness is affected here, which is no longer used in our benchmarks / regression runs. Might end up taking it out entirely. But also easy to clean up while we're here. |
||
| 'Use the provided tools to inspect and modify the workspace and run commands.' | ||
| ); | ||
| } | ||
| return ( | ||
| 'You are an agent solving a Supabase eval task. ' + | ||
| 'Use the provided tools to inspect and modify the project.' | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Assemble the system prompt handed to the agent. Every block is ai-sdk-only — | ||
| * the base framing, the tool-surface addendum, the skills listing — so a CLI | ||
| * harness ends up with `''`, and the CLI engine then stages no system-prompt | ||
| * file at all rather than an empty one. | ||
| * | ||
| * The callers' blocks are already gated by their producers, so a non-empty one | ||
| * arriving here means an experiment is misconfigured. Throw rather than drop it: | ||
| * dropping is as silent as injecting, and the block may be load-bearing. An MCP | ||
| * server carrying a `promptAddendum` is the live path in. Only | ||
| * `executorMcpServer` has one, and its text is the pause/resume protocol its | ||
| * tools require, not a tool description. A CLI harness paired with it would get | ||
| * the tools and none of the protocol, then stall on the first paused execution | ||
| * with a recorded prompt of `''` explaining nothing. | ||
| */ | ||
| export function buildSystemPrompt( | ||
| agent: AgentHarnessId, | ||
| mode: EvalMode, | ||
| addendum?: string, | ||
| skillContext?: string | ||
| ): string { | ||
| if (agent !== 'ai-sdk') { | ||
| for (const [arg, block] of [ | ||
| ['addendum', addendum], | ||
| ['skillContext', skillContext], | ||
| ] as const) { | ||
| if (block) { | ||
| throw new Error( | ||
| `buildSystemPrompt got a non-empty ${arg} for '${agent}', which must receive ` + | ||
| 'no system prompt. Whichever runtime or MCP server produced it is not gated ' + | ||
| 'on the agent harness.' | ||
| ); | ||
| } | ||
| } | ||
| return ''; | ||
| } | ||
| const blocks = [basePromptFor(agent, mode), addendum, skillContext].filter( | ||
| Boolean | ||
| ); | ||
| return blocks.join('\n\n'); | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If we end up keeping this for terminal agents, I'd clarify here that it's more of an optional thing appended to the system prompt, not replacing it with this exact string alone.