diff --git a/README.md b/README.md index 0ae4e3f2..27af033a 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ To use a skill in an experiment, reference its directory name in the experiment' Both runtimes load skills lazily ([progressive disclosure](https://ai-sdk.dev/cookbook/guides/agent-skills)): only each skill's name+description is in the system prompt, and the agent pulls a skill's full instructions on demand. They differ only in how the body is fetched, because the tools-mode agent has no filesystem: -- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network) under `.claude/skills/`. When a task matches, the agent reads `.claude/skills//SKILL.md` (and any files it references) with its file tools. +- **Local-stack (sandbox) mode:** skills are installed into the workspace with [Vercel's `skills` CLI](https://github.com/vercel-labs/skills) (baked into the sandbox image, sourced from the local `skills/` directory — never the network), into every project scope the CLI harnesses discover natively: `.claude/skills/` for Claude Code, `.agents/skills/` for Codex and OpenCode. Each CLI discovers the scope it reads and surfaces those skills to the model itself. The framework also still injects a listing naming them, so a CLI harness hears about them twice; removing that is a separate change. - **Tools mode:** no filesystem, so a `load_skill` tool returns a skill's full instructions when the agent calls it with the skill's name. ## Framework Checks diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0ede1fc1..29dfc384 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -420,6 +420,7 @@ async function runOne( : undefined; await using session = disposable( await exp.localStack.startSession({ + agent: exp.agent.id, cliVersion: ev.metadata.cliVersion, localDir: ev.localDir, includeServices: ev.metadata.services, @@ -494,6 +495,7 @@ async function runOne( await using cliSandbox = agentRunsInSandbox ? disposable( await createBareSandbox({ + agent: exp.agent.id, skills: skillSources, mounts: supabaseMcpServerMounts(), }) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index bb327242..968707fe 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -430,6 +430,11 @@ export type SandboxMount = { }; export type LocalStackSessionArgs = { + /** + * The agent harness this session serves. The prompt addendum builders take it + * but do not branch on it yet; a follow-up change gates their output on it. + */ + agent: AgentHarnessId; /** Supabase CLI version this scenario requires, overriding the runtime default. */ cliVersion?: string; /** diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 2a3de137..85bf980b 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -38,7 +38,7 @@ export interface AgentEnvironmentOptions { cliVersion?: string; /** Host directory whose contents seed the workspace. */ localDir?: string; - /** Skills to install into the sandbox (the agent reads them with its file tools). */ + /** Skills to install into the sandbox (in every CLI harness's project scope). */ skills?: readonly SkillSource[]; /** * Run the Supabase local stack. Present → local-stack mode; omitted → tools @@ -92,7 +92,8 @@ export async function createAgentEnvironment( } else if (options.localDir) { await sandbox.copyToContainer(options.localDir, sandbox.workdir); } - // Skills are installed in both modes; the agent reads SKILL.md with its file tools. + // Skills are installed in both modes, into every CLI harness's native + // project scope (see installSkills) so each agent discovers them itself. const skills = await installSkills(sandbox, options.skills ?? []); return { sandbox, skills, close: () => sandbox.stop() }; } catch (err) { diff --git a/packages/sandbox/src/bare-sandbox.ts b/packages/sandbox/src/bare-sandbox.ts index 4c51a14d..47c4199c 100644 --- a/packages/sandbox/src/bare-sandbox.ts +++ b/packages/sandbox/src/bare-sandbox.ts @@ -1,4 +1,5 @@ import type { + AgentHarnessId, AgentSandbox, SandboxMount, SkillSource, @@ -14,6 +15,21 @@ export interface BareSandboxHandle { close(): Promise; } +export interface BareSandboxOptions { + /** Harness driving this sandbox. */ + agent: AgentHarnessId; + /** Supabase CLI version baked into the sandbox image. */ + cliVersion?: string; + /** Skills to install into the sandbox. */ + skills?: readonly SkillSource[]; + /** + * Extra host directories bind-mounted into the sandbox (read-only by + * default) — e.g. a local MCP server build the in-container agent must be + * able to launch. See `supabaseMcpServerMounts`. + */ + mounts?: readonly SandboxMount[]; +} + /** * The agent's execution environment for tools mode: the shared agent * environment (image, tooling, skills) **without** the Supabase local stack. @@ -24,11 +40,7 @@ export interface BareSandboxHandle { * platform-lite via `host.docker.internal` on the default bridge). */ export async function createBareSandbox( - options: { - cliVersion?: string; - skills?: readonly SkillSource[]; - mounts?: readonly SandboxMount[]; - } = {} + options: BareSandboxOptions ): Promise { const env = await createAgentEnvironment({ cliVersion: options.cliVersion, @@ -37,7 +49,7 @@ export async function createBareSandbox( }); return { sandbox: toAgentSandbox(env.sandbox), - promptAddendum: buildSkillsPrompt(env.skills), + promptAddendum: buildSkillsPrompt(options.agent, env.skills), close: env.close, }; } diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index cf486449..793a5e5e 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -19,13 +19,17 @@ export type { SetupSupabaseSandboxOptions } from './supabase.js'; export { buildLocalStackScoringContext, buildLocalStackTools, + buildToolSurfaceAddendum, localStackRuntime, toAgentSandbox, } from './local-stack-runtime.js'; export type { LocalStackRuntimeOptions } from './local-stack-runtime.js'; export { SKILLS_CLI_VERSION, + SKILLS_INSTALL_AGENTS, SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, + buildSkillsAddCommand, buildSkillsPrompt, frontmatterDescription, installSkills, @@ -33,7 +37,7 @@ export { } from './skills.js'; export type { SkillEntry } from './skills.js'; export { createBareSandbox } from './bare-sandbox.js'; -export type { BareSandboxHandle } from './bare-sandbox.js'; +export type { BareSandboxHandle, BareSandboxOptions } from './bare-sandbox.js'; export { createAgentEnvironment } from './agent-environment.js'; export type { AgentEnvironment, diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 0bed4432..c38d01a6 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -3,6 +3,7 @@ import { jsonSchema, tool, type ToolSet } from 'ai'; import { createClient } from '@supabase/supabase-js'; import { supabaseMcpServer, + type AgentHarnessId, type AgentSandbox, type HostedLink, type LocalStackRuntime, @@ -73,6 +74,7 @@ export function localStackRuntime( return { id: 'local-stack', async startSession({ + agent, cliVersion, localDir, includeServices, @@ -105,22 +107,14 @@ export function localStackRuntime( const mcpServers = await resolveMcpServers(options, hosted); - let baseAddendum = - 'docker, psql, git, and curl are installed in the workspace. ' + - 'Use the bash tool to run commands (the working directory is always the workspace root) ' + - 'and the files tools to inspect and modify files.'; - - if (!skipCliInstall) { - baseAddendum = 'The Supabase CLI (`supabase`), ' + baseAddendum; - baseAddendum += - ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; - } - return { tools: buildLocalStackTools(sandbox), sandbox: toAgentSandbox(sandbox), mcpServers, - promptAddendum: [baseAddendum, buildSkillsPrompt(env.skills)] + promptAddendum: [ + buildToolSurfaceAddendum(agent, { skipCliInstall }), + buildSkillsPrompt(agent, env.skills), + ] .filter(Boolean) .join('\n\n'), scoringContext: buildLocalStackScoringContext(sandbox, hosted), @@ -135,6 +129,27 @@ export function localStackRuntime( }; } +/** + * Describes the session's tool surface: the binaries installed in the workspace + * and the in-process `bash`/`files_*` tools from `buildLocalStackTools`. + */ +export function buildToolSurfaceAddendum( + agent: AgentHarnessId, + options: { skipCliInstall?: boolean } = {} +): string { + let addendum = + 'docker, psql, git, and curl are installed in the workspace. ' + + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + + 'and the files tools to inspect and modify files.'; + + if (!options.skipCliInstall) { + addendum = 'The Supabase CLI (`supabase`), ' + addendum; + addendum += + ' Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + } + return addendum; +} + /** * Build the MCP server map for a session. An explicit `options.mcpServers` * wins. Otherwise, when the eval links to a hosted project, expose a Supabase diff --git a/packages/sandbox/src/skills.ts b/packages/sandbox/src/skills.ts index 37f48248..809a891b 100644 --- a/packages/sandbox/src/skills.ts +++ b/packages/sandbox/src/skills.ts @@ -1,36 +1,92 @@ /** - * Agent skills inside the local-stack sandbox. + * Agent skills inside the sandbox. * * Skills are reusable instruction sets (a SKILL.md plus bundled reference - * files) that the agent discovers and loads on demand — the AI SDK - * "agent skills" pattern (https://ai-sdk.dev/cookbook/guides/agent-skills). - * Rather than preloading every skill's full text into the system prompt, the - * sandbox advertises only each skill's name+description and tells the agent to - * read a skill's SKILL.md (with the existing file tools) when a task matches — - * progressive disclosure. This only works where the agent has a filesystem — - * the sandbox. Tools-mode evals (no filesystem) inject skills into the system - * prompt instead. + * files) that the agent discovers and loads on demand — progressive disclosure, + * rather than preloading every skill's full text into the system prompt. * * Skills are installed with Vercel's `skills` CLI (baked into the sandbox * image), sourcing from local directories — never the network. Each requested - * skill is staged outside the workspace, then `skills add` copies it into the - * workspace's `.claude/skills/` (claude-code project scope), where the agent's - * file tools can reach the SKILL.md and the files it references. + * skill is staged outside the workspace, then `skills add --agent …` copies it + * into every project scope the CLI harnesses discover natively: + * + * - `.claude/skills/` — Claude Code's project scope. + * - `.agents/skills/` — Codex's and OpenCode's project scope. + * + * Each CLI then discovers, advertises and loads the skills itself, in its own + * words (Codex injects a `` block, OpenCode exposes a + * `skill` tool). `buildSkillsPrompt` renders the harness's own discovery listing + * on top of that, for the in-process `ai-sdk` agent — which has no such + * mechanism — and for the CLI harnesses. */ -import type { SkillSource } from '@supabase-evals/core'; +import { agentHarnessIdSchema } from '@supabase-evals/core'; +import type { AgentHarnessId, SkillSource } from '@supabase-evals/core'; import type { DockerSandbox } from './docker-sandbox.js'; /** Version of Vercel's `skills` CLI baked into the sandbox image (pinned). */ export const SKILLS_CLI_VERSION = '1.5.11'; +/** Claude Code's project scope. */ +const CLAUDE_CODE_SKILLS_DIR = '.claude/skills'; +/** Codex's and OpenCode's shared project scope. */ +const AGENTS_SKILLS_DIR = '.agents/skills'; + +/** + * The workspace-relative project scope each harness discovers skills in, or + * `null` for one that has none. Everything below is derived from this map, and + * the `Record` makes it exhaustive: adding a harness to + * `agentHarnessIdSchema` fails this file to compile until its scope is + * declared, rather than silently running that harness's evals skill-less. + * + * `ai-sdk` is `null` — it runs in-process with no filesystem scope of its own, + * and is served by `buildSkillsPrompt`'s listing instead. + */ +const SKILLS_PATH_BY_AGENT: Record = { + 'ai-sdk': null, + 'claude-code': CLAUDE_CODE_SKILLS_DIR, + codex: AGENTS_SKILLS_DIR, + opencode: AGENTS_SKILLS_DIR, +}; + +const installAgents: AgentHarnessId[] = []; +const installDirs: string[] = []; +for (const id of agentHarnessIdSchema.options) { + const dir = SKILLS_PATH_BY_AGENT[id]; + if (dir === null) continue; + installAgents.push(id); + if (!installDirs.includes(dir)) installDirs.push(dir); +} + +/** + * `skills add --agent` ids we install for — every harness with a project scope. + * Installed unconditionally rather than only for the experiment's own harness: + * the ids collapse to just two directories, an unused one costs a directory + * copy of a few kilobytes, and keeping one code path means no agent id has to + * be threaded through `createAgentEnvironment` for correctness. + * + * Naming them explicitly also matters. With no `--agent` flag the CLI falls + * back to *every* agent it knows when it cannot detect an installed one, + * littering the scored, exported workspace with ~52 stray entries. That + * fallback does happen to cover both directories we want, so it is not why + * skills reach an agent; it is a workspace-pollution and determinism problem. + * Detection depends on the surrounding environment, so naming the agents is + * what makes the install predictable. + */ +export const SKILLS_INSTALL_AGENTS: readonly AgentHarnessId[] = installAgents; + /** - * Where installed project-scoped skills are read from, relative to the - * workspace root (the CLI's cwd during install). We install for all agents - * (see installSkills), and `.claude/skills` is claude-code's project scope — - * the discovery listing points the agent here. + * Every workspace-relative directory `SKILLS_INSTALL_AGENTS` populates, deduped + * (`codex` and `opencode` share `.agents/skills`). Verified after install. */ -export const SKILLS_INSTALL_DIR = '.claude/skills'; +export const SKILLS_INSTALL_DIRS: readonly string[] = installDirs; + +/** + * The directory `buildSkillsPrompt`'s listing points at — it names one concrete + * path, read with the harness's own file tools. The CLI harnesses resolve their + * own paths natively. + */ +export const SKILLS_INSTALL_DIR = CLAUDE_CODE_SKILLS_DIR; /** Staging path (outside the workspace) host skill sources are copied to before install. */ const SKILLS_STAGING_DIR = '/tmp/skills-src'; @@ -39,7 +95,11 @@ const SKILLS_STAGING_DIR = '/tmp/skills-src'; export interface SkillEntry { name: string; description: string; - /** Workspace-relative directory of the installed skill. */ + /** + * Workspace-relative directory of the installed skill in the `.claude/skills` + * scope (`SKILLS_INSTALL_DIR`). The same tree exists under every entry of + * `SKILLS_INSTALL_DIRS`; this is the one `buildSkillsPrompt` cites. + */ dir: string; } @@ -75,12 +135,15 @@ export function frontmatterDescription(markdown: string): string { } /** - * Render the discovery prompt: only names+descriptions enter the system + * Render the skills discovery listing: only names+descriptions enter the system * prompt, keeping context lean. When a task matches, the agent reads that * skill's SKILL.md with the existing file tools (progressive disclosure). * Empty when no skills are installed. */ -export function buildSkillsPrompt(skills: readonly SkillEntry[]): string { +export function buildSkillsPrompt( + agent: AgentHarnessId, + skills: readonly SkillEntry[] +): string { if (skills.length === 0) return ''; return [ '## Available skills', @@ -94,12 +157,34 @@ export function buildSkillsPrompt(skills: readonly SkillEntry[]): string { ].join('\n'); } +/** + * The `skills add` invocation, as a string, so its shape is unit-testable + * without a container. + * + * Argument order is load-bearing: `--agent` is variadic (it consumes every + * following token that does not start with `-`), so the source directory must + * come *before* it — `skills add --agent codex ` swallows `` as an + * agent name and fails with "Missing required argument: source". `--skill` + * immediately after the agent list terminates it. + * + * `--copy` (rather than the default symlink) keeps the workspace self-contained + * once staging is gone, and skips the CLI's symlink-mode "only install if the + * agent's top-level directory already exists" branch. `--skill '*'` installs + * every staged skill; `--yes` is non-interactive. + */ +export function buildSkillsAddCommand( + stagingDir: string = SKILLS_STAGING_DIR +): string { + return `skills add '${stagingDir}' --agent ${SKILLS_INSTALL_AGENTS.join(' ')} --skill '*' --copy --yes`; +} + /** * Install agent skills into the sandbox with Vercel's `skills` CLI, sourcing * from local directories (never the network). Each skill is staged under * `/skills/` (the collection layout the CLI expects), then - * `skills add` copies it into the workspace's `.claude/skills/`. Returns the - * discovered registry (name+description+dir) used for progressive disclosure. + * `skills add` copies it into every project scope in `SKILLS_INSTALL_DIRS`, so + * each CLI harness finds it through its own native discovery. Returns the + * installed registry (name+description+dir), used for the discovery listing. * A no-op that returns `[]` when no skills are requested. */ export async function installSkills( @@ -120,38 +205,34 @@ export async function installSkills( ); } - // `skills add ` installs to the cwd's project scope, and runShell's cwd - // is the workspace, so skills land in //. --copy (not - // symlink) keeps the workspace self-contained once staging is gone; --skill - // '*' installs all staged skills; --yes is non-interactive. - // - // TODO: install only for the agent the experiment uses (--agent claude-code, - // codex, gemini-cli, …) once that is threaded through. We only run models via - // the AI SDK today, so for now we install for all agents and read claude-code's - // .claude/skills scope (SKILLS_INSTALL_DIR). - const install = await sandbox.runShell( - `skills add ${SKILLS_STAGING_DIR} --skill '*' --copy --yes` - ); + // `skills add ` installs into the cwd's project scopes, and runShell's + // cwd is the workspace, so skills land in /{.claude,.agents}/skills. + const install = await sandbox.runShell(buildSkillsAddCommand()); if (!install.ok) { throw new Error( `failed to install skills with the skills CLI: ${install.stderr || install.stdout}` ); } - // Confirm what landed and read each skill's description for the discovery - // listing. The name is the install directory; only the description is read. + // Confirm each skill landed in *every* agent scope — a missing one means the + // harness that reads it would silently see no skills at all — then read the + // description for the discovery listing. The name is the install directory. const entries: SkillEntry[] = []; for (const source of sources) { - const dir = `${SKILLS_INSTALL_DIR}/${source.name}`; - const skillPath = `${dir}/SKILL.md`; - if (!(await sandbox.fileExists(skillPath))) { - throw new Error( - `skills CLI did not install "${source.name}" (no ${skillPath} in the sandbox)` - ); + for (const installDir of SKILLS_INSTALL_DIRS) { + const skillPath = `${installDir}/${source.name}/SKILL.md`; + if (!(await sandbox.fileExists(skillPath))) { + throw new Error( + `skills CLI did not install "${source.name}" (no ${skillPath} in the sandbox)` + ); + } } + const dir = `${SKILLS_INSTALL_DIR}/${source.name}`; entries.push({ name: source.name, - description: frontmatterDescription(await sandbox.readFile(skillPath)), + description: frontmatterDescription( + await sandbox.readFile(`${dir}/SKILL.md`) + ), dir, }); } diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index 5107b2a3..bba78d49 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -18,7 +18,11 @@ import { SUPABASE_CLI_VERSION, teardownSupabaseProject, } from '../src/supabase.js'; -import { installSkills, SKILLS_INSTALL_DIR } from '../src/skills.js'; +import { + installSkills, + SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, +} from '../src/skills.js'; const TEST_TIMEOUT_MS = 600_000; @@ -200,18 +204,33 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( dir: `${SKILLS_INSTALL_DIR}/demo-skill`, }, ]); - // The full skill tree (including bundled references) is reachable in - // the workspace via the agent's file tools. - expect( - await sandbox.fileExists( - `${SKILLS_INSTALL_DIR}/demo-skill/SKILL.md` - ) - ).toBe(true); - expect( - await sandbox.readFile( - `${SKILLS_INSTALL_DIR}/demo-skill/references/extra.md` - ) - ).toBe('extra content'); + // The full skill tree (including bundled references) lands in every + // CLI harness's native project scope: .claude/skills for Claude Code, + // .agents/skills for Codex and OpenCode. + for (const installDir of SKILLS_INSTALL_DIRS) { + expect( + await sandbox.fileExists(`${installDir}/demo-skill/SKILL.md`) + ).toBe(true); + expect( + await sandbox.readFile( + `${installDir}/demo-skill/references/extra.md` + ) + ).toBe('extra content'); + } + // …and not in the scopes the no-`--agent` fallback would create. That + // fallback installs for every agent the CLI knows, littering the + // exported, scored workspace with ~52 stray roots. This is a sample of + // them, not the whole set, so it catches the fallback firing rather + // than proving nothing else was written. + for (const stray of [ + '.aider-desk', + '.factory', + '.windsurf', + 'data', + 'skills', + ]) { + expect(await sandbox.folderExists(stray)).toBe(false); + } } finally { rmSync(src, { recursive: true, force: true }); await sandbox.stop(); diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index d8596a58..ba720794 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { + buildToolSurfaceAddendum, resolveSandboxPath, truncateOutput, wrapSelectAsJson, @@ -17,9 +18,13 @@ import { import type { DockerSandbox } from '../src/docker-sandbox.js'; import { SKILLS_CLI_VERSION, + SKILLS_INSTALL_AGENTS, SKILLS_INSTALL_DIR, + SKILLS_INSTALL_DIRS, + buildSkillsAddCommand, buildSkillsPrompt, frontmatterDescription, + installSkills, } from '../src/skills.js'; import { ALL_SUPABASE_SERVICES } from '../src/types.js'; @@ -83,15 +88,17 @@ describe('frontmatterDescription', () => { }); describe('buildSkillsPrompt', () => { + const entries = [ + { name: 'supabase', description: 'Use for Supabase tasks.', dir: 'x' }, + { name: 'pg', description: 'Postgres tips.', dir: 'y' }, + ]; + it('is empty when no skills are installed', () => { - expect(buildSkillsPrompt([])).toBe(''); + expect(buildSkillsPrompt('ai-sdk', [])).toBe(''); }); it('lists name+description and points at the install dir for files_read', () => { - const prompt = buildSkillsPrompt([ - { name: 'supabase', description: 'Use for Supabase tasks.', dir: 'x' }, - { name: 'pg', description: 'Postgres tips.', dir: 'y' }, - ]); + const prompt = buildSkillsPrompt('ai-sdk', entries); expect(prompt).toContain(SKILLS_INSTALL_DIR); expect(prompt).toContain('files_read'); expect(prompt).toContain('SKILL.md'); @@ -100,6 +107,137 @@ describe('buildSkillsPrompt', () => { // Discovery only — the full body must not be inlined here. expect(prompt).not.toContain('# Body'); }); + + it('renders the same listing for every agent', () => { + // The listing is injected regardless of harness: each CLI also discovers + // the installed skills natively, so it hears about them twice. + for (const agent of ['claude-code', 'codex', 'opencode'] as const) { + expect(buildSkillsPrompt(agent, entries)).toBe( + buildSkillsPrompt('ai-sdk', entries) + ); + expect(buildSkillsPrompt(agent, [])).toBe(''); + } + }); +}); + +describe('buildSkillsAddCommand', () => { + it('installs for all three CLI harnesses, source before the variadic --agent', () => { + const command = buildSkillsAddCommand('/tmp/staging'); + expect(command).toBe( + "skills add '/tmp/staging' --agent claude-code codex opencode --skill '*' --copy --yes" + ); + // --agent is variadic: it eats every following non-flag token. The source + // dir must precede it (otherwise the CLI fails with "Missing required + // argument: source") and a flag must terminate the agent list. + expect(command.indexOf('/tmp/staging')).toBeLessThan( + command.indexOf('--agent') + ); + expect(command).toMatch(/--agent (?:[a-z-]+ )+--skill/); + }); + + it('names the agents explicitly rather than letting the CLI guess', () => { + // With no --agent the CLI falls back to all ~71 agents it knows, littering + // the exported workspace; the fallback is also install-order dependent. + expect(SKILLS_INSTALL_AGENTS).toEqual(['claude-code', 'codex', 'opencode']); + expect(buildSkillsAddCommand()).toContain('--agent'); + }); +}); + +describe('installSkills', () => { + /** A DockerSandbox stub that records shell commands and fakes the install. */ + function fakeSandbox(present: readonly string[]) { + const commands: string[] = []; + return { + commands, + sandbox: { + runShellAsRoot: async (command: string) => { + commands.push(command); + return { ok: true, exitCode: 0, stdout: '', stderr: '' }; + }, + runShell: async (command: string) => { + commands.push(command); + return { ok: true, exitCode: 0, stdout: '', stderr: '' }; + }, + copyToContainer: async () => {}, + fileExists: async (path: string) => present.includes(path), + readFile: async () => '---\ndescription: Demo skill.\n---\nbody', + } as unknown as DockerSandbox, + }; + } + + const installedEverywhere = SKILLS_INSTALL_DIRS.map( + (dir) => `${dir}/demo/SKILL.md` + ); + + it('is a no-op with no skills requested', async () => { + const { sandbox, commands } = fakeSandbox([]); + expect(await installSkills(sandbox, [])).toEqual([]); + expect(commands).toEqual([]); + }); + + it('runs the per-agent install and reports the .claude/skills tree', async () => { + const { sandbox, commands } = fakeSandbox(installedEverywhere); + const entries = await installSkills(sandbox, [ + { name: 'demo', dir: '/host/demo' }, + ]); + expect(entries).toEqual([ + { + name: 'demo', + description: 'Demo skill.', + dir: `${SKILLS_INSTALL_DIR}/demo`, + }, + ]); + expect(commands).toContain(buildSkillsAddCommand()); + }); + + it('installs into both .claude/skills and .agents/skills', () => { + // .claude/skills is Claude Code's project scope; .agents/skills is Codex's + // and OpenCode's. Codex does not read .claude/skills at all. + expect(SKILLS_INSTALL_DIRS).toEqual(['.claude/skills', '.agents/skills']); + expect(SKILLS_INSTALL_DIR).toBe('.claude/skills'); + }); + + it('throws when a skill is missing from any agent scope', async () => { + for (const missing of SKILLS_INSTALL_DIRS) { + const { sandbox } = fakeSandbox( + installedEverywhere.filter((p) => !p.startsWith(`${missing}/`)) + ); + await expect( + installSkills(sandbox, [{ name: 'demo', dir: '/host/demo' }]) + ).rejects.toThrow(`no ${missing}/demo/SKILL.md`); + } + }); +}); + +describe('buildToolSurfaceAddendum', () => { + it('describes the in-process tool surface for the ai-sdk agent', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk'); + // These are the tools buildLocalStackTools actually provides. + expect(addendum).toContain('bash tool'); + expect(addendum).toContain('files tools'); + expect(addendum).toContain('The Supabase CLI (`supabase`)'); + expect(addendum).toContain('supabase start'); + }); + + it('drops the CLI sentence when the agent installs the CLI itself', () => { + const addendum = buildToolSurfaceAddendum('ai-sdk', { + skipCliInstall: true, + }); + expect(addendum).not.toContain('The Supabase CLI (`supabase`)'); + expect(addendum).not.toContain('supabase start'); + expect(addendum).toContain('docker, psql, git, and curl'); + }); + + it('describes the same tool surface for every agent', () => { + for (const agent of ['claude-code', 'codex', 'opencode'] as const) { + expect(buildToolSurfaceAddendum(agent)).toBe( + buildToolSurfaceAddendum('ai-sdk') + ); + expect(buildToolSurfaceAddendum(agent, { skipCliInstall: true })).toBe( + buildToolSurfaceAddendum('ai-sdk', { skipCliInstall: true }) + ); + } + }); }); describe('SKILLS_CLI_VERSION', () => {