feat(core): launch a local mcp build via SUPABASE_MCP_SERVER_PATH

apps/framework/harness/run-eval.ts

@@ -34,6 +34,7 @@ import {
   buildSkillResult,
   rehydrateTruncatedDocsResults,
   getExperimentDisplayMetadata,
+  supabaseMcpServerMounts,
 } from '@supabase-evals/core';
 import type {
   ExperimentConfig,
@@ -431,6 +432,7 @@ async function runOne(
               }
             : undefined,
           skills: skillSources,
+          mounts: supabaseMcpServerMounts(),
           skipCliInstall: ev.metadata.skipCliInstall,
         })
       );
@@ -500,7 +502,12 @@ async function runOne(
     // platform-lite via host.docker.internal (so platform-lite binds 0.0.0.0).
     // An in-process agent runs host-side with no sandbox.
     await using cliSandbox = agentRunsInSandbox
-      ? disposable(await createBareSandbox({ skills: skillSources }))
+      ? disposable(
+          await createBareSandbox({
+            skills: skillSources,
+            mounts: supabaseMcpServerMounts(),
+          })
+        )
       : undefined;
     await using session = disposable(
       await exp.runtime.startSession({

packages/core/src/index.ts

@@ -1,11 +1,17 @@
 import vm from 'node:vm';
 import { createRequire } from 'node:module';
 import { createHash, createHmac } from 'node:crypto';
-import { execFile } from 'node:child_process';
+import { execFile, execFileSync } from 'node:child_process';
 import { createServer } from 'node:net';
 import { promisify } from 'node:util';
-import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
-import { basename, dirname, join } from 'node:path';
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+} from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
 import { fileURLToPath } from 'node:url';
 import type { ToolCall, ToolName } from './transcript/types.js';
@@ -409,6 +415,20 @@ export type AgentHarness = {
  */
 export type SkillSource = { name: string; dir: string };
 
+/**
+ * A host directory bind-mounted into the agent sandbox. Read-only by default;
+ * mounted at the identical container path unless `containerPath` overrides it
+ * (identical paths let one command config work on both host and container).
+ */
+export type SandboxMount = {
+  /** Host directory to mount. */
+  hostPath: string;
+  /** Mount point inside the container; defaults to `hostPath`. */
+  containerPath?: string;
+  /** Mount read-only (default true). */
+  readonly?: boolean;
+};
+
 export type LocalStackSessionArgs = {
   /** Supabase CLI version this scenario requires, overriding the runtime default. */
   cliVersion?: string;
@@ -444,6 +464,12 @@ export type LocalStackSessionArgs = {
    * instead, so they ignore this.
    */
   skills?: readonly SkillSource[];
+  /**
+   * Extra host directories to bind-mount into the sandbox (read-only by
+   * default) — e.g. a local MCP server build the in-container agent must be
+   * able to launch. See `supabaseMcpServerMounts`.
+   */
+  mounts?: readonly SandboxMount[];
   /**
    * Skip installing the real Supabase CLI into the sandbox (from
    * `skipCliInstall:` frontmatter), for scenarios whose prompt has the agent
@@ -929,8 +955,9 @@ export function supabaseMcpServer(
   return {
     name: 'supabase-mcp',
     async createConfig({ apiUrl, accessToken } = {}) {
-      const args = [
-        `@supabase/mcp-server-supabase@${version}`,
+      // Server flags are identical whether we launch the published package via
+      // npx or a local build directly with node.
+      const serverArgs = [
         // The server refuses to boot without a token; with only platform-
         // independent features (docs) it never authenticates against the
         // management API, so a well-formed throwaway is enough.
@@ -942,12 +969,111 @@ export function supabaseMcpServer(
       // Only point the server at a platform when one is given. `docs` is
       // platform-independent (it queries the public docs GraphQL API), so a
       // docs-only server runs standalone with no `--api-url`.
-      if (apiUrl) args.push('--api-url', apiUrl);
-      return { config: { command: 'npx', args } };
+      if (apiUrl) serverArgs.push('--api-url', apiUrl);
+
+      const local = resolveLocalMcpServer();
+      if (local) {
+        // `node`, not process.execPath: CLI agents run this command INSIDE the
+        // sandbox container, where the host's node binary path does not exist.
+        // Both container and host resolve `node` via PATH.
+        return {
+          config: { command: 'node', args: [local.entry, ...serverArgs] },
+        };
+      }
+
+      return {
+        config: {
+          command: 'npx',
+          args: [`@supabase/mcp-server-supabase@${version}`, ...serverArgs],
+        },
+      };
     },
   };
 }
 
+/**
+ * SUPABASE_MCP_SERVER_PATH swaps the published npx package for a local build
+ * (a repo/package dir or a direct .js/.mjs/.cjs entrypoint), so a workspace
+ * can test an unpublished server change without publishing to npm. Relative
+ * paths resolve against the evals checkout root (not the process CWD), so
+ * `submodules/mcp/packages/mcp-server-supabase` works from any directory.
+ *
+ * Memoized per env value: createConfig and the sandbox mounts both resolve,
+ * and each resolution spawns git (anchor + mount root) — cache so repeat
+ * calls within a run cost nothing. Keyed on the raw env string because tests
+ * (and in principle callers) change it between calls; the not-found error
+ * path is deliberately uncached so a fixed build is picked up on retry.
+ */
+type LocalMcpServer = { entry: string; baseDir: string; mountRoot: string };
+let localMcpServerCache: { key: string; value: LocalMcpServer } | null = null;
+
+function resolveLocalMcpServer(): LocalMcpServer | null {
+  const localServerPath = process.env.SUPABASE_MCP_SERVER_PATH;
+  if (!localServerPath) return null;
+  if (localMcpServerCache?.key === localServerPath)
+    return localMcpServerCache.value;
+
+  const anchor =
+    gitToplevel(dirname(fileURLToPath(import.meta.url))) ?? process.cwd();
+  const isEntryFile = /\.[cm]?js$/.test(localServerPath);
+  const base = resolve(anchor, localServerPath);
+  const probe = isEntryFile
+    ? base
+    : join(base, 'dist', 'transports', 'stdio.js');
+  if (!existsSync(probe)) {
+    throw new Error(
+      `SUPABASE_MCP_SERVER_PATH resolved to ${probe}, which does not exist — ` +
+        `build the server first (pnpm install && pnpm build in the mcp checkout); ` +
+        `see README "Running against an exact MCP server revision".`
+    );
+  }
+  // One filesystem view for command AND mount: the sandbox bind-mounts the
+  // realpath (Docker resolves sources against the daemon's view), so the
+  // command must reference the same view — an override under a symlinked dir
+  // (macOS /tmp -> /private/tmp) would otherwise exec a path that does not
+  // exist in-container. Canonicalize the BASE once and derive the entry from
+  // it (never realpath the entry separately: a symlinked dist/ target could
+  // resolve outside the mounted baseDir).
+  const realBase = realpathSync(base);
+  const baseDir = isEntryFile ? dirname(realBase) : realBase;
+  const value: LocalMcpServer = {
+    entry: isEntryFile
+      ? realBase
+      : join(realBase, 'dist', 'transports', 'stdio.js'),
+    baseDir,
+    // The whole git toplevel (not just dist/) because the build is unbundled:
+    // it requires its node_modules at runtime.
+    mountRoot: gitToplevel(baseDir) ?? baseDir,
+  };
+  localMcpServerCache = { key: localServerPath, value };
+  return value;
+}
+
+function gitToplevel(dir: string): string | null {
+  try {
+    return execFileSync('git', ['rev-parse', '--show-toplevel'], {
+      cwd: dir,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Sandbox mounts required to launch the SUPABASE_MCP_SERVER_PATH build inside
+ * a containerized agent sandbox. A CLI agent's MCP command runs INSIDE the
+ * container, where the host build is invisible — so the build's checkout is
+ * bind-mounted read-only at its identical (real) path, letting the same
+ * config work on both sides, with host rebuilds visible immediately (no
+ * re-copy). Empty when unset.
+ */
+export function supabaseMcpServerMounts(): SandboxMount[] {
+  const local = resolveLocalMcpServer();
+  return local ? [{ hostPath: local.mountRoot, readonly: true }] : [];
+}
+
 export function executorMcpServer(): McpServerDefinition {
   return {
     name: 'executor-mcp',

packages/core/src/mcp-server.test.ts

@@ -0,0 +1,158 @@
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  describe,
+  expect,
+  it,
+  vi,
+} from 'vitest';
+import { execFileSync } from 'node:child_process';
+import {
+  mkdirSync,
+  mkdtempSync,
+  realpathSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { join, relative } from 'node:path';
+import { tmpdir } from 'node:os';
+import {
+  MCP_SERVER_VERSION,
+  supabaseMcpServer,
+  supabaseMcpServerMounts,
+} from './index.js';
+
+// Stub (not mutate) env so a pre-existing SUPABASE_MCP_SERVER_PATH is restored
+// per test.
+function clearEnv() {
+  vi.stubEnv('SUPABASE_MCP_SERVER_PATH', undefined);
+}
+
+// A real on-disk build layout: the override path is existence-checked, so the
+// fixtures must actually exist for the happy paths (and not for the error one).
+let fixtureDir: string;
+let fixtureEntry: string;
+beforeAll(() => {
+  // realpath'd: the resolver realpaths the override (command must match the
+  // container mount view), so unresolved tmpdir paths (macOS /var symlink)
+  // would fail every exact-path assertion below.
+  fixtureDir = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-override-')));
+  fixtureEntry = join(fixtureDir, 'dist', 'transports', 'stdio.js');
+  mkdirSync(join(fixtureDir, 'dist', 'transports'), { recursive: true });
+  writeFileSync(fixtureEntry, '');
+});
+afterAll(() => rmSync(fixtureDir, { recursive: true, force: true }));
+
+describe('supabaseMcpServer().createConfig', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('defaults to the published package via npx', async () => {
+    clearEnv();
+    const { config } = await supabaseMcpServer().createConfig({
+      apiUrl: 'http://api.test',
+    });
+    expect(config.command).toBe('npx');
+    expect(config.args[0]).toBe(
+      `@supabase/mcp-server-supabase@${MCP_SERVER_VERSION}`
+    );
+    expect(config.args).toContain('--api-url');
+  });
+
+  it('launches a local build dir with node when SUPABASE_MCP_SERVER_PATH is set', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.command).toBe('node');
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('uses a direct .js override path as-is', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureEntry);
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('preserves --api-url on the local override path', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    const { config } = await supabaseMcpServer().createConfig({
+      apiUrl: 'http://api.test',
+    });
+    const i = config.args.indexOf('--api-url');
+    expect(i).toBeGreaterThan(-1);
+    expect(config.args[i + 1]).toBe('http://api.test');
+  });
+
+  it('fails fast with an actionable error when the override path does not exist', async () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', join(fixtureDir, 'not-built'));
+    await expect(supabaseMcpServer().createConfig({})).rejects.toThrow(
+      /does not exist.*build the server first/s
+    );
+  });
+  it('resolves a relative override path against the evals checkout root', async () => {
+    clearEnv();
+    const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+      cwd: process.cwd(),
+      encoding: 'utf8',
+    }).trim();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', relative(repoRoot, fixtureEntry));
+    const { config } = await supabaseMcpServer().createConfig({});
+    expect(config.args[0]).toBe(fixtureEntry);
+  });
+
+  it('realpaths a symlinked override so the command matches the container mount', async () => {
+    clearEnv();
+    const linkDir = mkdtempSync(join(tmpdir(), 'mcp-link-'));
+    const link = join(linkDir, 'pkg');
+    symlinkSync(fixtureDir, link);
+    try {
+      vi.stubEnv('SUPABASE_MCP_SERVER_PATH', link);
+      const { config } = await supabaseMcpServer().createConfig({});
+      expect(config.args[0]).toBe(fixtureEntry); // the real path, not the symlink
+      expect(supabaseMcpServerMounts()).toEqual([
+        { hostPath: realpathSync(fixtureDir), readonly: true },
+      ]);
+    } finally {
+      rmSync(linkDir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe('supabaseMcpServerMounts', () => {
+  afterEach(() => vi.unstubAllEnvs());
+
+  it('is empty when no override is set', () => {
+    clearEnv();
+    expect(supabaseMcpServerMounts()).toEqual([]);
+  });
+  it("mounts the override checkout root read-only (a CLI agent's MCP command runs in-container)", () => {
+    clearEnv();
+    // A git checkout wrapping the package dir: the mount must cover the whole
+    // checkout (the unbundled build needs its node_modules), not just dist/.
+    const checkout = realpathSync(mkdtempSync(join(tmpdir(), 'mcp-mount-')));
+    try {
+      execFileSync('git', ['init', '-q'], { cwd: checkout });
+      const pkgDir = join(checkout, 'packages', 'server');
+      mkdirSync(join(pkgDir, 'dist', 'transports'), { recursive: true });
+      writeFileSync(join(pkgDir, 'dist', 'transports', 'stdio.js'), '');
+      vi.stubEnv('SUPABASE_MCP_SERVER_PATH', pkgDir);
+      expect(supabaseMcpServerMounts()).toEqual([
+        { hostPath: checkout, readonly: true },
+      ]);
+    } finally {
+      rmSync(checkout, { recursive: true, force: true });
+    }
+  });
+
+  it('falls back to the package dir when the override is not inside a git checkout', () => {
+    clearEnv();
+    vi.stubEnv('SUPABASE_MCP_SERVER_PATH', fixtureDir);
+    expect(supabaseMcpServerMounts()).toEqual([
+      { hostPath: realpathSync(fixtureDir), readonly: true },
+    ]);
+  });
+});

packages/sandbox/src/agent-environment.ts

@@ -13,7 +13,7 @@
  * this builder, so adding/removing an environment component happens in one place.
  */
 
-import type { SkillSource } from '@supabase-evals/core';
+import type { SandboxMount, SkillSource } from '@supabase-evals/core';
 import { DockerSandbox } from './docker-sandbox.js';
 import {
   ensureSupabaseSandboxImage,
@@ -45,6 +45,12 @@ export interface AgentEnvironmentOptions {
    * mode. This is the only difference between the two environments.
    */
   localStack?: LocalStackSetup;
+  /**
+   * Extra host directories bind-mounted into the sandbox (read-only by
+   * default) — e.g. a local MCP server build the in-container agent must be
+   * able to launch.
+   */
+  mounts?: readonly SandboxMount[];
 }
 
 export interface AgentEnvironment {
@@ -71,6 +77,7 @@ export async function createAgentEnvironment(
     // stack and instead reaches host-side platform-lite over the default bridge
     // via host.docker.internal — so bridge there.
     network: options.localStack ? 'host' : undefined,
+    mounts: options.mounts,
   });
   try {
     if (options.localStack) {