feat(framework): add docs:local content API loop

Author: barryroodt

.gitignore

@@ -12,3 +12,4 @@ evals/*/local/supabase/.branches/
 results/*/
 .sync-tmp/
 
+/.local-docs/

apps/framework/harness/mcp-launch.ts

@@ -0,0 +1,80 @@
+import { existsSync, readFileSync, realpathSync } from 'node:fs';
+import { isAbsolute, join, resolve } from 'node:path';
+import { MCP_SERVER_VERSION } from '@supabase-evals/core';
+
+/** First stable MCP release containing supabase/mcp#343. */
+export const CONTENT_API_FLAG_MIN_VERSION = '0.10.0';
+
+const VERSION_RE = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/;
+
+export function supportsContentApiFlag(version: string): boolean {
+  const parse = (value: string, label: string) => {
+    const match = VERSION_RE.exec(value.trim());
+    if (!match) {
+      throw new Error(
+        `${label} is not a MAJOR.MINOR.PATCH version: ${JSON.stringify(value)}`
+      );
+    }
+    return {
+      numbers: [Number(match[1]), Number(match[2]), Number(match[3])],
+      prerelease: match[4] !== undefined,
+    };
+  };
+  const current = parse(version, 'mcp server version');
+  const minimum = parse(
+    CONTENT_API_FLAG_MIN_VERSION,
+    'CONTENT_API_FLAG_MIN_VERSION'
+  );
+  if (current.prerelease) return false;
+  for (let index = 0; index < 3; index++) {
+    if (current.numbers[index] !== minimum.numbers[index]) {
+      return current.numbers[index] > minimum.numbers[index];
+    }
+  }
+  return true;
+}
+
+export function resolveMcpServerPath(raw: string): string {
+  let path = isAbsolute(raw) ? raw : resolve(process.cwd(), raw);
+  if (!existsSync(path)) throw new Error(`--mcp path does not exist: ${path}`);
+  const packageDir = join(path, 'packages', 'mcp-server-supabase');
+  if (existsSync(packageDir)) path = packageDir;
+  if (!existsSync(join(path, 'dist', 'transports', 'stdio.js'))) {
+    throw new Error(
+      `no built server at ${path} (dist/transports/stdio.js missing) — build it first:\n  pnpm install && pnpm build`
+    );
+  }
+  try {
+    const localVersion = JSON.parse(
+      readFileSync(join(path, 'package.json'), 'utf8')
+    ).version;
+    if (localVersion && localVersion !== MCP_SERVER_VERSION) {
+      console.error(
+        `note: local mcp build is v${localVersion}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible`
+      );
+    }
+  } catch {
+    // An unversioned checkout is valid when it has the expected built entry.
+  }
+  return realpathSync(path);
+}
+
+const CONTENT_API_OPTION = /[,{]\s*(["'])(?:--)?content-api-url\1\s*:/;
+
+export function validateContentApi(
+  contentApiUrl: string,
+  mcpServerPath?: string
+) {
+  if (!mcpServerPath) {
+    if (supportsContentApiFlag(MCP_SERVER_VERSION)) return;
+    throw new Error(
+      `--content-api needs --mcp <path>: the harness launches the pinned v${MCP_SERVER_VERSION} package, which has no --content-api-url flag (added in v${CONTENT_API_FLAG_MIN_VERSION} via supabase/mcp#343), so search_docs would query production docs while the receipt claims ${contentApiUrl}`
+    );
+  }
+  const stdio = join(mcpServerPath, 'dist', 'transports', 'stdio.js');
+  if (!CONTENT_API_OPTION.test(readFileSync(stdio, 'utf8'))) {
+    throw new Error(
+      `the mcp build at ${mcpServerPath} has no --content-api-url flag (predates supabase/mcp#343) — search_docs would query production docs, not ${contentApiUrl}`
+    );
+  }
+}

apps/framework/harness/run-eval.ts

@@ -12,7 +12,7 @@ import {
   statSync,
   writeFileSync,
 } from 'node:fs';
-import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { dirname, join, relative } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { jsonSchema, tool, type ToolSet } from 'ai';
 import { parseEvalMarkdown } from '@supabase-evals/core/eval-markdown';
@@ -32,6 +32,7 @@ import {
 } from '../lib/cli-args.js';
 import { bootPlatformBackend } from './platform-backend.js';
 import { viteBuild, vitestRun } from './project-runner.js';
+import { resolveMcpServerPath, validateContentApi } from './mcp-launch.js';
 import {
   buildDocsResult,
   buildSkillResult,
@@ -73,6 +74,7 @@ const SMOKE = args.has('--smoke');
 const DRY = args.has('--dry');
 const STRICT = args.has('--strict');
 const MCP_PATH = readFlag(rawArgs, 'mcp');
+const CONTENT_API_URL = readFlag(rawArgs, 'content-api');
 const EXPERIMENT_FILTERS = readRepeatedFlag(rawArgs, 'experiment').map(
   normalizeExperimentName
 );
@@ -641,6 +643,7 @@ type Provenance = {
   generatedAt: string;
   host: { sha?: string; branch?: string; dirtyFiles: number };
   mcpOverride?: { path: string; sha?: string; dirtyFiles?: number };
+  contentApiUrl?: string;
   platform: string;
 };
 
@@ -654,7 +657,10 @@ function tryGit(args: string[], cwd: string): string | undefined {
   }
 }
 
-function collectProvenance(mcpPath?: string): Provenance {
+function collectProvenance(
+  mcpPath?: string,
+  contentApiUrl?: string
+): Provenance {
   const dirty = (cwd: string) =>
     (tryGit(['status', '--porcelain'], cwd) ?? '').split('\n').filter(Boolean)
       .length;
@@ -675,34 +681,10 @@ function collectProvenance(mcpPath?: string): Provenance {
       dirtyFiles: repository ? dirty(repository) : undefined,
     };
   }
+  if (contentApiUrl) provenance.contentApiUrl = contentApiUrl;
   return provenance;
 }
 
-function resolveMcpServerPath(raw: string): string {
-  let path = isAbsolute(raw) ? raw : resolve(process.cwd(), raw);
-  if (!existsSync(path)) throw new Error(`--mcp path does not exist: ${path}`);
-  const packageDir = join(path, 'packages', 'mcp-server-supabase');
-  if (existsSync(packageDir)) path = packageDir;
-  if (!existsSync(join(path, 'dist', 'transports', 'stdio.js'))) {
-    throw new Error(
-      `no built server at ${path} (dist/transports/stdio.js missing) — build it first:\n  pnpm install && pnpm build`
-    );
-  }
-  try {
-    const localVersion = JSON.parse(
-      readFileSync(join(path, 'package.json'), 'utf8')
-    ).version;
-    if (localVersion && localVersion !== MCP_SERVER_VERSION) {
-      console.error(
-        `note: local mcp build is v${localVersion}; the harness fixture (platform-lite) tracks the v${MCP_SERVER_VERSION} pin — endpoint drift is possible`
-      );
-    }
-  } catch {
-    // An unversioned checkout is valid when it has the expected built entry.
-  }
-  return realpathSync(path);
-}
-
 function validateJudgeKeys(evals: readonly EvalManifest[]) {
   if (process.env.OPENAI_API_KEY || DRY) return;
   const judged = evals
@@ -781,6 +763,10 @@ async function main() {
 
   const mcpPath = MCP_PATH ? resolveMcpServerPath(MCP_PATH) : undefined;
   if (mcpPath) process.env.SUPABASE_MCP_SERVER_PATH = mcpPath;
+  if (CONTENT_API_URL) {
+    validateContentApi(CONTENT_API_URL, mcpPath);
+    process.env.SUPABASE_CONTENT_API_URL = CONTENT_API_URL;
+  }
 
   const allExperiments = await loadExperiments();
   if (EXPERIMENT_FILTERS.length > 0) {
@@ -916,7 +902,7 @@ async function main() {
 
   validateJudgeKeys([...new Set(allWork.map(({ ev }) => ev))]);
   if (!DEBUG) console.error = () => undefined;
-  const provenance = collectProvenance(mcpPath);
+  const provenance = collectProvenance(mcpPath, CONTENT_API_URL);
 
   let localStackTurn = Promise.resolve();
   const errored: Error[] = [];

apps/framework/package.json

@@ -8,6 +8,7 @@
     "eval": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file-if-exists=../../.env --import tsx/esm harness/run-eval.ts --smoke",
+    "docs:local": "node --env-file-if-exists=../../.env --import tsx/esm scripts/local-docs.ts",
     "eval:vercel": "node --env-file=../../.env --import tsx/esm scripts/run-vercel-evals.ts",
     "typecheck": "tsc --noEmit",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",

apps/framework/scripts/docs/content-api-server.ts

@@ -0,0 +1,71 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+/**
+ * Standalone docs content GraphQL API for `search_docs`.
+ *
+ * Serves the docs app's own route handler (apps/docs/app/api/graphql/route.ts
+ * in a supabase/supabase checkout) over plain node:http — no Next server.
+ * Launched by `pnpm docs:local api` with the docs checkout's tsx so the
+ * route's TS + tsconfig conditions resolve; DOCS_ROUTE_PATH points at the
+ * checkout, PORT picks the listen port.
+ */
+import { createServer } from 'node:http';
+import { pathToFileURL } from 'node:url';
+
+const routePath = process.env.DOCS_ROUTE_PATH;
+if (!routePath) {
+  console.error(
+    'DOCS_ROUTE_PATH not set — run this through `pnpm docs:local api`'
+  );
+  process.exit(1);
+}
+// The docs checkout location is user-supplied at runtime; a static import
+// cannot name it.
+const route = await import(pathToFileURL(routePath).href);
+const handlers: Record<string, (req: Request) => Promise<Response>> = {
+  GET: route.GET,
+  OPTIONS: route.OPTIONS,
+  POST: route.POST,
+};
+const port = Number(process.env.PORT ?? 3001);
+
+createServer(async (incoming, outgoing) => {
+  const url = new URL(
+    incoming.url ?? '/',
+    `http://${incoming.headers.host ?? `127.0.0.1:${port}`}`
+  );
+  const handler = handlers[incoming.method ?? ''];
+  if (url.pathname !== '/docs/api/graphql' || !handler) {
+    outgoing.writeHead(404).end();
+    return;
+  }
+
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(incoming.headers)) {
+    if (Array.isArray(value))
+      for (const item of value) headers.append(name, item);
+    else if (value !== undefined) headers.set(name, value);
+  }
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of incoming) chunks.push(Buffer.from(chunk));
+  const body =
+    incoming.method === 'GET' || incoming.method === 'HEAD'
+      ? undefined
+      : Buffer.concat(chunks).toString('utf8');
+  const response = await handler(
+    new Request(url, { method: incoming.method, headers, body })
+  );
+
+  outgoing.writeHead(
+    response.status,
+    Object.fromEntries(response.headers.entries())
+  );
+  outgoing.end(Buffer.from(await response.arrayBuffer()));
+  // Bind every interface, advertise loopback — same rule as platform-lite in
+  // tools mode (see run-eval.ts: sandboxed CLI agents run their MCP servers
+  // INSIDE the container and reach host-side services via
+  // host.docker.internal, which arrives on the host's bridge interface, not
+  // loopback; a 127.0.0.1-only listener refuses those connections).
+}).listen(port, '0.0.0.0', () => {
+  console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`);
+});

apps/framework/scripts/docs/sentry-stub-loader.mjs

@@ -0,0 +1,11 @@
+// fallow-ignore-file unused-file -- registered at runtime by sentry-stub-register.mjs via module.register()
+// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub.
+const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href;
+
+// fallow-ignore-next-line unused-export -- Node loader-hook contract: the module system calls `resolve`
+export async function resolve(specifier, context, next) {
+  if (specifier === '@sentry/nextjs') {
+    return { url: stubUrl, shortCircuit: true };
+  }
+  return next(specifier, context);
+}

apps/framework/scripts/docs/sentry-stub-register.mjs

@@ -0,0 +1,9 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local
+// no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's
+// own hooks (ours only intercepts the one specifier). Uses module.register()
+// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22",
+// which an older 22.x install satisfies.
+import { register } from 'node:module';
+
+register('./sentry-stub-loader.mjs', import.meta.url);

apps/framework/scripts/docs/sentry-stub.mjs

@@ -0,0 +1,9 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+// No-op @sentry/nextjs stand-in for the standalone docs content API.
+// The route handler calls Sentry.captureException/flush; under plain tsx
+// (outside Next's Sentry instrumentation) the real package's ESM build
+// resolves without those functions and every request crashes. A local dev
+// adapter has no business sending telemetry anyway. Wired up by
+// sentry-stub-register.mjs (see local-docs.ts).
+export const captureException = () => '';
+export const flush = async () => true;