feat(framework): pnpm local docs, a local docs index for search_docs evals

    pnpm local docs up   --docs <path-to-supabase-monorepo>
    pnpm local docs seed             # full embed, ~$0.12 OpenAI, asks first
    pnpm local docs api  [--port N]  # serve the content API, foreground
    pnpm local docs down
    pnpm local run <eval> --content-api http://127.0.0.1:3001/docs/api/graphql --mcp <checkout>

Lets a docs edit be measured end to end. The docs checkout is YOURS, passed via
--docs, so there is no submodule and no patching: edit a page there, re-seed,
re-run.

The stack runs from a generated .local-docs/ workdir with its own project id and
a 443xx port block. That block is off the evals local-stack range (54321+) and
below the macOS ephemeral range (49152+), where a transient outbound socket
stealing a listen port is a real failure mode we hit. Files are copied rather
than symlinked, so it works on Windows too.

content-api-server.ts serves the docs app's own GraphQL route over plain
node:http, run with the docs checkout's tsx so the route's TS and its
react-server condition resolve. It binds every interface and advertises
loopback, matching platform-lite in tools mode: a sandboxed agent reaches
host-side services through host.docker.internal, which arrives on the bridge
interface and not on loopback. The Sentry stub is required, not cosmetic — the
real package resolves without captureException outside Next's instrumentation
and every request would crash.

--content-api is gated, because ungated it is a trap: the server would query
PRODUCTION docs while the receipt recorded the local URL, so a paid run would
measure the wrong world and report the right one. The gate consults the
MCP_SERVER_VERSION pin, since the --content-api-url flag shipped in v0.10.0
(supabase/mcp#343), and only asks for --mcp when the pin cannot honour it. It
probes the built server for the FLAG rather than the env var, because the flag
is what we pass: createConfig forwards the URL in argv, as a CLI agent spawns
that command inside the container where our environment is not inherited.

Version handling is deliberately strict. Malformed versions throw, since
reading 0.10.foo as 0.10.0 would report "capable" and buy exactly the bad run.
No prerelease counts either: semver defines precedence, not content, so a
0.11.0-beta could come off a branch that forked before the flag landed.

`docs seed` is full-embed only on purpose. The upstream incremental path has
known bugs (guide checksums are never set, so every guide re-embeds; a source
skipped for missing credentials has its still-valid rows purged), so it waits
for those fixes to land in supabase/supabase.

CLI output is buffered and replayed only on failure, because `supabase start`
prints ANON_KEY, SERVICE_ROLE_KEY, SECRET_KEY and JWT_SECRET on every run and
those do not belong on a screen recording. Dropping stdout is a stream
boundary, not a redaction regex, so there is no pattern to keep in step.

Verified: smoke 27/27 (8 new), core 115/115, sandbox 39 passed, tsc and biome
clean.

Author: barryroodt

.gitignore

@@ -12,5 +12,7 @@ evals/*/local/supabase/.branches/
 results/*/
 .sync-tmp/
 
+
 # local-dev runner (apps/framework/scripts/local.ts)
 /results-local/
+/.local-docs/

apps/framework/scripts/docs/content-api-server.ts

@@ -0,0 +1,71 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+/**
+ * Standalone docs content GraphQL API for `search_docs`.
+ *
+ * Serves the docs app's own route handler (apps/docs/app/api/graphql/route.ts
+ * in a supabase/supabase checkout) over plain node:http — no Next server.
+ * Launched by `pnpm local docs api` with the docs checkout's tsx so the
+ * route's TS + tsconfig conditions resolve; DOCS_ROUTE_PATH points at the
+ * checkout, PORT picks the listen port.
+ */
+import { createServer } from 'node:http';
+import { pathToFileURL } from 'node:url';
+
+const routePath = process.env.DOCS_ROUTE_PATH;
+if (!routePath) {
+  console.error(
+    'DOCS_ROUTE_PATH not set — run this through `pnpm local docs api`'
+  );
+  process.exit(1);
+}
+// The docs checkout location is user-supplied at runtime; a static import
+// cannot name it.
+const route = await import(pathToFileURL(routePath).href);
+const handlers: Record<string, (req: Request) => Promise<Response>> = {
+  GET: route.GET,
+  OPTIONS: route.OPTIONS,
+  POST: route.POST,
+};
+const port = Number(process.env.PORT ?? 3001);
+
+createServer(async (incoming, outgoing) => {
+  const url = new URL(
+    incoming.url ?? '/',
+    `http://${incoming.headers.host ?? `127.0.0.1:${port}`}`
+  );
+  const handler = handlers[incoming.method ?? ''];
+  if (url.pathname !== '/docs/api/graphql' || !handler) {
+    outgoing.writeHead(404).end();
+    return;
+  }
+
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(incoming.headers)) {
+    if (Array.isArray(value))
+      for (const item of value) headers.append(name, item);
+    else if (value !== undefined) headers.set(name, value);
+  }
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of incoming) chunks.push(Buffer.from(chunk));
+  const body =
+    incoming.method === 'GET' || incoming.method === 'HEAD'
+      ? undefined
+      : Buffer.concat(chunks).toString('utf8');
+  const response = await handler(
+    new Request(url, { method: incoming.method, headers, body })
+  );
+
+  outgoing.writeHead(
+    response.status,
+    Object.fromEntries(response.headers.entries())
+  );
+  outgoing.end(Buffer.from(await response.arrayBuffer()));
+  // Bind every interface, advertise loopback — same rule as platform-lite in
+  // tools mode (see run-eval.ts: sandboxed CLI agents run their MCP servers
+  // INSIDE the container and reach host-side services via
+  // host.docker.internal, which arrives on the host's bridge interface, not
+  // loopback; a 127.0.0.1-only listener refuses those connections).
+}).listen(port, '0.0.0.0', () => {
+  console.log(`Docs content API: http://127.0.0.1:${port}/docs/api/graphql`);
+});

apps/framework/scripts/docs/sentry-stub-loader.mjs

@@ -0,0 +1,11 @@
+// fallow-ignore-file unused-file -- registered at runtime by sentry-stub-register.mjs via module.register()
+// Loader-thread resolve hook: '@sentry/nextjs' -> the no-op stub.
+const stubUrl = new URL('./sentry-stub.mjs', import.meta.url).href;
+
+// fallow-ignore-next-line unused-export -- Node loader-hook contract: the module system calls `resolve`
+export async function resolve(specifier, context, next) {
+  if (specifier === '@sentry/nextjs') {
+    return { url: stubUrl, shortCircuit: true };
+  }
+  return next(specifier, context);
+}

apps/framework/scripts/docs/sentry-stub-register.mjs

@@ -0,0 +1,9 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+// Registers a resolve hook that short-circuits '@sentry/nextjs' to the local
+// no-op stub. Injected via NODE_OPTIONS from `pnpm local docs api`; chains with tsx's
+// own hooks (ours only intercepts the one specifier). Uses module.register()
+// (Node 20.6+) rather than registerHooks() (22.15+) — mise pins node "22",
+// which an older 22.x install satisfies.
+import { register } from 'node:module';
+
+register('./sentry-stub-loader.mjs', import.meta.url);

apps/framework/scripts/docs/sentry-stub.mjs

@@ -0,0 +1,9 @@
+// fallow-ignore-file unused-file -- loaded at runtime (spawned/injected by local-docs.ts), never statically imported
+// No-op @sentry/nextjs stand-in for the standalone docs content API.
+// The route handler calls Sentry.captureException/flush; under plain tsx
+// (outside Next's Sentry instrumentation) the real package's ESM build
+// resolves without those functions and every request crashes. A local dev
+// adapter has no business sending telemetry anyway. Wired up by
+// sentry-stub-register.mjs (see local-docs.ts).
+export const captureException = () => '';
+export const flush = async () => true;