refactor(core): share the docs guide-read check across docs evals

evals/build-docs-002-rls-guide/EVAL.ts

@@ -1,7 +1,6 @@
 import {
-  buildDocsResult,
+  checkDocsGuideRead,
   type CheckResult,
-  type LocalStackEvalContext,
   type LocalStackScorer,
 } from '@supabase-evals/core';
 import {
@@ -34,8 +33,6 @@ import {
   checkTestFilesExist,
 } from './tests.js';
 
-const GUIDE_PATH = 'guides/database/postgres/row-level-security';
-
 const scorer: LocalStackScorer = async (ctx) => {
   try {
     // Snapshot the catalog before running the agent's pgTAP suite, so nothing
@@ -82,7 +79,10 @@ const scorer: LocalStackScorer = async (ctx) => {
       testFiles,
       suite,
       await checkTestsExerciseAccessControl(ctx),
-      checkGuideWasRead(ctx),
+      checkDocsGuideRead(ctx.toolCalls, {
+        path: 'guides/database/postgres/row-level-security',
+        label: 'Row Level Security guide',
+      }),
     ];
 
     return { passed: checks.every((check) => check.passed), checks };
@@ -102,22 +102,3 @@ const scorer: LocalStackScorer = async (ctx) => {
 };
 
 export default scorer;
-
-// A search_docs hit carries the guide's url in its result, not its request, so
-// reuse the harness's own resolution rather than scanning the raw tool call.
-function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult {
-  const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) =>
-    call.pages?.some((page) => page.url.includes(GUIDE_PATH))
-  );
-  const withContent = calls.filter((call) => call.hasContent);
-  return {
-    name: 'the agent read the Row Level Security guide the prompt referenced',
-    passed: withContent.length > 0,
-    notes:
-      withContent.length > 0
-        ? `${withContent.map((call) => call.source).join(', ')}`
-        : calls.length > 0
-          ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content`
-          : 'no docs call reached the guide',
-  };
-}

packages/core/src/docs-results.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
 import { parseClaudeCodeToolCall } from './agents/claude-code/parser.js';
 import {
   buildDocsResult,
+  checkDocsGuideRead,
   rehydrateTruncatedDocsResults,
 } from './docs-results.js';
 import type { ToolCallRecord } from './index.js';
@@ -1038,3 +1039,83 @@ describe('rehydrateTruncatedDocsResults', () => {
     expect(readFile).not.toHaveBeenCalled();
   });
 });
+
+describe('checkDocsGuideRead', () => {
+  const guide = {
+    path: 'guides/getting-started/api-keys',
+    label: 'API keys guide',
+  };
+
+  it('passes and names the route when a call retrieved the guide', () => {
+    const check = checkDocsGuideRead(
+      [
+        toolCall(
+          'WebFetch',
+          {
+            url: 'https://supabase.com/docs/guides/getting-started/api-keys.md',
+            prompt: 'Where does each key belong',
+          },
+          {
+            url: 'https://supabase.com/docs/guides/getting-started/api-keys.md',
+            name: 'web_fetch',
+          }
+        ),
+      ],
+      guide
+    );
+
+    expect(check).toEqual({
+      name: 'the agent read the API keys guide the prompt referenced',
+      passed: true,
+      notes: 'web_fetch',
+    });
+  });
+
+  it('fails when a call reached the guide but retrieved no page content', () => {
+    const check = checkDocsGuideRead(
+      [
+        toolCall(
+          'mcp__supabase-mcp__search_docs',
+          {
+            graphql_query:
+              '{ searchDocs(query: "api keys") { nodes { title href } } }',
+          },
+          {
+            result: {
+              searchDocs: {
+                nodes: [
+                  {
+                    title: 'API keys',
+                    href: 'https://supabase.com/docs/guides/getting-started/api-keys',
+                  },
+                ],
+              },
+            },
+          }
+        ),
+      ],
+      guide
+    );
+
+    expect(check.passed).toBe(false);
+    expect(check.notes).toBe(
+      'reached the guide via search_docs but retrieved no page content'
+    );
+  });
+
+  it('fails when no docs call reached the guide', () => {
+    const check = checkDocsGuideRead(
+      [
+        toolCall(
+          'WebFetch',
+          { url: 'https://supabase.com/docs/guides/auth' },
+          { url: 'https://supabase.com/docs/guides/auth', name: 'web_fetch' }
+        ),
+      ],
+      guide
+    );
+
+    expect(check.passed).toBe(false);
+    expect(check.notes).toBe('no docs call reached the guide');
+  });
+});

packages/core/src/docs-results.ts

@@ -1,4 +1,9 @@
-import type { DocsCall, DocsCallPage, DocsResult } from './eval-metadata.js';
+import type {
+  CheckResult,
+  DocsCall,
+  DocsCallPage,
+  DocsResult,
+} from './eval-metadata.js';
 import type { ToolCallRecord } from './index.js';
 import { isRecord } from './json.js';
 
@@ -359,3 +364,28 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult {
 
   return { calls };
 }
+
+/**
+ * Whether the agent retrieved the guide at `path`, and by what route. A
+ * search_docs hit carries the url in its result, not its request, so this
+ * reads the harness's resolution rather than raw tool calls.
+ */
+export function checkDocsGuideRead(
+  toolCalls: ToolCallRecord[],
+  guide: { path: string; label: string }
+): CheckResult {
+  const calls = buildDocsResult(toolCalls).calls.filter((call) =>
+    call.pages?.some((page) => page.url.includes(guide.path))
+  );
+  const withContent = calls.filter((call) => call.hasContent);
+  return {
+    name: `the agent read the ${guide.label} the prompt referenced`,
+    passed: withContent.length > 0,
+    notes:
+      withContent.length > 0
+        ? withContent.map((call) => call.source).join(', ')
+        : calls.length > 0
+          ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content`
+          : 'no docs call reached the guide',
+  };
+}

packages/core/src/index.ts

@@ -103,6 +103,7 @@ export { parseEvalMarkdown } from './eval-markdown.js';
 export { buildSkillResult } from './skill-results.js';
 export {
   buildDocsResult,
+  checkDocsGuideRead,
   rehydrateTruncatedDocsResults,
 } from './docs-results.js';
 export type { DocsResultSandbox } from './docs-results.js';