diff --git a/evals/build-docs-002-rls-guide/EVAL.ts b/evals/build-docs-002-rls-guide/EVAL.ts index e2893136..42c3f5ec 100644 --- a/evals/build-docs-002-rls-guide/EVAL.ts +++ b/evals/build-docs-002-rls-guide/EVAL.ts @@ -1,7 +1,6 @@ import { - buildDocsResult, + checkDocsGuideRead, type CheckResult, - type LocalStackEvalContext, type LocalStackScorer, } from '@supabase-evals/core'; import { @@ -34,8 +33,6 @@ import { checkTestFilesExist, } from './tests.js'; -const GUIDE_PATH = 'guides/database/postgres/row-level-security'; - const scorer: LocalStackScorer = async (ctx) => { try { // Snapshot the catalog before running the agent's pgTAP suite, so nothing @@ -82,7 +79,10 @@ const scorer: LocalStackScorer = async (ctx) => { testFiles, suite, await checkTestsExerciseAccessControl(ctx), - checkGuideWasRead(ctx), + checkDocsGuideRead(ctx.toolCalls, { + path: 'guides/database/postgres/row-level-security', + label: 'Row Level Security guide', + }), ]; return { passed: checks.every((check) => check.passed), checks }; @@ -102,22 +102,3 @@ const scorer: LocalStackScorer = async (ctx) => { }; export default scorer; - -// A search_docs hit carries the guide's url in its result, not its request, so -// reuse the harness's own resolution rather than scanning the raw tool call. -function checkGuideWasRead(ctx: LocalStackEvalContext): CheckResult { - const calls = buildDocsResult(ctx.toolCalls).calls.filter((call) => - call.pages?.some((page) => page.url.includes(GUIDE_PATH)) - ); - const withContent = calls.filter((call) => call.hasContent); - return { - name: 'the agent read the Row Level Security guide the prompt referenced', - passed: withContent.length > 0, - notes: - withContent.length > 0 - ? `${withContent.map((call) => call.source).join(', ')}` - : calls.length > 0 - ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` - : 'no docs call reached the guide', - }; -} diff --git a/packages/core/src/docs-results.test.ts b/packages/core/src/docs-results.test.ts index 681a8f79..1f13ebb3 100644 --- a/packages/core/src/docs-results.test.ts +++ b/packages/core/src/docs-results.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'; import { parseClaudeCodeToolCall } from './agents/claude-code/parser.js'; import { buildDocsResult, + checkDocsGuideRead, rehydrateTruncatedDocsResults, } from './docs-results.js'; import type { ToolCallRecord } from './index.js'; @@ -1038,3 +1039,83 @@ describe('rehydrateTruncatedDocsResults', () => { expect(readFile).not.toHaveBeenCalled(); }); }); + +describe('checkDocsGuideRead', () => { + const guide = { + path: 'guides/getting-started/api-keys', + label: 'API keys guide', + }; + + it('passes and names the route when a call retrieved the guide', () => { + const check = checkDocsGuideRead( + [ + toolCall( + 'WebFetch', + { + url: 'https://supabase.com/docs/guides/getting-started/api-keys.md', + prompt: 'Where does each key belong', + }, + { + url: 'https://supabase.com/docs/guides/getting-started/api-keys.md', + name: 'web_fetch', + } + ), + ], + guide + ); + + expect(check).toEqual({ + name: 'the agent read the API keys guide the prompt referenced', + passed: true, + notes: 'web_fetch', + }); + }); + + it('fails when a call reached the guide but retrieved no page content', () => { + const check = checkDocsGuideRead( + [ + toolCall( + 'mcp__supabase-mcp__search_docs', + { + graphql_query: + '{ searchDocs(query: "api keys") { nodes { title href } } }', + }, + { + result: { + searchDocs: { + nodes: [ + { + title: 'API keys', + href: 'https://supabase.com/docs/guides/getting-started/api-keys', + }, + ], + }, + }, + } + ), + ], + guide + ); + + expect(check.passed).toBe(false); + expect(check.notes).toBe( + 'reached the guide via search_docs but retrieved no page content' + ); + }); + + it('fails when no docs call reached the guide', () => { + const check = checkDocsGuideRead( + [ + toolCall( + 'WebFetch', + { url: 'https://supabase.com/docs/guides/auth' }, + { url: 'https://supabase.com/docs/guides/auth', name: 'web_fetch' } + ), + ], + guide + ); + + expect(check.passed).toBe(false); + expect(check.notes).toBe('no docs call reached the guide'); + }); +}); diff --git a/packages/core/src/docs-results.ts b/packages/core/src/docs-results.ts index c8e9af39..d6ec1175 100644 --- a/packages/core/src/docs-results.ts +++ b/packages/core/src/docs-results.ts @@ -1,4 +1,9 @@ -import type { DocsCall, DocsCallPage, DocsResult } from './eval-metadata.js'; +import type { + CheckResult, + DocsCall, + DocsCallPage, + DocsResult, +} from './eval-metadata.js'; import type { ToolCallRecord } from './index.js'; import { isRecord } from './json.js'; @@ -359,3 +364,28 @@ export function buildDocsResult(toolCalls: ToolCallRecord[]): DocsResult { return { calls }; } + +/** + * Whether the agent retrieved the guide at `path`, and by what route. A + * search_docs hit carries the url in its result, not its request, so this + * reads the harness's resolution rather than raw tool calls. + */ +export function checkDocsGuideRead( + toolCalls: ToolCallRecord[], + guide: { path: string; label: string } +): CheckResult { + const calls = buildDocsResult(toolCalls).calls.filter((call) => + call.pages?.some((page) => page.url.includes(guide.path)) + ); + const withContent = calls.filter((call) => call.hasContent); + return { + name: `the agent read the ${guide.label} the prompt referenced`, + passed: withContent.length > 0, + notes: + withContent.length > 0 + ? withContent.map((call) => call.source).join(', ') + : calls.length > 0 + ? `reached the guide via ${calls.map((call) => call.source).join(', ')} but retrieved no page content` + : 'no docs call reached the guide', + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ab0929ae..4f6772cb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -103,6 +103,7 @@ export { parseEvalMarkdown } from './eval-markdown.js'; export { buildSkillResult } from './skill-results.js'; export { buildDocsResult, + checkDocsGuideRead, rehydrateTruncatedDocsResults, } from './docs-results.js'; export type { DocsResultSandbox } from './docs-results.js';