diff --git a/.github/scripts/verify-packed-retrieval-parity.mjs b/.github/scripts/verify-packed-retrieval-parity.mjs index 89851355..1ebf5c52 100644 --- a/.github/scripts/verify-packed-retrieval-parity.mjs +++ b/.github/scripts/verify-packed-retrieval-parity.mjs @@ -58,16 +58,16 @@ function assertPackageMeasurement(record, tarballPath) { const evaluationTooling = manifest.items?.find((item) => item.id === 'evaluation-tooling') const activePhase = manifest.items?.find((item) => item.id === manifest.current?.active_phase) const budget = activePhase?.npm_package_budget ?? evaluationTooling?.npm_package_budget - const receipt = manifest.current + const receipt = activePhase?.corrective?.package_candidate ?? manifest.current const actual = { npm_files: requiredNumber(record.entryCount, 'npm pack entryCount'), npm_packed_bytes: requiredNumber(record.size, 'npm pack size'), npm_unpacked_bytes: requiredNumber(record.unpackedSize, 'npm pack unpackedSize'), } for (const [field, value] of Object.entries(actual)) { - if (requiredNumber(receipt?.[field], `current release package receipt ${field}`) !== value) { + if (requiredNumber(receipt?.[field], `active package receipt ${field}`) !== value) { throw new Error( - `Current release package receipt is stale: ${field}=${receipt?.[field]}, freshly packed artifact=${value}`, + `Active package receipt is stale: ${field}=${receipt?.[field]}, freshly packed artifact=${value}`, ) } } @@ -76,13 +76,13 @@ function assertPackageMeasurement(record, tarballPath) { || receipt?.npm_integrity !== record.integrity ) { throw new Error( - 'Current release artifact identity is stale for the freshly packed artifact', + 'Active package artifact identity is stale for the freshly packed artifact', ) } const artifactSha256 = createHash('sha256').update(readFileSync(tarballPath)).digest('hex') if (receipt?.npm_artifact_sha256 !== artifactSha256) { throw new Error( - 'Current release SHA-256 is stale for the freshly packed artifact', + 'Active package SHA-256 is stale for the freshly packed artifact', ) } if ( diff --git a/docs/core-reset/removal-manifest.yml b/docs/core-reset/removal-manifest.yml index 23eb615b..d494997a 100644 --- a/docs/core-reset/removal-manifest.yml +++ b/docs/core-reset/removal-manifest.yml @@ -74,8 +74,13 @@ current: parent_authorization_receipt: https://github.com/mohanagy/madar/issues/629#issuecomment-5155128626 target_branch: next npm_dist_tag: next - publication_state: authorized_pending_protected_next_release_merge - qualification_state: pending_manual_test_and_formal_631_gate + publication_state: published_from_protected_next + publication_receipt: https://github.com/mohanagy/madar/issues/631#issuecomment-5155719419 + release_merge: 81045cc08f1df797ecb86748c9bce09db62aeebd + release_tree: 9d3d90fe7feb3e55042c4f38ac8925abca6e83d1 + trusted_publishing_run: https://github.com/mohanagy/madar/actions/runs/30734176943 + published_at: 2026-08-02T05:42:52Z + qualification_state: owner_manual_test_found_corrective_defects_formal_631_gate_pending package_candidate: npm_files: 102 npm_packed_bytes: 155157 @@ -2666,7 +2671,47 @@ items: registry_metadata_publication: forbidden tag: forbidden main_target: forbidden - notes: 'Issues #632 and #630 are complete. The owner separately authorized exactly @lubab/madar@0.40.0-beta.5 under npm next from the forthcoming protected-next release merge for manual testing before formal #631 qualification. #631 will compare that installed package with 0.32.0, 0.40.0-beta.4, one pinned Graphify commit and a no-tool control. No comparative claim or provider campaign exists yet; stable/latest, GitHub Release, Registry publication, a tag and main remain forbidden.' + corrective: + authorization_receipt: https://github.com/mohanagy/madar/issues/631#issuecomment-5157008370 + parent_receipt: https://github.com/mohanagy/madar/issues/629#issuecomment-5157009594 + protected_base: 81045cc08f1df797ecb86748c9bce09db62aeebd + protected_base_tree: 9d3d90fe7feb3e55042c4f38ac8925abca6e83d1 + target_branch: next + modified_sources: + - src/adapters/mcp/protocol.ts + - src/domain/query/plan.ts + evaluation_sources: + - tools/eval/lib/infrastructure/compare.ts + production_loc_budget: + added_max: 8 + removed_min: 8 + net_max: 0 + source_measurement: + production_typescript_files: 44 + production_typescript_loc: 15871 + added: 8 + removed: 8 + net: 0 + diff_sha256: 51a23988d596afd5d060d603ad98f83903ed06c8bf6cf6bcba0bac66a59e12fe + replacement_measurement: + source_loc: 1424 + emitted_bytes: 60983 + package_candidate: + npm_files: 102 + npm_packed_bytes: 155253 + npm_unpacked_bytes: 653988 + npm_shasum: fe8d482e4bbca13df65a4eb4efd45fde1c35f4a4 + npm_integrity: sha512-8vuEuekerfMzLeIky14BCZa0CUBT34vL8eL1D/ees90u04TGnPdHra1dAsV47AnsiEp3UAPiAsnLKUVpEc1FEg== + npm_artifact_sha256: f63056b7bce91beb59dcdbb341092100500234ff859caac6199c1e84706f0b8e + constraints: + new_production_files: forbidden + dependency_or_lock_change: forbidden + graph_or_index_change: forbidden + query_result_schema_change: forbidden + retrieval_budget_change: forbidden + package_ceiling_change: forbidden + publication: forbidden + notes: 'Issues #632 and #630 are complete and beta.5 is immutable published manual-test history. The beta.5 owner test found one generic workflow-bound parser defect and one ready-state client-handoff defect before formal #631 qualification. Corrective receipt 5157008370 authorizes exactly two zero-net production-source modifications plus the zero-fallback evaluator correction from protected next 81045cc08f1df797ecb86748c9bce09db62aeebd. Both exact owner prompts now return ready 7/7 against the same real-GoValidate flow/evidence corridor without changing graph, index, selection, result schema, dependencies, retrieval budgets, or package ceilings. No comparative claim or provider campaign exists yet; publication, stable/latest, GitHub Release, Registry publication, a tag and main remain forbidden.' exit_gate: The installed exact-head package matches or beats the strongest frozen baseline, scores at least 90 mean with no run below 85 or critical error, makes one Madar retrieval with zero repository-tool fallback in natural-client runs, and passes closed-book, parity, budget, CI, independent-review, and zero-thread gates. - id: non-core-graph-products diff --git a/src/adapters/mcp/protocol.ts b/src/adapters/mcp/protocol.ts index a0708d76..01d6017d 100644 --- a/src/adapters/mcp/protocol.ts +++ b/src/adapters/mcp/protocol.ts @@ -52,7 +52,7 @@ export const MCP_TOOLS: readonly McpToolDefinition[] = Object.freeze([ Object.freeze({ name: 'retrieve', description: - 'Return an authenticated answer dossier or exact gaps.', + 'Return an authenticated answer dossier or exact gaps. Call once per repository question. If state is ready, answer only from the dossier; do not use repository tools or read source files. If state is non-ready, report its exact condition and verify only the named gaps.', inputSchema: Object.freeze({ type: 'object', additionalProperties: false, @@ -135,7 +135,7 @@ export async function handleMcpProtocolRequest( version: context.version, }, instructions: - 'Call once. Ready is complete; otherwise report its exact condition.', + 'Call retrieve once per repository question. If state is ready, answer only from the dossier; do not use repository tools or read source files. If state is non-ready, report its exact condition and verify only the named gaps.', }) case 'ping': return success(id, {}) diff --git a/src/domain/query/plan.ts b/src/domain/query/plan.ts index 77df1636..6bb8a434 100644 --- a/src/domain/query/plan.ts +++ b/src/domain/query/plan.ts @@ -119,12 +119,12 @@ function flowSubject(text: string): SubjectMatch { RegExp(`\\bhow (?:does|do|did) (.+?) get (?:${FW})\\b`), ]) if (direct) return [direct, []] - const active = RegExp( - `\\bhow (?:(?:${AUX}) )?(.+?) (${FW}) (.+?)(?= (?:${CLAUSE})\\b| end to end\\b|$)`, + const match = RegExp( + `\\bhow (?:(?:${AUX}) )?(.+?) (${FW}) (.*?)(?=(?:${CLAUSE})\\b|end to end\\b|$)`, ).exec(text) - if (active) { - const object = content(active[3]!, 'workflow').join(' ') - if (object) return [object, content(active[1]!, 'workflow', true)] + if (match) { + const object = content((match[3] || match[1])!, 'workflow').join(' ') + if (object) return [object, content(match[1]!, 'workflow', true)] } const topic = pick(text, 'workflow', [ /^follow (?:\S+ )*?(\S+) from .+ until (?:\S+ )*?(\S+) is/, @@ -160,7 +160,7 @@ function flowBounds(text: string): FlowBounds { ?? read(/\bfrom .+?\bthrough (?!to\b)(.+?)(?= to\b| how\b|$)/u) ?? (walked ? undefined : read(/\bthrough (?!to\b)(.+?)(?= to\b| how\b|$)/u)), - terminal: read(/\b(?:through to|to) (.+?)(?= how\b|$)/u), + terminal: read(/(? { ) }) + it('allows neutral tool discovery before the single retrieve', async () => { + const { graphPath } = fixture() + const result = await executeNativeAgentCompare( + { + graphPath, + question: 'Trace login', + outputDir: 'out/compare', + execTemplate: 'runner {prompt_file}', + }, + { runner: runnerFor({ madarTools: ['ToolSearch', 'mcp__madar__retrieve'] }) }, + ) + + expect(result.report.attribution_status).toBe('verified') + }) + it('fails attribution when repository exploration precedes retrieve', async () => { const { graphPath } = fixture() const result = await executeNativeAgentCompare( @@ -175,6 +190,28 @@ describe('compare command contract', () => { }) }) + it.each([ + 'Read', + 'read', + 'search', + 'list', + 'shell-read-only', + 'mcp__filesystem__read_file', + ])('fails attribution when %s follows retrieve', async (tool) => { + const { graphPath } = fixture() + const result = await executeNativeAgentCompare( + { + graphPath, + question: 'Trace login', + outputDir: 'out/compare', + execTemplate: 'runner {prompt_file}', + }, + { runner: runnerFor({ madarTools: ['mcp__madar__retrieve', tool] }) }, + ) + + expect(result.report.attribution_status).toBe('violated') + }) + it.each([ { label: 'baseline invokes Madar', diff --git a/tests/unit/core-reset-governance.test.ts b/tests/unit/core-reset-governance.test.ts index 8354f146..2815ceaf 100644 --- a/tests/unit/core-reset-governance.test.ts +++ b/tests/unit/core-reset-governance.test.ts @@ -507,6 +507,29 @@ const BETA_5_AUTHORIZATION = 'https://github.com/mohanagy/madar/issues/631#issuecomment-5155128419' const BETA_5_PARENT_AUTHORIZATION = 'https://github.com/mohanagy/madar/issues/629#issuecomment-5155128626' +const BETA_5_RELEASE_MERGE = '81045cc08f1df797ecb86748c9bce09db62aeebd' +const BETA_5_RELEASE_TREE = '9d3d90fe7feb3e55042c4f38ac8925abca6e83d1' +const BETA_5_PUBLICATION = + 'https://github.com/mohanagy/madar/issues/631#issuecomment-5155719419' +const NO_FALLBACK_CORRECTIVE_AUTHORIZATION = + 'https://github.com/mohanagy/madar/issues/631#issuecomment-5157008370' +const NO_FALLBACK_CORRECTIVE_PARENT = + 'https://github.com/mohanagy/madar/issues/629#issuecomment-5157009594' +const NO_FALLBACK_CORRECTIVE_SOURCES = [ + 'src/adapters/mcp/protocol.ts', + 'src/domain/query/plan.ts', +] as const +const NO_FALLBACK_CORRECTIVE_DIFF_SHA256 = + '51a23988d596afd5d060d603ad98f83903ed06c8bf6cf6bcba0bac66a59e12fe' +const NO_FALLBACK_CORRECTIVE_PACKAGE = { + npm_files: 102, + npm_packed_bytes: 155_253, + npm_unpacked_bytes: 653_988, + npm_shasum: 'fe8d482e4bbca13df65a4eb4efd45fde1c35f4a4', + npm_integrity: + 'sha512-8vuEuekerfMzLeIky14BCZa0CUBT34vL8eL1D/ees90u04TGnPdHra1dAsV47AnsiEp3UAPiAsnLKUVpEc1FEg==', + npm_artifact_sha256: 'f63056b7bce91beb59dcdbb341092100500234ff859caac6199c1e84706f0b8e', +} as const const CAPABILITY_VALIDATION_V2_PROPOSAL_SHA256 = '4906405cbb806c850c0612305ef460e023e2060b5338734ae0af12303901cbd0' const CAPABILITY_VALIDATION_V2_ISSUE = 'https://github.com/mohanagy/madar/issues/612' @@ -1286,6 +1309,11 @@ describe('core reset governance', () => { target_branch: string npm_dist_tag: string publication_state: string + publication_receipt: string + release_merge: string + release_tree: string + trusted_publishing_run: string + published_at: string qualification_state: string package_candidate: { npm_files: number @@ -1480,8 +1508,13 @@ describe('core reset governance', () => { parent_authorization_receipt: BETA_5_PARENT_AUTHORIZATION, target_branch: 'next', npm_dist_tag: 'next', - publication_state: 'authorized_pending_protected_next_release_merge', - qualification_state: 'pending_manual_test_and_formal_631_gate', + publication_state: 'published_from_protected_next', + publication_receipt: BETA_5_PUBLICATION, + release_merge: BETA_5_RELEASE_MERGE, + release_tree: BETA_5_RELEASE_TREE, + trusted_publishing_run: 'https://github.com/mohanagy/madar/actions/runs/30734176943', + published_at: '2026-08-02T05:42:52Z', + qualification_state: 'owner_manual_test_found_corrective_defects_formal_631_gate_pending', package_candidate: BETA_5_PACKAGE, stable_or_latest: 'forbidden', github_release: 'forbidden', @@ -2394,12 +2427,48 @@ describe('core reset governance', () => { tag: 'forbidden', main_target: 'forbidden', }, + corrective: { + authorization_receipt: NO_FALLBACK_CORRECTIVE_AUTHORIZATION, + parent_receipt: NO_FALLBACK_CORRECTIVE_PARENT, + protected_base: BETA_5_RELEASE_MERGE, + protected_base_tree: BETA_5_RELEASE_TREE, + target_branch: 'next', + modified_sources: [...NO_FALLBACK_CORRECTIVE_SOURCES], + evaluation_sources: ['tools/eval/lib/infrastructure/compare.ts'], + production_loc_budget: { added_max: 8, removed_min: 8, net_max: 0 }, + source_measurement: { + production_typescript_files: 44, + production_typescript_loc: 15_871, + added: 8, + removed: 8, + net: 0, + diff_sha256: NO_FALLBACK_CORRECTIVE_DIFF_SHA256, + }, + replacement_measurement: { source_loc: 1_424, emitted_bytes: 60_983 }, + package_candidate: NO_FALLBACK_CORRECTIVE_PACKAGE, + }, }) - expect(execFileSync( + const correctiveSources = execFileSync( + git, + ['diff', '--name-only', BETA_5_RELEASE_MERGE, '--', 'src'], + { encoding: 'utf8' }, + ).trim().split('\n').filter(Boolean).sort() + expect(correctiveSources).toEqual([...NO_FALLBACK_CORRECTIVE_SOURCES].sort()) + const correctiveEvaluatorSources = execFileSync( git, - ['diff', '--name-only', OBLIGATION_RETRIEVAL_MERGE, '--', 'src'], + ['diff', '--name-only', BETA_5_RELEASE_MERGE, '--', 'tools/eval/lib'], { encoding: 'utf8' }, - ).trim()).toBe('') + ).trim().split('\n').filter(Boolean).sort() + expect(correctiveEvaluatorSources).toEqual(['tools/eval/lib/infrastructure/compare.ts']) + expect(() => execFileSync( + git, + ['diff', '--exit-code', BETA_5_RELEASE_MERGE, '--', 'package.json', 'package-lock.json'], + )).not.toThrow() + expect(createHash('sha256').update(execFileSync( + git, + ['diff', '--binary', '--full-index', BETA_5_RELEASE_MERGE, '--', + ...NO_FALLBACK_CORRECTIVE_SOURCES], + )).digest('hex')).toBe(NO_FALLBACK_CORRECTIVE_DIFF_SHA256) const changedEvidenceSkeletonProduction = execFileSync( git, [ @@ -6241,6 +6310,19 @@ describe('core reset governance', () => { diff_sha256: string } } + corrective?: { + protected_base: string + modified_sources: string[] + production_loc_budget: { added_max: number; removed_min: number; net_max: number } + source_measurement: { + production_typescript_files: number + production_typescript_loc: number + added: number + removed: number + net: number + diff_sha256: string + } + } candidate_source?: { production_typescript_files: number production_typescript_loc: number @@ -6253,6 +6335,39 @@ describe('core reset governance', () => { const { current } = manifest const activePhase = manifest.items.find((item) => item.id === current.active_phase) const completedPhase = manifest.items.find((item) => item.id === current.completed_phase) + const corrective = activePhase?.corrective + if (corrective) { + const baseline = corrective.protected_base + expect(execFileSync(git, ['cat-file', '-t', `${baseline}^{commit}`], { + encoding: 'utf8', + }).trim()).toBe('commit') + expect(() => execFileSync( + git, ['merge-base', '--is-ancestor', baseline, 'HEAD'], + )).not.toThrow() + const inventory = sourceInventory() + const delta = productionSourceDelta(baseline) + expect(inventory.filesystemViolations).toEqual([]) + expect(delta.added).toBeLessThanOrEqual(corrective.production_loc_budget.added_max) + expect(delta.removed).toBeGreaterThanOrEqual(corrective.production_loc_budget.removed_min) + expect(delta.net).toBeLessThanOrEqual(corrective.production_loc_budget.net_max) + const { diff_sha256: expectedDiffSha256, ...sourceMeasurement } = + corrective.source_measurement + expect({ + production_typescript_files: inventory.files, + production_typescript_loc: inventory.loc, + ...delta, + }).toEqual(sourceMeasurement) + const changedProduction = execFileSync( + git, ['diff', '--name-only', baseline, '--', 'src'], { encoding: 'utf8' }, + ).trim().split('\n').filter(Boolean).sort() + expect(changedProduction).toEqual([...corrective.modified_sources].sort()) + const diffSha256 = createHash('sha256').update(execFileSync( + git, + ['diff', '--binary', '--full-index', baseline, '--', ...corrective.modified_sources], + )).digest('hex') + expect(diffSha256).toBe(expectedDiffSha256) + return + } const candidatePhase = activePhase?.candidate?.source_measurement ? activePhase : completedPhase diff --git a/tests/unit/mcp-schema-budget.test.ts b/tests/unit/mcp-schema-budget.test.ts index 9f642044..855f6199 100644 --- a/tests/unit/mcp-schema-budget.test.ts +++ b/tests/unit/mcp-schema-budget.test.ts @@ -40,6 +40,15 @@ describe('MCP schema budget', () => { .toEqual(['question', 'budget']) }) + it('makes ready terminal and scopes non-ready verification to named gaps', () => { + const description = MCP_TOOLS[0]?.description ?? '' + + expect(description).toContain('state is ready') + expect(description).toContain('answer only from the dossier') + expect(description).toContain('do not use repository tools') + expect(description).toContain('named gaps') + }) + it('seals the public schema against runtime mutation', () => { expect(Object.isFrozen(MCP_TOOLS)).toBe(true) expect(Object.isFrozen(MCP_TOOLS[0])).toBe(true) diff --git a/tests/unit/query-plan.test.ts b/tests/unit/query-plan.test.ts index 43df3d61..e5c03a6f 100644 --- a/tests/unit/query-plan.test.ts +++ b/tests/unit/query-plan.test.ts @@ -344,6 +344,46 @@ describe('planQuestion', () => { expect(suffix).toEqual(prefix) }) + it('keeps end-to-end wording separate from explicit workflow bounds', () => { + const result = plan( + 'How does generating the idea report work end to end — from the generate request through the pipeline to the saved report?', + ) + + expect(result.subject).toBe('idea report') + expect(result.obligations.find(({ kind }) => kind === 'entry')?.target) + .toBe('generate request') + expect(result.obligations.find(({ kind }) => kind === 'stage')?.target) + .toBe('pipeline') + expect(result.obligations.find(({ kind }) => kind === 'terminal')?.target) + .toBe('report') + }) + + it.each([ + 'a saved report', + 'its saved report', + 'the final saved report', + ])('normalizes a saved-report terminal after determiners: %s', (terminal) => { + const result = plan( + `How does generating the idea report work end to end from the generate request through the pipeline to ${terminal}`, + ) + + expect(result.obligations.find(({ kind }) => kind === 'terminal')?.target) + .toBe('report') + }) + + it.each([ + ['stored procedure', 'store procedure'], + ['store credit', 'store credit'], + ['write concern', 'write concern'], + ])('preserves a compound workflow terminal: %s', (terminal, expected) => { + const result = plan( + `Trace the transaction workflow from request through database to ${terminal}`, + ) + + expect(result.obligations.find(({ kind }) => kind === 'terminal')?.target) + .toBe(expected) + }) + it('extracts the workflow object from event phrasing', () => { const result = plan('What happens when a user requests an idea report?') diff --git a/tests/unit/retrieve-context.test.ts b/tests/unit/retrieve-context.test.ts index ca76e256..cf11c8fa 100644 --- a/tests/unit/retrieve-context.test.ts +++ b/tests/unit/retrieve-context.test.ts @@ -168,21 +168,37 @@ describe('retrieveContext v2', () => { question: 'How does GoValidate generate an idea report end to end?', budget: 4_000, } + const manualIncident = { + question: 'How does generating the idea report work end to end — from the generate request through the pipeline to the saved report?', + budget: 4_000, + } + const manualControl = { + question: 'Explain the report generation pipeline stages and how the idea report is produced', + budget: 4_000, + } const first = retrieveContext(index, active) const second = retrieveContext(index, passive) const third = retrieveContext(index, named) + const fourth = retrieveContext(index, manualIncident) + const fifth = retrieveContext(index, manualControl) expect(first.state).toBe('ready') expect(second.state).toBe('ready') expect(third.state).toBe('ready') + expect(fourth.state).toBe('ready') + expect(fifth.state).toBe('ready') if (first.state !== 'ready' || second.state !== 'ready' - || third.state !== 'ready') return + || third.state !== 'ready' || fourth.state !== 'ready' + || fifth.state !== 'ready') return expect(second.dossier.query.subject).toBe(first.dossier.query.subject) expect(second.dossier.flow).toEqual(first.dossier.flow) expect(second.dossier.evidence).toEqual(first.dossier.evidence) expect(third.dossier.query.subject).toBe(first.dossier.query.subject) expect(third.dossier.flow).toEqual(first.dossier.flow) expect(third.dossier.evidence).toEqual(first.dossier.evidence) + expect(fourth.dossier.query.subject).toBe(first.dossier.query.subject) + expect(fourth.dossier.flow).toEqual(fifth.dossier.flow) + expect(fourth.dossier.evidence).toEqual(fifth.dossier.evidence) const proofRows = new Map(first.dossier.evidence.proofs.map((proof) => [proof.id, proof])) const excerptRows = new Map(first.dossier.evidence.excerpts.map((excerpt) => [ diff --git a/tests/unit/stdio-server.test.ts b/tests/unit/stdio-server.test.ts index c6f08e26..0cac3183 100644 --- a/tests/unit/stdio-server.test.ts +++ b/tests/unit/stdio-server.test.ts @@ -176,6 +176,10 @@ describe('MCP tools-only protocol', () => { expect(Object.keys( result(initialized!).capabilities as Record, )).toEqual(['tools']) + expect(result(initialized!).instructions).toContain('state is ready') + expect(result(initialized!).instructions).toContain('answer only from the dossier') + expect(result(initialized!).instructions).toContain('do not use repository tools') + expect(result(initialized!).instructions).toContain('named gaps') expect(result(listed!).tools?.map((tool) => tool.name)).toEqual(['retrieve']) expect(result(listed!).tools?.[0]).toMatchObject({ description: expect.stringContaining('authenticated answer dossier'), diff --git a/tools/eval/core-reset/verify-isolation.mjs b/tools/eval/core-reset/verify-isolation.mjs index 3a15272a..de3563ff 100644 --- a/tools/eval/core-reset/verify-isolation.mjs +++ b/tools/eval/core-reset/verify-isolation.mjs @@ -71,6 +71,8 @@ const obligationRetrieval = manifest.items?.find( (item) => item.id === "obligation-driven-retrieval-630", ) const packageBudget = activePhase?.npm_package_budget ?? evaluationPackageBudget +const replacementReceipt = activePhase?.corrective?.replacement_measurement + ?? obligationRetrieval?.candidate?.replacement_measurement function assert(condition, message) { if (!condition) throw new Error(message) @@ -299,10 +301,10 @@ assert( assert( replacementSources.length === 3 && replacementMeasurement.source_loc - === obligationRetrieval?.candidate?.replacement_measurement?.source_loc + === replacementReceipt?.source_loc && replacementMeasurement.emitted_bytes - === obligationRetrieval?.candidate?.replacement_measurement?.emitted_bytes, - `#630 replacement receipt drifted: ${replacementMeasurement.source_loc} source LOC / ${replacementMeasurement.emitted_bytes} emitted bytes`, + === replacementReceipt?.emitted_bytes, + `replacement receipt drifted: ${replacementMeasurement.source_loc} source LOC / ${replacementMeasurement.emitted_bytes} emitted bytes`, ) assert( replacementMeasurement.source_loc diff --git a/tools/eval/lib/infrastructure/compare.ts b/tools/eval/lib/infrastructure/compare.ts index a4cfb08d..150b5169 100644 --- a/tools/eval/lib/infrastructure/compare.ts +++ b/tools/eval/lib/infrastructure/compare.ts @@ -195,13 +195,19 @@ function isMadarTool(name: string): boolean { } function isRepositoryTool(name: string): boolean { - return /^(?:Read|Glob|Grep|Bash|Agent|Search|WebSearch)$/i.test(name) + return /^(?:Read|Glob|Grep|Bash|Agent|Search|WebSearch|list|shell-read-only)$/i.test(name) +} + +function isNeutralDiscoveryTool(name: string): boolean { + return /^ToolSearch$/i.test(name) } function extractAttribution(stdout: string): { madarCallCount: number firstMadarToolName: string | null + repositoryCallCount: number repositoryCallsBeforeMadar: number + unclassifiedCallCount: number counts: NativeAgentToolCallCountsEntry } | undefined { const records = parseJsonRecords(stdout) @@ -219,7 +225,10 @@ function extractAttribution(stdout: string): { return { madarCallCount: madarUses.length, firstMadarToolName: firstMadar ? canonicalToolName(firstMadar) : null, + repositoryCallCount: uses.filter(isRepositoryTool).length, repositoryCallsBeforeMadar, + unclassifiedCallCount: uses.filter((name) => + !isMadarTool(name) && !isRepositoryTool(name) && !isNeutralDiscoveryTool(name)).length, counts: { total: uses.length, read: uses.filter((name) => name === 'Read').length, @@ -401,7 +410,9 @@ export async function executeNativeAgentCompare( : baselineTrace.madarCallCount === 0 && madarTrace.madarCallCount === 1 && madarTrace.firstMadarToolName === 'retrieve' && - madarTrace.repositoryCallsBeforeMadar === 0 + madarTrace.repositoryCallsBeforeMadar === 0 && + madarTrace.repositoryCallCount === 0 && + madarTrace.unclassifiedCallCount === 0 ? 'verified' : 'violated' const report: NativeAgentCompareReport = {