From b0eca00d5bcccdce132f87eebfd3742bd121b765 Mon Sep 17 00:00:00 2001 From: Barry Roodt Date: Thu, 13 Aug 2026 17:16:13 +0200 Subject: [PATCH] feat(framework): pnpm local compare against the published baseline pnpm local compare [--experiment ] [--runs N] [--mcp ] pnpm local compare --suite [same flags] Runs the treatment arm exactly as `run` does, then diffs it against the newest published result for that eval and experiment on origin/main, and exits 1 when the treatment regressed from PASS to FAIL. It says out loud what it is. A flip is a SCREEN, not causal proof: the published arm ran in the scheduled CI world, with the published mcp package, the production docs index, and whatever model state existed at refresh time. So the output prints that caveat, and the receipt records the published result's commit, its first parent, and its age in days. The confound is explicit in the artifact rather than left for the reader to remember. Refusals stay pre-spend. An eval with no published row for the requested experiment is refused before any model call, listing the experiments that ARE published for it and pointing at `run` when there is no baseline at all. `--suite` expands to every eval the published export carries for the experiment, and is rejected in run mode or alongside explicit ids. `--runs` defaults to the published row's attempt count, so the arms match unless you say otherwise. Verified: smoke 19/19 (8 new), tsc and biome clean. --- apps/framework/scripts/local.ts | 192 +++++++++++++++++++++++--- apps/framework/scripts/smoke-local.ts | 70 +++++++++- 2 files changed, 240 insertions(+), 22 deletions(-) diff --git a/apps/framework/scripts/local.ts b/apps/framework/scripts/local.ts index 472a5a7b..eb46023d 100755 --- a/apps/framework/scripts/local.ts +++ b/apps/framework/scripts/local.ts @@ -1,19 +1,22 @@ #!/usr/bin/env tsx /** * local.ts — local-dev runner. Run evals against YOUR inputs (an edited skills - * tree, a local MCP build) with provenance receipts. + * tree, a local MCP build) with provenance receipts, and optionally compare + * against the latest published results on `origin/main`. * - * pnpm local run [--experiment ] [--runs N] [--mcp ] + * pnpm local run [--experiment ] [--runs N] [--mcp ] + * pnpm local compare [same flags] * pnpm local experiments * * Design notes: * - Treatment-only: nothing here ever mutates a git tree, so concurrent * sessions/worktrees cannot interfere and in-flight work is never at risk. + * - `compare` is a SCREEN, not causal proof: the published arm ran in the + * scheduled CI world (published MCP package, prod docs index, model state + * at refresh time). The receipt records the published result commit, its + * parent, and its age so the gap is explicit. * - Explicit over magic: this does not build your MCP checkout for you; it - * reports what world it measured. Build it with `pnpm build` in your mcp - * checkout and pass `--mcp`. - * - Gates run before any model call, because the harness SKIPs an experiment - * with exit 0 on missing credentials and a wasted agent run costs real money. + * reports what world it measured. */ import { execFileSync, spawnSync } from 'node:child_process'; import { @@ -115,6 +118,14 @@ type PublishedFile = { committedAt: string; }; +type Baseline = { + row: RawEvalResult; + file: string; + commit: string; + parent: string; + committedAt: string; +}; + /** Load one published export file from origin/main with its commit metadata. */ function loadPublishedFile(file: string): PublishedFile | undefined { let rows: RawEvalResult[]; @@ -153,6 +164,46 @@ function loadPublished(): PublishedFile[] { ); } +/** + * Freshest published row per requested eval for the experiment. Refuses + * (pre-spend) when any requested eval has no published row, listing the + * experiments that ARE published for it. + */ +function resolveBaselines( + evalIds: string[], + experiment: string, + files: PublishedFile[] +): Map { + const best = new Map(); + const failures: string[] = []; + for (const id of evalIds) { + const candidates: Baseline[] = files.flatMap( + ({ file, rows, commit, parent, committedAt }) => + rows + .filter((row) => row.eval === id) + .map((row) => ({ row, file, commit, parent, committedAt })) + ); + const match = candidates + .filter((c) => c.row.experiment === experiment) + .sort((a, b) => Date.parse(b.committedAt) - Date.parse(a.committedAt))[0]; + if (match) { + best.set(id, match); + continue; + } + const alts = [...new Set(candidates.map((c) => c.row.experiment))]; + failures.push( + alts.length + ? `no published ${experiment} result for ${id} on origin/main (published experiments: ${alts.join(', ')})` + : `no published result for ${id} on origin/main at all — use \`pnpm local run\` (no baseline needed)` + ); + } + if (failures.length) { + for (const msg of failures) console.error(msg); + process.exit(1); + } + return best; +} + // ---------- eval validation (fail before spending) ---------- function validateEvals(evalIds: string[]) { @@ -336,14 +387,54 @@ function reportRow( return `${label.padEnd(10)} passed=${String(r?.passed).padEnd(5)} checks=${checksSummary.padEnd(6)} docs.calls=${String(docsCalls).padEnd(3)} ${extra}`; } -/** Run one eval in the treatment world, write its receipt, report. */ +/** Print the published-vs-treatment delta; true when treatment regressed. */ +function reportComparison( + id: string, + b: Baseline, + result: RawEvalResult +): boolean { + writeFileSync( + join(OUT_DIR, `${id}.published.json`), + `${JSON.stringify({ ...b.row, publishedProvenance: { file: b.file, commit: b.commit, parent: b.parent, committedAt: b.committedAt } }, null, 1)}\n` + ); + const ageDays = Math.round( + (Date.now() - new Date(b.committedAt).getTime()) / 86_400_000 + ); + console.log( + reportRow( + 'published', + b.row, + `main@${b.commit.slice(0, 7)} ${b.committedAt.slice(0, 10)} (${ageDays}d old, attempts ${b.row.attempts})` + ) + ); + console.log(reportRow('treatment', result, 'your world')); + const d = (result.passed ? 1 : 0) - (b.row.passed ? 1 : 0); + console.log( + d > 0 + ? '-> IMPROVED vs published (FAIL->PASS)' + : d < 0 + ? '-> REGRESSED vs published (PASS->FAIL)' + : '-> no pass/fail change (compare checks / docs.calls)' + ); + console.log( + 'screen only: the published arm ran in the scheduled CI world — a flip is a signal, not causal proof' + ); + console.log(`saved: results-local/${id}.{published,treatment}.json`); + return d < 0; +} + +/** Run one eval in the treatment world, write its receipt, report; true when it regressed vs published. */ function runTreatment( id: string, experiment: string, runs: number, - opts: { env: Record; mcpPath?: string } -): void { - const { env, mcpPath } = opts; + opts: { + env: Record; + mcpPath?: string; + baseline?: Baseline; + } +): boolean { + const { env, mcpPath, baseline } = opts; console.log( `== treatment: ${id} (${experiment}, runs=${runs}${mcpPath ? ', mcp override' : ''}) ==` ); @@ -368,9 +459,13 @@ function runTreatment( `${JSON.stringify(receipt, null, 1)}\n` ); - console.log(`\n=== local run: ${id} (${experiment}) ===`); + console.log( + `\n=== local ${baseline ? 'compare' : 'run'}: ${id} (${experiment}) ===` + ); + if (baseline) return reportComparison(id, baseline, result); console.log(reportRow('treatment', result, 'your world')); console.log(`saved: results-local/${id}.treatment.json`); + return false; } // ---------- subcommands ---------- @@ -392,15 +487,42 @@ async function cmdExperiments() { mod.default as ExperimentConfig ); console.log( - `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes' : '-'}` + `${name.padEnd(36)} ${(display.agent ?? '?').padEnd(12)} ${(display.modelId ?? '?').padEnd(22)} ${(display.reasoningEffort ?? '-').padEnd(8)} ${published.has(name) ? 'yes (compare)' : '-'}` ); } } +/** Expand --suite to every eval the published export carries for the experiment. */ +function expandSuite( + suite: string, + experiment: string, + files: PublishedFile[] +): string[] { + const file = PUBLISHED_EXPORTS[suite]; + if (!file) + fail( + `unknown suite: ${suite} (available: ${Object.keys(PUBLISHED_EXPORTS).join(', ')})` + ); + const rows = files.filter((f) => f.file === file).flatMap((f) => f.rows); + const ids = [ + ...new Set( + rows.filter((r) => r.experiment === experiment).map((r) => r.eval) + ), + ].sort(); + if (!ids.length) + fail( + `no ${suite} rows published for ${experiment} (published experiments: ${[...new Set(rows.map((r) => r.experiment))].join(', ')})` + ); + console.log( + `suite ${suite} for ${experiment}: ${ids.length} evals (one model run each)\n ${ids.join('\n ')}` + ); + return ids; +} + const RUN_USAGE = - 'usage: pnpm local run [--experiment ] [--runs N] [--mcp ]'; + 'usage: pnpm local [--experiment ] [--runs N] [--mcp ]\n pnpm local compare --suite [same flags] # every eval published for the experiment'; -async function cmdRun(argv: string[]) { +async function cmdRunOrCompare(mode: 'run' | 'compare', argv: string[]) { const parsed = (() => { try { return parseArgs({ @@ -408,6 +530,7 @@ async function cmdRun(argv: string[]) { options: { experiment: { type: 'string' }, runs: { type: 'string' }, + suite: { type: 'string' }, mcp: { type: 'string' }, }, allowPositionals: true, @@ -419,9 +542,27 @@ async function cmdRun(argv: string[]) { const { values, positionals } = parsed; const experiment = values.experiment ?? DEFAULT_EXPERIMENT; validateExperiment(experiment); - const evalIds = positionals; + let published: PublishedFile[] = []; + if (mode === 'compare') { + fetchMain(); + published = loadPublished(); + } + let evalIds = positionals; + if (values.suite) { + if (mode !== 'compare') + fail( + '--suite expands from the published exports and only makes sense with compare' + ); + if (evalIds.length) fail('pass either eval ids or --suite, not both'); + evalIds = expandSuite(values.suite, experiment, published); + } if (!evalIds.length) fail(RUN_USAGE); + const baselines = + mode === 'compare' + ? resolveBaselines(evalIds, experiment, published) + : new Map(); + validateEvals(evalIds); // these gates are spend-relevant only for real runs; the test hook fakes them if (!process.env.LOCAL_EVAL_CMD) { @@ -435,10 +576,17 @@ async function cmdRun(argv: string[]) { if (mcpPath) env.SUPABASE_MCP_SERVER_PATH = mcpPath; mkdirSync(OUT_DIR, { recursive: true }); + let exitCode = 0; for (const id of evalIds) { - const runs = Number(values.runs ?? 1); - runTreatment(id, experiment, runs, { env, mcpPath }); + const runs = Number(values.runs ?? baselines.get(id)?.row.attempts ?? 1); + const regressed = runTreatment(id, experiment, runs, { + env, + mcpPath, + baseline: baselines.get(id), + }); + if (regressed) exitCode = 1; } + process.exit(exitCode); } // ---------- entry ---------- @@ -446,13 +594,15 @@ async function cmdRun(argv: string[]) { const [command, ...rest] = process.argv.slice(2); switch (command) { case 'run': - await cmdRun(rest); + case 'compare': + await cmdRunOrCompare(command, rest); break; case 'experiments': await cmdExperiments(); break; default: - fail(`usage: pnpm local ... - run run eval(s) in your world (skills tree as-is; --mcp override) - experiments list experiments (agent, model, effort, published availability)`); + fail(`usage: pnpm local ... + run run eval(s) in your world (skills tree as-is; --mcp override) + compare run + diff against the latest published result on origin/main + experiments list experiments (agent, model, effort, published-baseline availability)`); } diff --git a/apps/framework/scripts/smoke-local.ts b/apps/framework/scripts/smoke-local.ts index 389cb118..c092503e 100644 --- a/apps/framework/scripts/smoke-local.ts +++ b/apps/framework/scripts/smoke-local.ts @@ -189,7 +189,14 @@ function ck(name: string, fn: () => void) { // --- refusals happen pre-spend, with actionable messages --- { - const r = local(['run', EVAL, '--experiment', 'bogus-model']); + const r = local(['compare', 'no-such-eval-xyz']); + ck('unknown eval refused', () => { + assert.equal(r.status, 1); + assert.match(r.out, /no published result for no-such-eval-xyz/); + }); +} +{ + const r = local(['compare', EVAL, '--experiment', 'bogus-model']); ck('unknown experiment refused with the available list', () => { assert.equal(r.status, 1); assert.match(r.out, /unknown experiment: bogus-model/); @@ -204,6 +211,39 @@ function ck(name: string, fn: () => void) { }); } +// --- compare: delta table + receipts with published provenance --- +{ + const r = local(['compare', EVAL]); + ck('compare prints both rows and the screen caveat', () => { + assert.equal(r.status, 0); + assert.match(r.out, new RegExp(`=== local compare: ${EVAL}`)); + assert.match(r.out, /published .*main@[0-9a-f]{7}/); + assert.match(r.out, /treatment .*your world/); + assert.match(r.out, /screen only:/); + }); + ck('published receipt carries commit provenance', () => { + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.published.json`), 'utf8') + ); + assert.match(receipt.publishedProvenance.commit, /^[0-9a-f]{40}$/); + // Exactly ONE sha, and a real date: %P expands to every parent, so a + // space-split of `%H %P %cI` puts a second parent where the timestamp + // belongs on a merge commit (Date.parse -> NaN, "NaNd old" in the report). + assert.match(receipt.publishedProvenance.parent, /^[0-9a-f]{40}$/); + assert.ok( + !Number.isNaN(Date.parse(receipt.publishedProvenance.committedAt)), + `committedAt is not a date: ${receipt.publishedProvenance.committedAt}` + ); + }); + ck('treatment receipt carries host provenance', () => { + const receipt = JSON.parse( + readFileSync(join(OUT, `${EVAL}.treatment.json`), 'utf8') + ); + assert.match(receipt.provenance.host.sha, /^[0-9a-f]{40}$/); + assert.equal(typeof receipt.provenance.host.dirtyFiles, 'number'); + }); +} + // --- run: no baseline required (custom evals), receipt only --- { const r = local(['run', EVAL]); @@ -255,6 +295,34 @@ function ck(name: string, fn: () => void) { rmSync(fake, { recursive: true, force: true }); } +// --- --suite: expands to the published set; guarded against misuse --- +{ + const r = local(['compare', '--suite', 'regression']); + ck('suite expands and runs every published eval', () => { + assert.equal(r.status, 0); + assert.match(r.out, /suite regression for claude-code-sonnet-5: \d+ evals/); + assert.ok( + (r.out.match(/=== local compare: /g) ?? []).length >= 2, + 'expected multiple compare blocks' + ); + }); + const wrongMode = local(['run', '--suite', 'regression']); + ck('suite refused in run mode', () => { + assert.equal(wrongMode.status, 1); + assert.match(wrongMode.out, /only makes sense with compare/); + }); + const both = local(['compare', EVAL, '--suite', 'regression']); + ck('suite plus ids refused', () => { + assert.equal(both.status, 1); + assert.match(both.out, /not both/); + }); + const bogus = local(['compare', '--suite', 'nope']); + ck('unknown suite lists available', () => { + assert.equal(bogus.status, 1); + assert.match(bogus.out, /unknown suite: nope.*regression, benchmark/); + }); +} + // --- judge-key gate: refused pre-spend, before any agent spawn --- { // needs an eval whose scorer really uses the judge; EVAL may not