diff --git a/.gitignore b/.gitignore index 5f47958..a7daa77 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ dist-ssr .env .env.* !.env.example + +# Evaluation run artifacts (Phase 3A). Runs are reproducible from the +# committed benchmark plus a commit hash, and their output is large, machine- +# specific in its timings, and — for live runs — derived from paid API calls. +# Nothing here is ever committed; see docs/evaluation/RUNBOOK.md. +.eval-runs/ diff --git a/README.md b/README.md index 001b6ad..37469a8 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,50 @@ Default local addresses: Never commit `.env` or API keys. +## Evaluating the pipeline + +Phase 3A added a local-first evaluation harness under [`evals/`](./evals/README.md) +and `decision-benchmark-v1`, a versioned benchmark of 16 fully synthetic cases. +It exists so that a future change to a prompt, model, or scoring rule can be +shown to improve or regress something, rather than argued about. + +Validate the benchmark without running anything: + +```bash +npm run eval:validate +``` + +Run the real pipeline against offline fake providers — no network, no API key, +no cost, deterministic decision content, and a nonzero exit on any required +failure: + +```bash +npm run eval:fixtures +``` + +Compare two recorded runs (`improved` / `regressed` / `unchanged` / +`inconclusive`): + +```bash +npm run eval:compare -- --baseline .eval-runs/run-a --candidate .eval-runs/run-b +``` + +`npm run eval:live` runs against the real OpenAI API and is gated hard: it +requires `--live`, an API key, an explicit budget limit, and a deliberate case +selection, and it refuses to run in CI by default. Run artifacts go to +`.eval-runs/`, which is git-ignored. + +**Scope.** `decision-benchmark-v1` is a *development* benchmark. It is not +scientifically validated, not representative of real hiring decisions, not +evidence of fairness or demographic neutrality, not a legal-compliance test, +not a calibrated-confidence benchmark, and not a production service-level +objective. Every candidate and company in it is invented. A passing fixture run +proves the orchestration, deterministic computation, and graders behave as +specified — it says nothing about prompt quality. + +Details: [`docs/evaluation/`](./docs/evaluation/EVALUATION_ARCHITECTURE.md) and +[ADR-0009](./docs/decisions/ADR-0009-local-first-evaluation-harness.md). + ## V2 documentation - [Phase 0 baseline audit](./docs/PHASE_0_BASELINE_AUDIT.md) @@ -168,6 +212,10 @@ Never commit `.env` or API keys. - [Branch strategy](./docs/BRANCH_STRATEGY.md) - [V2 roadmap](./docs/V2_ROADMAP.md) - [Learning checkpoints](./docs/LEARNING_CHECKPOINTS.md) +- [Evaluation architecture](./docs/evaluation/EVALUATION_ARCHITECTURE.md) +- [Benchmark v1](./docs/evaluation/BENCHMARK_V1.md) +- [Human review guide](./docs/evaluation/HUMAN_REVIEW_GUIDE.md) +- [Evaluation runbook](./docs/evaluation/RUNBOOK.md) - [ADR-0001: main is the V2 line](./docs/decisions/ADR-0001-main-is-v2.md) - [ADR-0002: provider abstraction (superseded by ADR-0004)](./docs/decisions/ADR-0002-provider-abstraction.md) - [ADR-0003: runtime provider configuration](./docs/decisions/ADR-0003-runtime-provider-configuration.md) @@ -176,6 +224,7 @@ Never commit `.env` or API keys. - [ADR-0006: retain Node and Express](./docs/decisions/ADR-0006-retain-node-express.md) - [ADR-0007: npm-only lockfile](./docs/decisions/ADR-0007-npm-only-lockfile.md) - [ADR-0008: React Router 7 migration](./docs/decisions/ADR-0008-react-router-7-migration.md) +- [ADR-0009: local-first evaluation harness](./docs/decisions/ADR-0009-local-first-evaluation-harness.md) - [Dependency audit (Phase 2B-2; updated Phase 2C, Phase 2D)](./docs/security/DEPENDENCY_AUDIT.md) - [Accessibility checklist](./docs/testing/ACCESSIBILITY_CHECKLIST.md) diff --git a/docs/LEARNING_CHECKPOINTS.md b/docs/LEARNING_CHECKPOINTS.md index 6434c85..10fb256 100644 --- a/docs/LEARNING_CHECKPOINTS.md +++ b/docs/LEARNING_CHECKPOINTS.md @@ -201,3 +201,90 @@ The maintainer should be able to explain the current system in approximately two composition roots small? 10. Which successful-pairing invariants are public transport guarantees, and which complete-coverage guarantee remains internal to the pipeline? + +## Phase 3A understanding (evaluation harness and synthetic benchmark) + +You understand Phase 3A if you can answer these without reopening the code. + +**Why did evaluation come before prompt optimisation?** Because without a +measurement, "this prompt is better" is an opinion. Optimising first produces +improvements nobody can demonstrate and regressions nobody sees until a user +finds them. It also matters specifically here: ScenarioRank's central claim is +that LLMs interpret evidence while deterministic code computes the ranking — +and that claim is exactly what most of the graders check. Those checks had to +exist before anyone started changing the parts that could quietly break them. + +**Why build the harness locally instead of using the hosted OpenAI Evals API?** +Four reasons, in order of weight: a hosted service sees prompts and completions +but cannot see whether `mapPairResultsByIdentity` rejected a reversed duplicate, +which is most of what is worth checking; it must run offline and free, because +an evaluation that costs money will not be run; the harness is itself code that +can be wrong, so it needs its own tests; and binding evaluation to one vendor +would reintroduce, at the evaluation layer, the coupling the provider contract +removed. See ADR-0009. + +**What is the difference between a deterministic expectation and a rubric +dimension?** A deterministic expectation is objectively true or false about a +response — does every candidate appear exactly once, does the reported winner +hold the highest deterministic score, was every expected pair evaluated. A +rubric dimension is human judgment — is this claim grounded, is the trade-off +real. Phase 3A automates the first and refuses to automate the second, because +an LLM judging an LLM would produce numbers nobody could defend. + +**Why do some cases list several allowed winners, and some none at all?** +Because pretending a single answer is correct would be dishonest for a case +that is genuinely close. `case-002` allows all three candidates; `case-008` and +`case-009` make no winner claim at all, because the evidence is too thin or too +contradictory to justify one — and what is checked instead is that every +candidate gets flagged for human review. + +**What does a passing fixture run actually prove?** That the orchestration runs +end to end, the deterministic scoring and ranking behave as specified, batch +identity validation rejects what it should, stage and attempt accounting are +coherent, and the graders work. It proves **nothing** about prompt quality, +model behaviour, real-world accuracy, or fairness. The fake provider is +scripted. + +**Why does the artifact schema deliberately not enforce the public response +contract?** Because the `contract-validity` grader exists to detect a response +that violates that contract. If the artifact schema enforced it too, the +harness would crash while recording the very defect it exists to find. +Validation happens in exactly one place — the grader — which reports the +violation instead of destroying the evidence. This was learned the hard way: +the first fixture run crashed for precisely this reason. + +**What is `SR-P3A-001` and why was it not fixed?** `computeRiskAdjustedScore` +can return a negative value for a weak candidate, but the public contract +bounds `risk_adjusted_score` to 0-100 and `server/http/routes.js` validates its +own response before sending — so a run with a weak enough candidate returns a +generic 500 *after* the OpenAI calls have been paid for. It was found by the +first fixture run. It was not fixed because Phase 3A was explicitly forbidden +from changing scoring or contracts, and because either candidate fix moves the +baseline the harness was built to measure from. It is tracked as a known defect +that raises a required failure if it ever stops reproducing. + +**Why do known defects not fail the run, and why is that safe?** Because one +real finding leaving the baseline permanently red trains everyone to ignore it. +It is safe only because of the inverse check: if a grader listed as a known +defect stops failing anywhere in its case, a **required** failure fires +demanding the record be removed. A known-defect record cannot outlive the +defect it describes. + +**Why can a comparison never say "improved" because a run got cheaper?** +Because cost is not correctness. Only required-grader invariants move the +verdict; cost, token, and duration deltas are reported raw and marked +`significance: "not_assessed"`, because two runs cannot support a significance +claim. An output change with no invariant change is `inconclusive`, not +`unchanged` — the honest answer is that the benchmark cannot tell you which +output is better. + +**Why does live mode refuse a model with no recorded pricing?** Because a +budget that cannot be computed cannot be enforced, and guessing a price would +defeat the purpose of having a budget at all. + +### Proof exercise + +Run `npm run eval:fixtures -- --case case-015 --profile missing-pair --no-write` +and explain, without looking, why it exits nonzero, which grader fires, and why +the pipeline reported `"unavailable"` rather than a best pair chosen from the +five pairs it did receive. diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index bb468e6..8c816c4 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -146,6 +146,32 @@ real OpenAI call at any point in Phase 2D; Phase 3 remains unstarted. The recommended next action is Phase 3A evaluation infrastructure planning. See "Phase 2D" below for full detail. +**Phase 3A is in draft and not merged.** Branch +`v2/phase-3a-evaluation-harness`, targeting `main` at +`93dfd4f517bf32ea949a5efeda2140478f62c702`. It adds `evals/` — a +local-first, offline-capable evaluation harness — and +`decision-benchmark-v1`, a versioned benchmark of 16 fully synthetic cases +(21 scenario executions) with 11 deterministic graders, an 8-dimension +anchored human-review rubric, seven offline fake-provider profiles, a +gated live runner, and a four-verdict comparison command. **No production +behaviour changed**: no prompt, model, structured-output schema, scoring +formula, ranking rule, pairing behaviour, HTTP contract, or frontend +component was touched, and no coding agent made a real OpenAI call at any +point. Current verification: 653 tests (103 frontend + 224 server + 326 +evaluation). The frontend count remains unchanged from `main`; the server count +includes repository documentation-guard regression tests only. +**The harness found a real, previously-unknown production defect on its +first run** — `SR-P3A-001`, where `computeRiskAdjustedScore` can return a +negative value while the public contract bounds `risk_adjusted_score` to +0-100, so `server/http/routes.js` rejects its own response and returns a +generic 500 *after* the OpenAI calls have been paid for. It was +deliberately **not** fixed, because Phase 3A is scoped to measurement and +forbidden from changing scoring or contracts; it is recorded in +`docs/architecture/KNOWN_LIMITATIONS.md` (P0.7) and tracked by the +benchmark's known-defect mechanism, which raises a required failure if the +defect ever stops reproducing. Phase 3B has not started. See "Phase 3A" +below for full detail. + ## Project objective ScenarioRank AI V2 is a post-award engineering refinement of the BMW @@ -1871,3 +1897,208 @@ independently reviewable follow-up. actual usage and should be revisited only if this app adopts RSC, Framework Mode, or Data Mode, or when a deliberate React Router 8 migration is separately scoped. + +## Phase 3A — evaluation harness and synthetic benchmark (draft PR, not merged) + +**Branch `v2/phase-3a-evaluation-harness`**, targeting `main` at +`93dfd4f517bf32ea949a5efeda2140478f62c702`. Draft pull request, not merged. + +### Objective + +Build the measurement infrastructure needed to evaluate ScenarioRank's AI +pipeline **before** changing prompts, models, structured-output schemas, +deterministic scoring, ranking, or pairing. Explicitly out of scope: prompt +optimisation, model switching, temperature experiments, production score/ +ranking/pairing changes, LLM-as-judge grading, hosted OpenAI Evals integration, +external benchmark publishing, demographic fairness testing, real applicant +data, database persistence, evaluation dashboards, UI redesign, and Phase 3B. + +### Why evaluation precedes prompt optimisation + +Without a measurement, "this prompt is better" is an opinion — improvements +cannot be demonstrated and regressions stay invisible until a user finds them. +There is also a project-specific reason: ScenarioRank's central architectural +claim is that LLMs interpret evidence while deterministic code computes the +ranking, and most of what the new graders check is exactly that claim. Those +checks needed to exist before anyone began adjusting the parts of the system +that could quietly break them. + +### What was added + +`evals/`, a repository-native harness that executes the **real** production +pipeline (`server/pipeline/runPipeline.js`) — real prompts, real schemas, real +deterministic scoring, real batch-identity validation. + +- **`decision-benchmark-v1`** — 16 fully synthetic cases, 21 scenario + executions. Immutable benchmark ID; enforced versioning policy (case IDs never + change or get reused; a meaning change requires a new `benchmark_version`; a + cosmetic change increments `metadata_revision`; runners refuse an unsupported + `schema_version`; reports record benchmark version and git commit). Closed tag + vocabulary. Every case declares `synthetic: true` and + `data_policy: "synthetic-only"` as schema literals. +- **11 deterministic graders** — contract validity, candidate coverage, + scenario coverage, ranking consistency, score integrity, pairing integrity, + pipeline accounting, `not_measured` honesty, winner expectations, unsupported + claims, uncertainty acknowledgement. Score integrity recomputes every + recomputable deterministic value using the production formulas in + `server/domain/scoring.js`. +- **An 8-dimension anchored human-review rubric** (0-4), with + `not_applicable`/`cannot_determine` never coerced into numbers, per-dimension + scores always retained, and the convenience aggregate withheld below five + scored dimensions. No LLM-as-judge grading. +- **Seven offline fake-provider profiles** — three valid (usable by a committed + case) and four deliberately invalid, so the graders are *proven* to catch + real defects rather than only ever observed passing. +- **A gated live runner**, **a four-verdict comparison command**, and + **permutation/stability utilities**. + +### Commands + +`npm run eval:validate`, `npm run eval:fixtures`, `npm run eval:live`, +`npm run eval:compare` — all support `--help`, emit no ANSI escape codes, and +return nonzero on a required failure. + +### Boundaries held + +`evals/` imports production; production imports **nothing** from `evals/`, and +`evals/repositoryProtection.test.js` enforces the direction across `server/`, +`src/`, `shared/`, `scripts/`, and `server.mjs`. No competing schema copies were +created: the decision output continues to validate through the real +`completedPipelineResponseSchema`. + +One deliberate exception, worth recording because it was learned by a crash: +the run-artifact schema stores the pipeline response *without* enforcing the +public contract. The `contract-validity` grader exists to detect a response +that violates that contract; if the artifact schema enforced it too, the +harness would crash while recording the very defect it exists to find. +Validation happens in exactly one place — the grader — which reports the +violation instead of destroying the evidence. + +### Live-mode safeguards (never executed) + +`--live` required; `OPENAI_API_KEY` required; CI refused unless `--allow-ci`; +an explicit positive, finite budget required (`--max-budget-usd` or +`EVAL_MAX_BUDGET_USD`); the plan (model, cases, repetitions, worst-case calls, +worst-case cost, budget) displayed before the first request; the run refused +outright if the worst case exceeds the budget; a model with no recorded pricing +refused, because a budget that cannot be computed cannot be enforced; spend +re-checked between executions, stopping *before* an execution that could +breach the limit; default repetitions 1; default case selection nothing, with +`--all-cases` required to run the whole benchmark. The provider factory is +imported lazily, so a refused invocation never constructs an OpenAI client. + +**No real OpenAI call was made at any point during Phase 3A**, and no automated +test in this repository calls OpenAI. + +### Artifacts + +Written to `.eval-runs//` (git-ignored): `run-manifest.json`, +`case-results.jsonl`, `summary.json`, `summary.md`, `permutations.json`, +`human-review-template.json`. Never recorded: API keys, headers, request or +response bodies, machine-specific absolute paths. Every artifact is +schema-validated and scanned for secret- and absolute-path-shaped strings +**before** it is written. + +### Fixture baseline + +```text +run state: pass_with_known_defects +fixture machinery: passed +16/16 cases completed without unexpected failure +clean cases: 12 known-defect observations: 8 affected executions: 4 +unexpected failures: 0 unexpected defect resolutions: 0 +``` + +Scenario sensitivity is demonstrated, not assumed: `case-004` produces +different winners across its two scenarios, `case-005` produces three different +specialist winners across three scenarios while never ranking the consistently +moderate candidate first, and `case-016` selects a best pair +(`finnegan-adler::hollis-nakamura`) that is *not* the two strongest +individuals. + +### Real defect found — `SR-P3A-001`, deliberately not fixed + +The first fixture run ever executed found a genuine, previously-unrecorded +production defect. `computeRiskAdjustedScore` (`server/domain/scoring.js`) can +legitimately return a negative value for a weak candidate, while +`completedPipelineResponseSchema` bounds `risk_adjusted_score` to `0-100` — and +`server/http/routes.js` calls `completedPipelineResponseSchema.parse(result)` +before responding. A run containing a sufficiently weak candidate therefore +throws inside the route handler and the user receives a generic +`500 Pipeline failed. Please try again.` **after** every OpenAI call for that +run has already been made and paid for. Observed values: `-30`, `-11.94`, +`-0.4`. + +It was **not fixed**: Phase 3A is scoped to measurement and explicitly +forbidden from changing scoring, ranking, or public contracts, and either +candidate fix (clamping the score, or widening the contract) moves the baseline +the harness was built to measure from. It is recorded in +`docs/architecture/KNOWN_LIMITATIONS.md` (P0.7) and carried in the benchmark as +a documented known defect on the four cases that reproduce it. Known-defect +failures do not gate the exit status — a permanently red baseline trains +everyone to ignore it — but a case-level check raises a **required** failure if +the defect ever stops reproducing, so the fix cannot land silently and the +record cannot outlive the defect. + +A second, smaller gap was surfaced while building `case-002`: ScenarioRank has +**no near-tie uncertainty signal** at all. The deterministic +confidence-and-evidence review keys only on reported confidence and evidence +length, so a decision separated by noise is never flagged for human review. +Recorded as P2.5. + +### Verification + +| Check | Result | +|---|---| +| `npm ci` | passes | +| `npm run lint` | 0 problems | +| `npm run lint:server` | 0 problems (now also covers `evals/`) | +| `npm run typecheck` | passes | +| `npm run check:decision-readability` | passes | +| `npm run check:unused-template` | passes | +| `npm run check:toolchain` | passes | +| `npm run eval:validate` | 16 cases valid, 8 rubric dimensions, 11 graders | +| `npm run eval:fixtures` | `pass_with_known_defects`; fixture machinery passed; 12 clean cases, 8 known-defect observations, 4 affected executions | +| `npm test` | **653 tests** (103 frontend + 224 server + 326 evaluation) | +| `npm run build` | passes | +| `node --check server.mjs` | passes | +| `npm audit` | 2 high advisories: `brace-expansion` (`GHSA-rgw5-rvv9-x895`) and React Router RSC mode (`GHSA-qwww-vcr4-c8h2`); neither was introduced by this Phase 3A documentation pass, and no dependencies changed | +| Dependency/lockfile changes | none | +| `git diff --check` | clean | +| Real OpenAI calls | **none** | +| Router / Vite versions | `react-router` 7.18.2, `vite` 6.4.3, `esbuild` 0.25.12 — unchanged | +| Archive branch and tag | `archive/bmw-award-original` and `bmw-award-original` untouched | + +The frontend test count remains **unchanged from `main`** (103). The server +count includes repository documentation-guard tests only; no production +behaviour was altered. + +### What Phase 3A does not claim + +`decision-benchmark-v1` is a development benchmark. It is not scientifically +validated, not representative of real hiring decisions, not evidence of +fairness or demographic neutrality, not a legal-compliance test, not a +calibrated-confidence benchmark, and not a production service-level objective. +A passing fixture run proves the orchestration, deterministic computation, and +graders behave as specified — it says nothing about prompt quality. The +disclaimer is carried inside every run artifact so it cannot be separated from +the numbers. Full limitation list: `docs/evaluation/BENCHMARK_V1.md`. + +**Phase 3B has not started.** The recommended next milestone is to decide and +apply the `SR-P3A-001` fix, re-baseline the benchmark, and only then begin +prompt and model work with before/after comparison. +# Phase 3A current state (2026-08-05) + +Phase 3A is `pass_with_known_defects`. Fixture machinery: passed. 16/16 cases +completed without unexpected failure: 12 clean cases and four cases containing +8 known-defect observations across 4 affected executions. 0 unexpected failures. 0 unexpected defect resolutions. + +Current verification totals are 103 frontend tests, 224 server tests, 326 +evaluation tests, and 653 total tests. The 2026-08-05 `npm audit` verification +reached the endpoint and reported two high advisories: `brace-expansion` +(`GHSA-rgw5-rvv9-x895`) and React Router RSC mode (`GHSA-qwww-vcr4-c8h2`). +Neither was introduced by this Phase 3A documentation pass; no dependency +changed in Phase 3A. + +SR-P3A-001 remains unfixed. No live evaluation or OpenAI request has occurred; +Phase 3B remains unstarted. diff --git a/docs/REPOSITORY_MAP.md b/docs/REPOSITORY_MAP.md index a7e6baf..7be9ab7 100644 --- a/docs/REPOSITORY_MAP.md +++ b/docs/REPOSITORY_MAP.md @@ -117,3 +117,42 @@ The backend boundary above is now concrete frontend boundary equally explicit. Evaluation and results are directories of cohesive components rather than alternate monoliths, and all active decision source is guarded against lines longer than 180 characters. + +## Evaluation harness (Phase 3A) + +`evals/` is the evaluation harness. It is **not** part of the application: no +HTTP route, frontend component, or build step touches it, and it is invoked +only through its four CLI commands. + +| Path | Ownership | +|---|---| +| `evals/README.md` | entry point and rules the harness follows | +| `evals/datasets/loadBenchmark.js` | strict, fail-closed benchmark loading and cross-checks | +| `evals/datasets/decision-benchmark-v1/` | `manifest.json`, `rubric.json`, `cases/case-0NN.json` (16 synthetic cases) | +| `evals/schemas/` | benchmark case, manifest/rubric, run-artifact, and report schemas | +| `evals/fixtures/fakeProviderProfiles.js` | seven offline fake-provider profiles | +| `evals/graders/` | 11 deterministic graders, human-review template, review aggregation | +| `evals/runners/` | case/benchmark execution, comparison, variants, live gating, request observer | +| `evals/reporters/` | JSON run artifacts and markdown summaries | +| `evals/cli/` | `validate.mjs`, `fixtures.mjs`, `live.mjs`, `compare.mjs` | +| `.eval-runs/` | run artifacts — git-ignored, never committed | + +### Dependency direction + +The harness imports production: `server/pipeline/runPipeline.js`, +`server/domain/scoring.js`, `server/ai/schemas/criteriaKeys.js`, +`server/ai/pricing/openaiPricing.js`, `server/ai/providerFactory.js` (lazily, +live mode only), `server/config/env.js`, and `shared/contracts/`. + +Production imports **nothing** from `evals/`. `evals/repositoryProtection.test.js` +enforces the direction across `server/`, `src/`, `shared/`, `scripts/`, and +`server.mjs`. + +### Test configuration + +`vitest.evals.config.ts` runs `evals/**/*.test.js` as a separate project, so +frontend, backend, and evaluation test counts stay independently reportable. +`npm test` runs all three. + +Documentation: `docs/evaluation/` and +`docs/decisions/ADR-0009-local-first-evaluation-harness.md`. diff --git a/docs/V2_ROADMAP.md b/docs/V2_ROADMAP.md index 6019728..5d963d2 100644 --- a/docs/V2_ROADMAP.md +++ b/docs/V2_ROADMAP.md @@ -353,3 +353,75 @@ versions, dev/build/preview verification, and the complete verification record: `docs/PROJECT_STATUS.md` ("Phase 2D") and `docs/security/DEPENDENCY_AUDIT.md` ("Phase 2D update"). Phase 3 was not started; React Router 8 was not introduced; no real OpenAI call was made. + +### Phase 3A — evaluation harness and synthetic benchmark (draft, not merged) + +**Goal:** build the measurement infrastructure needed to evaluate +ScenarioRank's AI pipeline *before* changing prompts, models, +structured-output schemas, deterministic scoring, ranking, or pairing — and +nothing else. No prompt optimisation, no model switching, no temperature +experiments, no LLM-as-judge grading, no hosted-Evals integration, no +dashboards, no persistence, no Phase 3B. + +Added `evals/`: a local-first, offline-capable evaluation harness, and +`decision-benchmark-v1`, a versioned benchmark of **16 fully synthetic cases** +(21 scenario executions) covering basic ranking, multi-scenario behaviour, +evidence quality, input-permutation robustness, and pairing — with an immutable +benchmark ID, an enforced versioning policy, and a closed tag vocabulary. + +- **11 deterministic graders**: contract validity, candidate coverage, scenario + coverage, ranking consistency, score integrity (recomputing every recomputable + deterministic value from `server/domain/scoring.js`), pairing integrity, + pipeline accounting, `not_measured` honesty, winner expectations, unsupported + claims, and uncertainty acknowledgement. +- **An 8-dimension anchored human-review rubric** (0-4, with `not_applicable` + and `cannot_determine` never coerced into numbers). No LLM-as-judge grading. +- **Offline fixture mode** (`npm run eval:fixtures`): runs the real pipeline + with scripted fake providers — no network, no API key, no cost, deterministic + decision content, CI-suitable, nonzero exit on a required failure. +- **Gated live mode** (`npm run eval:live`): requires `--live`, an API key, an + explicit positive budget, and a deliberate case selection; refuses CI by + default; refuses an unpriced model; displays and enforces a worst-case cost + estimate. **Never executed** — no real OpenAI call was made at any point. +- **Comparison** (`npm run eval:compare`): `improved`/`regressed`/`unchanged`/ + `inconclusive`, where only deterministic invariants move the verdict and + numeric deltas are explicitly marked `not_assessed` for significance. +- **Permutation and stability support**, with run-to-run stability reported as + `not assessed` below two repetitions rather than a meaningless 100%. + +Architecture and reasoning: +`docs/decisions/ADR-0009-local-first-evaluation-harness.md` and +`docs/evaluation/` (architecture, benchmark, human-review guide, runbook). + +**No production behaviour changed.** No prompt, model, schema, scoring formula, +ranking rule, pairing behaviour, HTTP contract, or frontend component was +touched. Current verification: 653 tests (103 frontend + 224 server + 326 +evaluation). The frontend count remains unchanged from `main`; the server count +includes repository documentation-guard regression tests only. + +**The harness found a real defect on its first run** and deliberately did not +fix it: `SR-P3A-001` — `computeRiskAdjustedScore` can return a negative value +while the public contract bounds `risk_adjusted_score` to 0-100, so +`server/http/routes.js` rejects its own response and returns a generic 500 +*after* the OpenAI calls have been paid for. Recorded in +`docs/architecture/KNOWN_LIMITATIONS.md` (P0.7) and tracked by the benchmark's +known-defect mechanism, which raises a required failure if it stops +reproducing. Deciding the fix is the first Phase 3B question, because either +candidate fix moves the baseline the harness measures from. + +### Phase 3B — not started + +Deliberately unstarted. The recommended next milestone is to decide and apply +the `SR-P3A-001` fix (clamp the score, or widen the contract), re-baseline the +benchmark, and only then begin prompt and model work with before/after +comparison. Remaining Phase 3 items from the original plan — structured error +classes and cancellation, and CI wiring — also belong here. +# Phase 3A hardening status + +Phase 3A remains measurement-only. The committed fixture baseline is +`pass_with_known_defects`; fixture machinery: passed. 16/16 cases completed +without unexpected failure: 12 clean cases, 8 known-defect observations, and 4 +affected executions. 0 unexpected failures. 0 unexpected defect resolutions. +Current verification totals: 103 frontend tests, 224 server tests, 326 +evaluation tests, and 653 total tests. SR-P3A-001 remains unfixed; no live +evaluation or OpenAI request has occurred; Phase 3B has not started. diff --git a/docs/architecture/CURRENT_ARCHITECTURE.md b/docs/architecture/CURRENT_ARCHITECTURE.md index 4256042..741c75c 100644 --- a/docs/architecture/CURRENT_ARCHITECTURE.md +++ b/docs/architecture/CURRENT_ARCHITECTURE.md @@ -324,3 +324,56 @@ different candidates may share a name. Each completed pair reference is checked against `candidate_evaluations` for both ID existence and ordered name/ID agreement. Pairing-enabled SSE and JSON route integrations exercise this final transport check. + +## Evaluation harness (Phase 3A) + +`evals/` measures the pipeline described above. It is a sibling of the +application, not a part of it: no HTTP route, frontend component, or build step +references it, and it runs only through its four CLI commands +(`eval:validate`, `eval:fixtures`, `eval:live`, `eval:compare`). + +```text +evals/cli ──> evals/runners ──> server/pipeline/runPipeline.js (real, unchanged) + │ │ + │ └─> server/domain/scoring.js + │ server/ai/schemas/ + ├──> evals/graders ──────> server/domain/scoring.js (recomputation) + │ shared/contracts/decisionApi.js + └──> evals/reporters ───> .eval-runs// (git-ignored) +``` + +The dependency arrow only ever points that way. Production imports nothing from +`evals/`, and `evals/repositoryProtection.test.js` enforces it. + +### What it adds to the architecture + +- **A versioned benchmark** (`decision-benchmark-v1`): 16 fully synthetic + cases, 21 scenario executions, with an immutable ID and an enforced + versioning policy. +- **11 deterministic graders** checking contract validity, candidate coverage, + scenario coverage, ranking consistency, score integrity, pairing integrity, + pipeline accounting, `not_measured` honesty, winner expectations, unsupported + claims, and uncertainty acknowledgement. +- **An offline execution mode** that runs the real pipeline with scripted fake + providers — no network, no API key, no cost, deterministic decision content. +- **A gated live mode** that has never been executed. +- **A four-verdict comparison** (`improved`/`regressed`/`unchanged`/ + `inconclusive`) in which only deterministic invariants can move the verdict. + +### What it changed in the application + +Nothing. Phase 3A altered no prompt, model, structured-output schema, scoring +formula, ranking rule, pairing behaviour, HTTP contract, or frontend component. +The pipeline stages, run metadata, and communication model documented above are +unchanged. + +The one architecturally interesting consequence is a **defect the harness found +in the existing system**: `computeRiskAdjustedScore` can return a negative +value while `completedPipelineResponseSchema` bounds `risk_adjusted_score` to +0-100, so `server/http/routes.js` rejects its own response for a sufficiently +weak candidate. Recorded as `SR-P3A-001` / +`docs/architecture/KNOWN_LIMITATIONS.md` P0.7, deliberately unfixed in this +phase, and tracked by the benchmark's known-defect mechanism. + +Detail: `docs/evaluation/EVALUATION_ARCHITECTURE.md` and +`docs/decisions/ADR-0009-local-first-evaluation-harness.md`. diff --git a/docs/architecture/DATA_FLOW.md b/docs/architecture/DATA_FLOW.md index 313441b..bb39a90 100644 --- a/docs/architecture/DATA_FLOW.md +++ b/docs/architecture/DATA_FLOW.md @@ -165,3 +165,65 @@ and non-streaming route tests parse the completed payload through this contract. Phase 2A adds a transport-validation checkpoint on both sides of this flow: malformed browser input stops at Express with a safe 400/error event; malformed server data stops before the UI renders it with a safe frontend error. + +## 4. Evaluation run (Phase 3A, offline by default) + +A path that exists only for measurement. It never involves the browser, the +Express app, or any HTTP transport. + +```text +npm run eval:fixtures + -> evals/cli/fixtures.mjs + -> loadBenchmark() validates manifest, rubric, and all 16 cases, + including each case's decision input against the + production evaluationRequestSchema + -> runBenchmark() + for each case, for each repetition, for each scenario: + createEvalFakeProvider({ benchmarkCase, scenarioIndex }) + -> createObservingProvider(...) records requested candidate IDs and + canonical pair keys only — never + prompt text, response bodies, or + headers + -> runPipeline(provider, model, request, onUpdate, { maxCandidates }) + ... the real production pipeline, unchanged ... + -> stage snapshots collected via onUpdate + -> runGraders(EXECUTION_GRADERS, { response, trace, stageSnapshots }) + -> applyKnownDefects(...) documented pre-existing defects + -> runGraders(CASE_GRADERS, ...) scenario coverage + -> checkKnownDefectsStillReproduce() required failure if a defect is gone + -> computeStability() / analysePermutations() + -> writeRunArtifacts() schema-validated and policy-scanned BEFORE write + -> .eval-runs// git-ignored +``` + +A case with N scenarios at R repetitions produces N x R executions. The +production request contract takes exactly one scenario, so the harness executes +one pipeline run per scenario rather than inventing a request shape the server +cannot serve. + +### Trust boundaries specific to evaluation + +**Harness to pipeline.** One-way. The harness imports production; production +imports nothing from `evals/`. Enforced by test. + +**Provider request to trace.** The observing provider records derived +identifiers only. Prompt text, system text, response bodies, headers, and API +keys are never retained, so a trace is safe to write into an artifact. + +**Run result to artifact.** Every artifact is schema-validated and scanned for +secret- and absolute-path-shaped strings before it is written. A violation +throws rather than writing — a leaked value in a run directory is worse than a +failed run. + +One deliberate asymmetry: the artifact schema stores the pipeline response +*without* enforcing `completedPipelineResponseSchema`. The `contract-validity` +grader exists to detect a response that violates that contract; if the artifact +schema also enforced it, the harness would crash while recording the very +defect it exists to find. Contract validation happens in exactly one place — +the grader — which reports the violation instead of destroying the evidence. + +**Live mode to network.** The only path in the harness that reaches the +network, and only after `--live`, an API key, an explicit positive budget, a +deliberate case selection, a non-CI environment (or `--allow-ci`), and a +pre-flight worst-case cost check have all passed. The provider factory is +imported lazily, so a refused invocation never constructs an OpenAI client. diff --git a/docs/architecture/KNOWN_LIMITATIONS.md b/docs/architecture/KNOWN_LIMITATIONS.md index 25c7b26..6d4dd79 100644 --- a/docs/architecture/KNOWN_LIMITATIONS.md +++ b/docs/architecture/KNOWN_LIMITATIONS.md @@ -117,6 +117,42 @@ out of scope for this phase. Full detail: ~~The pairing stage's *outer* fallback — returning a generic default pair when every pair call in a run fails — means a result can look complete when the underlying evaluation didn't succeed for any pair.~~ The pairing stage's `?? default`-style fallbacks for individual metric fields were removed in Phase 1B (the production schema now requires all six pairing metrics, so a response missing one is rejected and retried rather than defaulted). Post-review correction (Phase 1D): the remaining *outer* fallback — a fabricated "Default pair" with invented scores (`pair_score: 7.0`, `scenario_coverage: 0.75`, etc.) — was removed entirely. When every pair evaluation in a run fails, `pairing_result` is now `{ "status": "unavailable", "reason": "All pair evaluations failed.", "best_pair": null, "top_pairs": [] }` instead of an invented pair, and the frontend's pairing tab shows a plain "Pairing Unavailable" message rather than a fake recommendation. Regression tests (`server/pipeline/runPipeline.test.js`, "pairing failure modes never fabricate a pair") cover full success, partial pair failure, and all-pairs-failed, and assert none of the old fabricated values appear anywhere in the response. **Post-review correction (ADR-0004):** pairing was later redesigned from one provider request per pair to a single batch request for every relevant pair; the same honesty guarantee carries over and was later tightened further: a duplicate, unrequested, *or merely-missing* pair in the batch is now rejected (one corrective retry, then the stage fails), because a successful pairing result must cover every expected pair — a subset is never classified as a successful "best pair" analysis. Only a batch that still fails to cover every expected pair after the retry falls back to the honest `{"status":"unavailable","reason":"Complete pair analysis was unavailable.","best_pair":null,"top_pairs":[]}` shape (reason text updated from the earlier "All pair evaluations failed."). +### P0.7 Negative `risk_adjusted_score` violates the public response contract — OPEN (found by the Phase 3A evaluation harness) + +**Defect ID `SR-P3A-001`.** Found by the first fixture run ever executed +against `decision-benchmark-v1`. + +`computeRiskAdjustedScore` (`server/domain/scoring.js`) can legitimately return +a negative value for a sufficiently weak candidate — its risk penalties are +subtracted from the weighted fit score with no lower clamp. But +`completedPipelineResponseSchema` (`shared/contracts/decisionApi.js`) bounds +`risk_adjusted_score` to `0-100`, and `server/http/routes.js` (the +`/api/decision` handler) calls `completedPipelineResponseSchema.parse(result)` +before responding. + +The consequence is user-visible and expensive: an evaluation containing a +sufficiently weak candidate throws inside the route handler and the user +receives a generic `500 Pipeline failed. Please try again.` — **after** every +OpenAI call for that run has already been made and paid for. The SSE path +performs the same validation. + +Observed values across the benchmark: `-30` (case-001, case-011), `-11.94` +(case-006, second scenario), `-0.4` (case-008). A candidate scoring 3/10 across +the board is enough to trigger it. + +**Deliberately not fixed in Phase 3A.** That phase was explicitly scoped to +building measurement infrastructure and forbidden from changing scoring, +ranking, or public contracts. Fixing it means choosing between clamping the +score (changes scoring output) and widening the contract (changes the public +API) — and either choice moves the baseline the harness was built to measure +*from*. This is the first thing Phase 3B should decide. + +The benchmark records it as a documented known defect on the four cases that +reproduce it. Known-defect failures do not gate the evaluation exit status, but +a case-level check raises a **required** failure if the defect ever stops +reproducing, so the fix cannot land silently and the record cannot outlive the +defect. See `docs/evaluation/BENCHMARK_V1.md`. + ### P0.6 Candidate scoring depends on very limited evidence Still open — unchanged. Short user-written descriptions are treated as sufficient evidence for detailed leadership judgments. The source, completeness, and reliability of those descriptions are unknown. @@ -163,9 +199,25 @@ HTTP/SSE data. The browser imports those schemas and derives types with ~~Request validation, SSE event ordering, timeouts, and error propagation are untested.~~ `server/http/routes.test.js` (8 tests) exercises the real Express app on an ephemeral port: SSE stage ordering through to `complete`, error events for invalid input and for a failing pipeline stage (asserted to resolve within 5s — no hang), AI-unavailable handling, `/health` secret-safety, and both `/api/decision` success and 503 paths. -### P2.3 No model evaluation dataset +### P2.3 No model evaluation dataset — SUBSTANTIALLY RESOLVED (Phase 3A), with real remaining gaps + +~~There are no golden examples, expected score ranges, consistency checks, prompt regression tests, or human-labeled benchmarks.~~ Phase 3A added `evals/` — a local-first, offline-capable evaluation harness and `decision-benchmark-v1`, a versioned benchmark of **16 fully synthetic cases** (21 scenario executions) covering basic ranking, multi-scenario behaviour, evidence quality, input-permutation robustness, and pairing. 11 deterministic graders check contract validity, candidate coverage, scenario coverage, ranking consistency, score integrity (by recomputing every recomputable deterministic value from `server/domain/scoring.js`), pairing integrity, pipeline accounting, `not_measured` honesty, winner expectations, unsupported claims, and uncertainty acknowledgement. `npm run eval:fixtures` runs the real pipeline offline with fake providers at zero cost. See `docs/evaluation/` and `docs/decisions/ADR-0009-local-first-evaluation-harness.md`. + +Deliberately **not** resolved, and stated plainly rather than implied: + +- **No golden-output snapshots**, by choice — snapshots of model text fail on any wording change, which trains people to re-bless them without reading (ADR-0009, "Alternatives considered"). +- **No human labels yet.** The 8-dimension anchored rubric and the review template exist; no human review has been performed, so every qualitative dimension is currently unscored. +- **A fixture run says nothing about prompt quality.** It exercises orchestration and deterministic computation with a scripted provider. Prompt-regression measurement needs live runs, which Phase 3A gated but never executed. +- **Run-to-run variance is unmeasured.** The baseline uses one repetition, and the harness reports `not assessed` rather than a meaningless 100%. +- **16 synthetic cases written by the system's own author** encode that author's expectations. This is a development benchmark, not evidence of fairness, calibration, or real-world accuracy. -Still open — unchanged. There are no golden examples, expected score ranges, consistency checks, prompt regression tests, or human-labeled benchmarks. Deferred to Phase 3 (`docs/V2_ROADMAP.md`). +Full limitation list: `docs/evaluation/BENCHMARK_V1.md`, "Known limitations of this benchmark". + +### P2.5 No near-tie uncertainty signal — OPEN (surfaced by Phase 3A) + +The deterministic confidence-and-evidence review (`confidenceEvidenceReview` in `server/pipeline/runPipeline.js`) keys only on model-reported confidence and evidence-string length. It has no notion of *ranking margin*, so a decision separated by noise — two candidates within a fraction of a point — is never flagged for human review, while a clear-cut decision with slightly terse evidence is. + +Surfaced while building `case-002` (a deliberate close call between three well-evidenced, confidently-scored candidates): the case cannot honestly carry the `uncertainty` tag, because the pipeline has no mechanism that would fire. A Phase 3B candidate. ### P2.4 No reproducibility controls — RESOLVED (Phase 1B) @@ -232,3 +284,47 @@ maintainability cleanup; Phase 3 evaluation/reliability work has not begun. This cleanup does not add persistence, rate limiting, calibration, evaluation datasets, or production-readiness guarantees. + +## Phase 3A evaluation harness — draft, not merged + +Phase 3A added the measurement infrastructure described in `docs/evaluation/` +and `docs/decisions/ADR-0009-local-first-evaluation-harness.md`. It changed no +prompt, model, structured-output schema, scoring formula, ranking rule, pairing +behaviour, or public contract, and no coding agent made a real OpenAI call at +any point. + +What the harness itself does **not** do, stated so a green run is not misread: + +1. **It does not validate the product.** `decision-benchmark-v1` is a + development benchmark. It is not scientifically validated, not + representative of real hiring decisions, not evidence of fairness or + demographic neutrality, not a legal-compliance test, not a + calibrated-confidence benchmark, and not a production service-level + objective. +2. **A fixture run proves the machinery, not the model.** The fake provider is + scripted. A passing run means orchestration, deterministic computation, and + the graders behave as specified. +3. **Wording and irrelevant-text variants cannot fail offline.** The fixture + scores by candidate ID and never reads description text, so `case-013` and + `case-014` are guaranteed to match their originals. They validate the + linkage and comparison machinery only. +4. **The unsupported-claim checks are conservative phrase matching**, scoped to + model-authored narrative fields. They catch a short list of specific + overclaims; they cannot judge whether an argument is sound. The + narrative-contradiction check is name-based and is deliberately skipped — + with the reason reported — when the winner's display name is shared by + another candidate. +5. **`weighted_fit_score` cannot be recomputed** from the public response, + because normalised criterion weights are not exposed. Everything derived + from it is recomputed. +6. **Exact ties are resolved by submission order.** The production ranking is a + stable sort over the submitted candidate array, so a candidate-order + permutation could legitimately change the winner on an exact tie. The + benchmark avoids exact ties rather than encoding that behaviour as a + guarantee. +7. **No adversarial, prompt-injection, or malformed-input cases**, and no + cross-scenario resilience measurement (the pipeline reports `not_measured` + and the harness checks that it keeps saying so). +8. **Live mode has never been executed.** Its gating, budget arithmetic, and + refusal paths are covered by tests that inject values; no automated test in + this repository calls OpenAI. diff --git a/docs/architecture/TECHNOLOGY_INVENTORY.md b/docs/architecture/TECHNOLOGY_INVENTORY.md index 23f5ce3..1d2d21c 100644 --- a/docs/architecture/TECHNOLOGY_INVENTORY.md +++ b/docs/architecture/TECHNOLOGY_INVENTORY.md @@ -75,3 +75,21 @@ For recruiter-facing documentation, describe technologies according to their act Phase 2A adds frontend API-client, SSE-parser, and workflow-hook tests. No browser E2E or real-provider test was added. + +## Evaluation technologies (Phase 3A) + +The evaluation harness added **no new dependency**. It uses Node built-ins +(`node:fs/promises`, `node:path`, `node:child_process`, `node:url`) plus `zod`, +which the application already depends on, and Vitest, which it already uses. + +| Considered | Decision | +|---|---| +| Hosted OpenAI Evals API | **Not adopted in Phase 3A.** It cannot observe the deterministic layer most of these checks target (batch-identity validation, ranking agreement, pair canonicalisation, stage accounting), requires network access and spend per run, and would couple the benchmark to one vendor. Boundaries were drawn so it can be added later as a provider factory plus a reporter — see ADR-0009. | +| A CLI framework (`commander`, `yargs`, `minimist`) | **Not adopted.** `evals/cli/args.js` is ~60 lines of dependency-free parsing. Adding supply-chain surface for argument splitting was not justified, and a repository-protection test asserts none of these appears in `package.json`. | +| A terminal-colour library (`chalk`) | **Not adopted.** CLI and artifact output is deliberately ANSI-free so it stays greppable and diffable; tests assert no escape codes are emitted. | +| Python + an evaluation framework | **Not adopted**, consistent with ADR-0006. A second language and toolchain for evaluation alone would duplicate working, tested infrastructure for no product requirement. | +| Snapshot/golden-output testing | **Not adopted** as the primary mechanism. Snapshots of model text fail on any wording change, which trains people to re-bless them without reading. | +| LLM-as-judge grading | **Deferred to a later phase.** Layering a second unvalidated model judgment on an unvalidated first one produces numbers nobody could defend. | + +Configuration added: `vitest.evals.config.ts` (separate test project) and four +`eval:*` npm scripts. `.eval-runs/` is git-ignored. diff --git a/docs/decisions/ADR-0009-local-first-evaluation-harness.md b/docs/decisions/ADR-0009-local-first-evaluation-harness.md new file mode 100644 index 0000000..9698ad7 --- /dev/null +++ b/docs/decisions/ADR-0009-local-first-evaluation-harness.md @@ -0,0 +1,181 @@ +# ADR-0009: Local-first evaluation harness and versioned benchmark + +- **Status:** Accepted (Phase 3A) +- **Date:** 2026-08-02 +- **Supersedes:** nothing +- **Related:** [ADR-0004](ADR-0004-single-openai-provider.md) (single OpenAI + provider), [ADR-0005](ADR-0005-shared-http-contracts.md) (shared contracts), + [ADR-0006](ADR-0006-retain-node-express.md) (Node/Express retained), + [ADR-0007](ADR-0007-npm-only-lockfile.md) (npm only) + +## Context + +Phase 3 is "reliability and evaluation." Before ScenarioRank changes a prompt, +a model, a structured-output schema, the deterministic scoring formulas, the +ranking, or the pairing behaviour, it needs a way to tell whether that change +made anything better or worse. Without one, every future change is an opinion. + +Two questions had to be answered before writing any of it. + +**Where should evaluation run?** OpenAI offers a hosted Evals API, and using it +would mean less code to maintain. + +**What is a benchmark result actually worth?** A benchmark that is described +as more than it is becomes a liability: it invites "our system was evaluated +and passed" from a set of sixteen invented cases. + +## Current Phase 3A status (2026-08-05) + +The committed offline baseline is `pass_with_known_defects`; fixture machinery: +passed. 16/16 cases completed without unexpected failure. There are 12 clean +cases, 8 known-defect observations, and 4 affected executions; 0 unexpected +failures and 0 unexpected defect resolutions. Current verification totals +are 103 frontend tests, 224 server tests, 326 evaluation tests, and 653 total +tests. SR-P3A-001 remains unfixed. No live evaluation or OpenAI request has +occurred, and Phase 3B remains unstarted. + +## Decision + +### 1. The harness is repository-native and local-first + +Phase 3A builds the evaluation system inside this repository, under `evals/`, +executing the real production pipeline directly. The hosted OpenAI Evals API +is deliberately **not** integrated in this phase. + +Reasons, in order of weight: + +1. **It must evaluate this application's real orchestration.** Most of what is + worth checking in ScenarioRank is not the model's text — it is the + deterministic layer around it: batch-identity validation, ranking, risk + formulas, pair canonicalisation, logical-stage accounting. A hosted + evaluation service sees prompts and completions. It cannot see whether + `mapPairResultsByIdentity` rejected a reversed duplicate. +2. **It must work offline, with no API key and no cost.** The owner has a + small real budget. An evaluation system that costs money every time it runs + will not be run, and an unused benchmark measures nothing. +3. **It must be testable without spending anything.** The harness is itself + code that can be wrong. It has 326 evaluation tests, none of which calls + OpenAI. +4. **It must remain provider-portable.** ScenarioRank reaches its provider + through a provider-neutral contract (ADR-0002/ADR-0004). Binding evaluation + to one vendor's evaluation product would reintroduce, at the evaluation + layer, exactly the coupling the provider contract removed. +5. **Benchmarks and schemas belong under version control with the code they + describe.** A benchmark that lives in a vendor dashboard cannot be reviewed + in a pull request, cannot be bisected, and cannot be pinned to a commit. + +The boundaries are drawn so a hosted service could be added later without +rewriting the benchmark: the dataset is plain JSON with its own schema, the +runner takes a provider factory rather than constructing a provider, and +grading is separate from execution. Adding a hosted adapter in a later phase +is a new `createProvider` implementation plus a reporter, not a rewrite. + +### 2. The benchmark is versioned, and its identity is immutable + +`decision-benchmark-v1` is fixed. The rules, enforced by schema and test: + +- `benchmark_id` never changes once published. +- A case ID never changes and is never reused. +- Changing what an existing case *means* requires a new `benchmark_version`. +- A change that cannot alter any result increments `metadata_revision` only. +- `schema_version` describes file shape; a runner **refuses** a version it does + not support rather than attempting a best-effort read. +- Every report records the benchmark version and the git commit. +- The comparison command refuses to compare across benchmark versions, so a + benchmark edit can never be mistaken for a pipeline improvement. + +### 3. Deterministic invariants and qualitative judgment are kept apart + +A case carries `deterministic_expectations` (objectively checkable: coverage, +pair completeness, stage accounting, whether the reported winner matches the +deterministic ranking) and `rubric_dimensions` (human judgment: grounding, +trade-off clarity, uncertainty handling). + +No case hardcodes one "correct" natural-language answer. Where more than one +winner is defensible, `allowed_winner_ids` lists all of them; where the +evidence is too thin to justify any winner claim, the case makes none. + +Phase 3A implements **no LLM-as-judge grading**. Qualitative dimensions are +scored only through a structured human-review format, and the report keeps +dimension-level scores even when it computes a convenience aggregate. + +### 4. Production never imports the harness + +`evals/` imports production code — the pipeline, the shared contracts, the +deterministic scoring functions. Production imports nothing from `evals/`, and +a repository-protection test enforces the direction. Evaluation-specific +schemas *wrap* production output with benchmark metadata; the decision output +itself continues to validate through the real public contract. + +### 5. Live mode is gated, budgeted, and off by default + +Live runs require `--live`, an API key, a positive explicit budget, and a +deliberate case selection (nothing runs by default; the whole benchmark needs +its own flag). CI is refused unless overridden. The worst-case cost is computed +and displayed before the first request, refused if it exceeds the budget, and +re-checked between executions. A model with no recorded pricing is refused +outright, because a budget that cannot be computed cannot be enforced. + +No real OpenAI call was made at any point during Phase 3A implementation. + +## Alternatives considered + +**Integrate the hosted OpenAI Evals API now.** Rejected for this phase: it +cannot observe the deterministic layer that most of these checks target, it +requires network access and spend for every run, and it would couple the +benchmark to one vendor. Reasonable to revisit once there is a question that +genuinely needs it (large-scale prompt sweeps, for instance). + +**Golden-output snapshot tests.** Rejected as the primary mechanism. Snapshots +of model text fail on any wording change, which trains people to re-bless them +without reading. Deterministic invariants plus an explicit human rubric say +what actually matters. + +**LLM-as-judge grading.** Deferred. It is a reasonable Phase 3B question, but +adding a second, unvalidated model judgment on top of an unvalidated first one +would produce numbers nobody could defend. The human-review format exists so +that a future judge can be checked against something. + +**A single quality score.** Rejected. Collapsing eight qualitative dimensions +into one number hides which dimension is weak, which is the only actionable +part. + +**Python for the harness.** Rejected, consistent with ADR-0006. Introducing a +second language and toolchain for evaluation alone would duplicate working, +tested infrastructure for no product requirement. + +## Consequences + +**Positive.** ScenarioRank can now detect regressions in coverage, ranking +agreement, pair integrity, stage accounting, and unsupported claims before a +change ships. The harness runs offline, free, in CI, in well under a second. +The first run already found a real production defect (SR-P3A-001, below). + +**Negative / accepted.** A fixture run says nothing about prompt quality — it +exercises orchestration with a scripted provider. Qualitative dimensions +require a human, and no human review has been performed yet. Sixteen synthetic +cases are a development benchmark, not evidence about real decisions. Every one +of these limits is recorded in `docs/evaluation/BENCHMARK_V1.md` and carried +inside the run artifacts themselves. + +**A real defect was found and deliberately not fixed.** `SR-P3A-001`: +`computeRiskAdjustedScore` can return a negative value for a weak candidate, +while `completedPipelineResponseSchema` bounds `risk_adjusted_score` to 0-100, +so `server/http/routes.js` rejects its own response and returns a generic 500 +*after* the model has been paid for. Phase 3A is explicitly forbidden from +changing scoring or contracts, so the benchmark records it as a documented +known defect on the four cases that reproduce it. Known defects do not gate the +exit status, but a case-level check raises a **required** failure if a known +defect stops reproducing — so the record cannot outlive the defect, and a fix +cannot land silently. See `docs/architecture/KNOWN_LIMITATIONS.md` (P0.7). + +## What this decision does not claim + +The benchmark is not scientifically validated, not representative of real +hiring decisions, not evidence of fairness or demographic neutrality, not a +legal-compliance test, not a calibrated-confidence benchmark, and not a +production service-level objective. Nothing in the harness, the documentation, +or the UI may imply otherwise. +# 2026-08-03 hardening addendum + +Known-defect suppression requires complete exact structured findings: an expected finding plus any unrelated finding remains a failure. Live budgeting derives from the frozen production cost/retry policy and verifies the plan before provider construction and before each execution. The released corpus is cross-checked against a repository-level integrity registry; only `eval:update-integrity -- --reason "..."` may add a provenance-bearing release record after reviewer confirmation. diff --git a/docs/evaluation/BENCHMARK_V1.md b/docs/evaluation/BENCHMARK_V1.md new file mode 100644 index 0000000..bb486a7 --- /dev/null +++ b/docs/evaluation/BENCHMARK_V1.md @@ -0,0 +1,247 @@ +# decision-benchmark-v1 + +The first ScenarioRank development benchmark: **16 fully synthetic decision +cases**, 21 scenario executions, covering basic ranking, multi-scenario +behaviour, evidence quality, robustness to input permutation, and pairing. + +## What this benchmark is not + +Stated first, because it is the part most likely to be misread. This benchmark +is **not**: + +- scientifically validated +- representative of real hiring decisions +- evidence of fairness +- evidence of demographic neutrality +- a legal-compliance test +- a calibrated-confidence benchmark +- a production service-level objective + +It is a development benchmark. Its purpose is to make regressions visible +before a prompt, model, or scoring change is attempted. A passing run means the +pipeline still behaves the way this benchmark's authors specified — nothing +more. No documentation, report, or UI in this project may imply otherwise, and +the disclaimer is carried inside every run artifact so it cannot be separated +from the numbers. + +## Synthetic-data policy + +Every candidate, company, role, and record in this benchmark is invented. No +real person, employee, applicant, medical record, protected attribute, or user +data appears anywhere in it. + +The policy is enforced, not just stated: + +- every case declares `synthetic: true` and `data_policy: "synthetic-only"` as + schema **literals**, not booleans a future case could quietly flip; +- the manifest declares `data_policy: "synthetic-only"`; +- a test scans every committed case for email addresses, phone numbers, URLs, + absolute filesystem paths, and secret-shaped strings; +- a test asserts every case file describes its content as fictional, invented, + or synthetic. + +The irrelevant-text variant (`case-014`) appends deliberately mundane, +non-demographic sentences. It tests robustness to noise. It is **not** a +fairness test, and the benchmark makes no demographic claims of any kind. + +## Versioning policy + +`benchmark_id` is `decision-benchmark-v1` and never changes. The rules are +enforced by schema and by test: + +| Change | Required action | +|---|---| +| Changing what an existing case means — its inputs, expectations, or what a pass implies | **New `benchmark_version`**, and by convention a new `benchmark_id` suffix (`decision-benchmark-v2`) | +| Fixing a typo or clarifying prose, with no possible effect on any result | Increment `metadata_revision` and deliberately refresh the reviewed content digest | +| Changing the case or manifest **file shape** | New `schema_version`; runners refuse a version they do not support | +| Adding a new case | New `benchmark_version`; case IDs are append-only | + +Additional invariants: + +- **A case ID never changes and is never reused after publication.** A result + recorded against `case-007` must always mean the same case. +- **Every report records the benchmark version and the git commit.** +- **Runners refuse an unsupported `schema_version`** rather than attempting a + best-effort read of an unknown format. +- **The comparison command refuses to compare across benchmark versions**, so a + benchmark edit can never be mistaken for a pipeline improvement. +- **The released corpus is content-locked.** `release-integrity.json` binds + the benchmark/version/schema/metadata revision to a canonical SHA-256 hash + of the manifest, rubric, and every listed case. Object-key order and JSON + whitespace do not affect it; array order does. Validation never rewrites the + lock. A reviewer must deliberately regenerate the digest after confirming + the appropriate versioning action. +- **Deterministic tests validate every committed case** on every test run. + +The manifest also declares `required_pipeline_version`. Honest scope note: +production does not emit a pipeline version string, and Phase 3A deliberately +did not add one (no production behaviour changed). That field is a marker +maintained by the harness, backed by a real structural probe +(`assertPipelineCompatibility`) that asserts the facts the expectations +actually depend on — seven scoring criteria, at most four logical stages — so a +drift between marker and reality cannot pass unnoticed. + +## Case categories + +| Category | Cases | +|---|---| +| Basic ranking | 001 (dominant + weak), 002 (close call), 003 (different but valid strengths), 007 (strong specific evidence) | +| Multiple scenarios | 004 (two scenarios favouring different skills), 005 (three scenarios requiring trade-offs, including a consistently moderate candidate), 006 (strong in one scenario, weak in another) | +| Evidence quality | 007 (strong specific), 008 (vague unsupported claims), 009 (conflicting evidence), 010 (decision-critical evidence missing) | +| Robustness | 011 (candidate-order permutation), 012 (scenario-order permutation), 013 (semantically equivalent wording), 014 (one irrelevant sentence added) | +| Pairing | 015 (clear complementary pair + duplicate display names, distinct IDs), 016 (the two strongest individuals are not the best pair); pairing is disabled in 001-014 | +| Uncertainty | 008, 009, 010 | + +Tag vocabulary is closed — an unknown tag is rejected, so "cases tagged X" can +never quietly mean "cases someone spelled X-ish": `basic-ranking`, +`multi-scenario`, `close-call`, `missing-evidence`, `conflicting-evidence`, +`permutation`, `duplicate-name`, `pairing`, `uncertainty`. + +## Deterministic expectations versus qualitative opinion + +No case hardcodes one "perfect" natural-language answer. + +**Deterministic expectations** (objectively checkable): + +```text +expected_candidate_ids every candidate, exactly once +pairing_enabled must match the request options +expected_pair_count fully determined by candidate count +expected_best_pair_ids which pair the deterministic score must pick +required_stage_count 3 without pairing, 4 with it +required_scenario_coverage every scenario, in order +allowed_winner_ids every defensible winner, or null for no claim +forbidden_winner_ids winners that would indicate a real problem +required_not_measured_fields concepts that must report "not_measured" +maximum_provider_attempts ceiling on real attempts per execution +expect_human_review_for_candidate_ids who must be flagged for review +``` + +Three levels of winner claim are used deliberately: + +- **A single allowed winner** (001, 007, 010) where one candidate genuinely + dominates or is the only one with relevant evidence. +- **Several allowed winners** (002, 003, 004, 005, 006) where more than one + outcome is legitimately defensible. +- **No winner claim at all** (008, 009) where the evidence is too thin or too + contradictory to justify asserting any winner. Claiming one would be exactly + the overclaiming this benchmark exists to detect. What *is* checked is that + every candidate gets flagged for human review. + +**Qualitative rubric** (human judgment, never automated in Phase 3A): eight +anchored dimensions in `rubric.json` — evidence grounding, scenario relevance, +trade-off clarity, clarity, uncertainty handling, recommendation consistency, +pairing usefulness, unsupported-claim avoidance. Each defines what is judged, a +0-4 scale, an anchor for every point, failure examples, whether human review is +required, and whether any deterministic automation is possible. Where a grader +covers part of a dimension, it covers a conservative subset only and is named +explicitly. See [`HUMAN_REVIEW_GUIDE.md`](HUMAN_REVIEW_GUIDE.md). + +## Current baseline + +`npm run eval:fixtures`, at benchmark version 1.0.0: + +```text +cases: 16/16 passed executions: 21 repetitions: 1 +required failures: 0 advisory failures: 0 known defects: 8 +stages: 65 attempts: 65 tokens: 0 cost: unavailable +fixture machinery: PASSED +production baseline: PASS WITH KNOWN DEFECTS +known defect observations: 8 across 4 scoped executions +unexpected failures: 0 +unexpected defect resolutions: 0 +``` + +Scenario sensitivity is real, not assumed: `case-004` produces different +winners in its two scenarios, and `case-005` produces three different +specialist winners across its three scenarios while never ranking the +consistently moderate candidate first. `case-016` selects +`finnegan-adler::hollis-nakamura` as the best pair even though the two +strongest individuals are `finnegan-adler` and `giselle-varga`. + +Tokens are zero and cost is `unavailable` because the fixture provider reports +no usage and its model name is not in the pricing table. That is correct +behaviour: `estimateCostUsd` returns `null` rather than guessing. + +## Known defect found by this benchmark + +**`SR-P3A-001` — negative `risk_adjusted_score` violates the public contract.** + +Found by the first fixture run ever executed against this benchmark. + +`computeRiskAdjustedScore` (`server/domain/scoring.js`) can legitimately return +a negative value for a sufficiently weak candidate. `completedPipelineResponseSchema` +(`shared/contracts/decisionApi.js`) bounds `risk_adjusted_score` to 0-100. And +`server/http/routes.js:80` calls `completedPipelineResponseSchema.parse(result)` +before responding. + +The consequence is real: an evaluation containing a weak enough candidate +throws inside the route handler, and the user receives a generic +`500 Pipeline failed. Please try again.` — **after** the OpenAI calls have +already been made and paid for. + +Observed values in the benchmark: `-30` (case-001, case-011), `-11.94` +(case-006 second scenario), `-0.4` (case-008). + +Phase 3A is explicitly forbidden from changing scoring, ranking, or public +contracts, so this was **not fixed**. It is recorded only for the four scoped +scenario executions that reproduce its stable semantic signature. It does not +gate the baseline, but a required failure fires if it ever stops reproducing in +that exact scope, so the fix cannot land silently. +Also recorded in `docs/architecture/KNOWN_LIMITATIONS.md` (P0.7). + +Deciding between the two candidate fixes — clamping the score, or widening the +contract — is a Phase 3B question, because either choice changes behaviour the +harness is supposed to be measuring from a fixed baseline. + +## Known limitations of this benchmark + +1. **A fixture run says nothing about prompt or model quality.** The fake + provider is scripted. It validates orchestration, deterministic computation, + and the graders — not the product. +2. **Wording and irrelevant-text variants cannot fail under the fixture.** The + fixture scores by candidate ID and never reads description text, so + `case-013` and `case-014` are guaranteed to match their originals offline. + They validate the linkage and comparison machinery. Only a live run can say + whether wording actually moves a real model's scores. +3. **No human review has been performed.** Every qualitative dimension is + currently unscored. The rubric and template exist; the judgments do not. +4. **Stability is unmeasured.** The baseline runs one repetition, and the + harness reports `not assessed` rather than a meaningless 100%. +5. **`weighted_fit_score` cannot be recomputed** from the public response, + because normalised criterion weights are not exposed. Everything derived + from it is recomputed. +6. **Exact ties are resolved by submission order.** The production ranking is a + stable sort over the submitted candidate array, so an exact tie keeps + submission order — meaning a candidate-order permutation *could* legitimately + change the winner on an exact tie. The benchmark avoids exact ties rather + than baking that behaviour in as an expectation, and the grader checks the + observed behaviour without claiming it is a designed guarantee. +7. **ScenarioRank has no near-tie uncertainty signal.** The deterministic + confidence-and-evidence review keys only on reported confidence and evidence + length, so a decision separated by noise is never flagged for human review. + This is why `case-002` is a close-call case but is *not* tagged + `uncertainty`. A real gap, and a Phase 3B candidate. +8. **Sixteen synthetic cases are a small sample**, written by the same author as + the system. They encode that author's expectations, which is exactly why + qualitative judgment is kept separate and human. +9. **No adversarial, prompt-injection, or malformed-input cases.** Deferred. +10. **No cross-scenario resilience measurement.** The pipeline reports + `not_measured`, and the benchmark checks that it keeps saying so. + +## Adding or changing a case + +1. Decide whether the change alters meaning. If it does, it needs a new + `benchmark_version` — not an edit in place. +2. Add the case file as `cases/case-0NN.json`, using the next unused ID. +3. Add the ID to `manifest.json` and bump `case_count`. +4. Run `npm run eval:validate`. It checks the schema, the manifest/disk + agreement, the production request contract, rubric references, variant + linkage, and pipeline compatibility. +5. Run `npm run eval:fixtures` and confirm the case behaves as designed — + not just that it passes. A case that passes for the wrong reason is worse + than no case. +6. Run `npm run test:evals`. +# Release integrity + +Normal validation cross-checks each local `release-integrity.json` against `evals/datasets/released-benchmark-registry.json`. Formatting-only JSON changes do not change the canonical digest; array order and values do. To update a reviewed release deliberately, change `benchmark_version` for semantic changes or `metadata_revision` for cosmetic changes, confirm that classification with a reviewer, then run `npm run eval:update-integrity -- --benchmark decision-benchmark-v1 --reason "..."`. The command records previous/new digests, reason, timestamp, and version metadata; it never runs automatically, commits, or contacts a network. diff --git a/docs/evaluation/EVALUATION_ARCHITECTURE.md b/docs/evaluation/EVALUATION_ARCHITECTURE.md new file mode 100644 index 0000000..8dd072d --- /dev/null +++ b/docs/evaluation/EVALUATION_ARCHITECTURE.md @@ -0,0 +1,256 @@ +# Evaluation architecture (Phase 3A) + +How ScenarioRank measures itself, why it is built this way, and what a result +is and is not worth. The architectural decision itself is recorded in +[ADR-0009](../decisions/ADR-0009-local-first-evaluation-harness.md). + +## Why evaluation comes before prompt optimisation + +Phase 3A deliberately changes no prompt, no model, no schema, no scoring +formula, no ranking rule, and no pairing behaviour. + +The reason is simple: without a measurement, "this prompt is better" is an +opinion. Optimising first and measuring afterwards produces a system whose +improvements cannot be demonstrated and whose regressions are invisible until a +user finds them. Building the benchmark first means every later change in +Phase 3B has a before-and-after that a reviewer can check. + +There is a second reason specific to this project. ScenarioRank's central +architectural claim is that **LLMs interpret evidence and deterministic code +computes the ranking**. That claim is testable, and most of what the harness +checks is exactly it: does the reported winner match the deterministic score, +can every deterministic value be recomputed, does the narrative ever contradict +the structured result. Those checks needed to exist before anyone started +adjusting the parts of the system that could quietly break them. + +## What the harness answers + +1. Does the pipeline return structurally valid results? +2. Does deterministic ranking agree with the reported winner? +3. Are all scenarios and candidates covered? +4. Does pairing cover every expected candidate pair? +5. Are explanations grounded in the supplied evidence? *(human review)* +6. Does the output acknowledge uncertainty and missing evidence? +7. Does changing candidate order improperly change the result? +8. Does adding scenarios produce sensible scenario-sensitive behaviour? +9. How stable are repeated model runs? *(needs repetitions > 1)* +10. What do runs cost and how long do they take? +11. Did a proposed change improve or regress the benchmark? + +Questions 1-4, 6-8, and 10-11 are answered deterministically. Question 5 is +human-only. Question 9 requires more than one repetition and reports +"not assessed" otherwise. + +## Layout + +```text +evals/ +├── README.md entry point +├── datasets/ +│ ├── loadBenchmark.js strict, fail-closed loading and cross-checks +│ └── decision-benchmark-v1/ +│ ├── manifest.json immutable benchmark identity +│ ├── rubric.json 8 anchored human-review dimensions +│ └── cases/case-0NN.json 16 fully synthetic cases +├── schemas/ +│ ├── benchmarkCase.js case shape, expectations, known defects +│ ├── benchmarkManifest.js manifest, rubric, compatibility probe +│ ├── evaluationRun.js run manifest, case results, artifact policy +│ └── evaluationReport.js human review, comparison report +├── fixtures/ +│ └── fakeProviderProfiles.js 7 offline provider profiles +├── graders/ +│ ├── deterministicGraders.js 11 graders + known-defect handling +│ ├── rubricTemplate.js blank human-review template construction +│ └── humanReview.js review parsing and aggregation +├── runners/ +│ ├── runCase.js one case: N scenarios x R repetitions +│ ├── runBenchmark.js orchestration, stability, permutations +│ ├── compareRuns.js four-verdict comparison +│ ├── caseVariants.js controlled permutation utilities +│ ├── liveRunner.js live gating and budget enforcement +│ └── observingProvider.js records requested IDs only, never payloads +├── reporters/ +│ ├── jsonReporter.js run artifacts, policy-scanned before write +│ └── markdownReporter.js summary.md, console output, comparison.md +└── cli/ validate, fixtures, live, compare +``` + +## Boundaries + +**One-way dependency.** `evals/` imports production — `server/pipeline`, +`server/domain/scoring.js`, `shared/contracts/`. Production imports nothing +from `evals/`. A repository-protection test enforces the direction across +`server/`, `src/`, `shared/`, `scripts/`, and `server.mjs`. + +**No competing schema copies.** The decision output continues to validate +through the real `completedPipelineResponseSchema`. Evaluation schemas *wrap* +production output with benchmark metadata, grader results, and human review; +they never restate a production shape. + +There is one deliberate exception, and it matters: the artifact schema stores +the pipeline response **without** enforcing the public contract. The whole +purpose of the `contract-validity` grader is to detect a response that violates +that contract. If the artifact schema also enforced it, the harness would crash +while recording the very defect it exists to find. Contract validation happens +in exactly one place — the grader — which reports the violation instead of +destroying the evidence. + +**Never in production paths.** No HTTP route, frontend component, or build step +touches the harness. It is invoked only through its four CLI commands. + +## Execution model + +A case declares one or more scenarios. The production request contract takes +exactly one scenario, so the harness executes **one pipeline run per scenario** +rather than inventing a multi-scenario request shape the server cannot serve. +A case with N scenarios at R repetitions produces N x R executions, which are +never collapsed — that separation is what makes per-scenario behaviour and +run-to-run stability visible at all. + +The runner is provider-agnostic: it takes a `createProvider` factory. Fixture +mode supplies an offline fake; live mode supplies the single real provider +instance resolved once. Both go through the identical code path, so a fixture +run genuinely exercises the same orchestration a live run does. + +## Deterministic grading versus qualitative judgment + +**Deterministic graders** answer questions that are objectively true or false. +There are 11, all `required` except where noted: + +| Grader | Checks | +|---|---| +| `contract-validity` | response, run metadata, and every stage event validate against the public contract; no non-finite number | +| `candidate-coverage` | every candidate exactly once, no unknown, contiguous ranks, duplicate names still distinct by ID, and the scoring stage requested the right set | +| `ranking-consistency` | winner is rank 1, order agrees with the deterministic sort field, ties follow documented submission-order behaviour | +| `score-integrity` | scores in range, and every recomputable deterministic value matches a fresh recomputation from `server/domain/scoring.js` | +| `pairing-integrity` | every expected pair evaluated exactly once, canonical IDs, no reversed duplicate, best pair in top pairs, names match IDs, nothing fabricated when disabled | +| `pipeline-accounting` | 3 logical stages without pairing and 4 with, attempts sum correctly, token and cost metadata internally coherent | +| `not-measured-fields` | unmeasured concepts report the literal `not_measured` | +| `winner-expectation` | winner is allowed and not forbidden; skips where a case makes no claim | +| `unsupported-claims` | no fairness, calibration, validation, cross-scenario, or stability overclaim; narrative does not contradict the structured result | +| `uncertainty-acknowledgement` | thin or conflicting evidence produces a human-review recommendation | +| `scenario-coverage` | *(case scope)* every scenario executed and correctly reflected; none silently ignored | + +`score-integrity` cannot recompute `weighted_fit_score`: the normalised +criterion weights are not part of the public response. Everything derived from +it is recomputed, and this limit is stated rather than glossed over. + +The unsupported-claim checks are deliberately conservative. Keyword matching is +not a reliable way to detect overclaiming, and treating it as authoritative +would be its own form of overclaiming. They target a short list of specific, +high-confidence phrases and are scoped to **model-authored narrative fields +only**, so the pipeline's own honest "has not been measured" wording can never +trip them. The narrative-contradiction check is name-based, and is therefore +skipped — with the reason reported — when the winner's display name is shared +by another candidate. + +**Qualitative dimensions** are scored by a human, on an anchored 0-4 scale, +across eight dimensions. See +[`HUMAN_REVIEW_GUIDE.md`](HUMAN_REVIEW_GUIDE.md). Phase 3A implements no +LLM-as-judge grading. + +## Known defects + +A case may declare `known_defects`: graders it is currently expected to fail +because of a documented, pre-existing product defect rather than a problem with +the case. A matching failure becomes `expected_failure` and stops gating the +exit status, so one real finding does not leave the whole baseline red — which +would train everyone to ignore it. + +Three rules keep this from becoming an ordinary suppression: + +- a known defect must name a documented reference; +- it must name the exact scenario indexes and a stable semantic finding code; +- if that grader/finding-code combination stops failing in a declared scenario, + a **required** failure is raised demanding the record be removed. A + known-defect record cannot outlive the defect it describes, and an unrelated + grader failure cannot be suppressed by sharing its grader ID. + +The reproduction check is evaluated per declared execution, because a defect +can legitimately reproduce in one scenario and not another — `case-006` is +exactly that shape. + +One known defect exists today: `SR-P3A-001`, found by the very first fixture +run. See [`BENCHMARK_V1.md`](BENCHMARK_V1.md) and +`docs/architecture/KNOWN_LIMITATIONS.md` (P0.7). + +## Fixture mode + +`npm run eval:fixtures` runs the real pipeline against offline fake providers. +No network access, no API key, no cost, and deterministic decision content. +Seven profiles exist; three are valid and may be declared by a committed case, +and four are deliberately invalid and exist so the graders can be *proven* to +catch real defects rather than only ever observed passing. + +| Profile | Valid | Purpose | +|---|---|---| +| `valid-standard` | yes | complete, well-formed responses | +| `valid-close-call` | yes | compresses scores so ranking margins are small | +| `valid-pairing` | yes | full, valid coverage of every expected pair | +| `malformed-once-then-success` | no | one incomplete batch, then a correct corrective retry — proves attempts rise while logical stages do not | +| `missing-pair` | no | pairing must report itself unavailable, never a partial best pair | +| `unknown-candidate` | no | scoring must fail rather than accept an unsubmitted candidate | +| `contradictory-explanation` | no | narrative recommends the runner-up; must be caught | + +Fixture scores are looked up **by candidate ID, never by array position**. A +candidate-order permutation therefore receives byte-identical scoring input, +which is what makes the permutation check a test of the pipeline rather than of +the fixture. + +## What a fixture run does and does not prove + +**Does prove:** the orchestration runs end to end; the deterministic scoring, +ranking, and pair computation behave as specified; batch-identity validation +rejects what it should; stage and attempt accounting are coherent; the public +contract holds (or, where it does not, the harness says so); the graders +themselves work. + +**Does not prove:** anything about prompt quality, model behaviour, real-world +accuracy, fairness, or stability under a real model. The fake provider is +scripted. A green fixture run means the machinery is sound, not that the +product is good. + +## Live mode + +Live mode exists to answer the questions a scripted provider cannot. It is +gated hard — see [`RUNBOOK.md`](RUNBOOK.md) for the full list. No real OpenAI +call was made at any point during Phase 3A implementation, and no automated +test in this repository calls OpenAI. + +## Comparison + +`npm run eval:compare` produces one of four verdicts: `improved`, `regressed`, +`unchanged`, `inconclusive`. + +Only required-grader invariants can produce `improved` or `regressed` — they +are the only measure in the report that is objectively better or worse. Cost, +token, and duration deltas are reported raw and explicitly marked +`significance: "not_assessed"`, because two runs cannot support a significance +claim. An output change with no invariant change is `inconclusive`, not +`unchanged`: if a candidate run picks a different winner while failing exactly +as many invariants, the honest answer is that the benchmark cannot tell you +which is better. + +The command refuses to compare different benchmarks or benchmark versions, so a +benchmark edit can never be mistaken for a pipeline change. + +## Artifacts + +Runs write to `.eval-runs//`, which is git-ignored. Every artifact is +schema-validated and scanned for secrets and absolute paths **before** it +touches the filesystem — writing first and checking later would leave a leaked +value on disk even if the command then failed. See +[`RUNBOOK.md`](RUNBOOK.md#artifacts-and-privacy). + +## Extending to a hosted evaluation service later + +The seams are already in place. A hosted service would be a new provider +factory plus a reporter. The dataset, the schemas, the graders, the comparison +logic, and the versioning policy are all independent of where execution +happens. This is why Phase 3A did not adopt one: nothing about doing it later +is harder than doing it now, and doing it now would have coupled the benchmark +to a vendor before it had proven itself locally. +# Final Phase 3A hardening + +Every failure detail is derived from a structured finding. A known defect is suppressible only when all and only the scoped findings match its declared observations; an unrelated failure cannot be hidden. Run-state precedence is `unexpected_failure`, `baseline_change_required`, `pass_with_known_defects`, then `clean_pass`. Comparison Markdown reports unchanged, disappeared, new, changed-signature, moved, and count-change observations with repetition-preserving identity. diff --git a/docs/evaluation/HUMAN_REVIEW_GUIDE.md b/docs/evaluation/HUMAN_REVIEW_GUIDE.md new file mode 100644 index 0000000..2ad57c3 --- /dev/null +++ b/docs/evaluation/HUMAN_REVIEW_GUIDE.md @@ -0,0 +1,149 @@ +# Human review guide + +How to score a ScenarioRank evaluation run's qualitative dimensions, and what +those scores are worth. + +## What these scores are + +Structured human opinion. They are **not** measurements. They are not +calibrated, not inter-rater validated, and not evidence that ScenarioRank is +fair, accurate, or production-ready. One reviewer's scores describe one +reviewer's judgment of one run. + +They exist because the deterministic graders genuinely cannot answer the +questions that matter most about an explanation: is this claim grounded in the +evidence supplied, is the trade-off real, would a decision-maker understand it. +Automating those with a second language model would produce numbers nobody +could defend, so Phase 3A does not. + +## Getting a template + +Every run writes `human-review-template.json` into its run directory: + +```bash +npm run eval:fixtures +``` + +The template contains one entry per **completed** execution. Failed executions +are omitted deliberately — there is no explanation to review, and a blank entry +would invite scoring something that does not exist. + +Each entry carries only the dimensions its case declares (`pairing_usefulness` +appears only for pairing cases), and each dimension carries its own anchors, so +you never have to hold the rubric open in another window. + +To record a review, copy the template to `human-review.json` **in the same run +directory**, fill it in, and leave it there. `npm run eval:compare` looks for +that filename. + +## The scale + +| Score | Meaning | +|---|---| +| 0 | unacceptable | +| 1 | major problems | +| 2 | mixed | +| 3 | good | +| 4 | excellent | + +Two non-scores are always available and are never coerced into numbers: + +- `not_applicable` — the dimension genuinely does not apply to this case. +- `cannot_determine` — the output does not contain enough for you to judge. + +**Do not split the difference with a 2.** A 2 means you judged the output and +found it mixed. `cannot_determine` means you did not judge it. Those are +different facts, and the aggregation keeps them apart: neither non-score +contributes to any mean, and both are counted separately. + +Use `reviewer_notes` wherever a score would otherwise be unexplainable to +someone else, and `overall_notes` for anything that spans dimensions. + +## The eight dimensions + +Full anchors for every point are in +`evals/datasets/decision-benchmark-v1/rubric.json` and in the template itself. +Summarised: + +| Dimension | What you are judging | Partial automation | +|---|---|---| +| `evidence_grounding` | Can each claim about a candidate be traced to something actually in that candidate's description? | none | +| `scenario_relevance` | Does the explanation engage with *this* role and *this* scenario, or would it read identically for any? | `scenario-coverage` (structural only) | +| `tradeoff_clarity` | Is what is genuinely given up stated concretely enough to act on? | none | +| `clarity` | Could a non-specialist read it once and correctly state who was recommended, why, and with what reservations? | none | +| `uncertainty_handling` | Are real evidence gaps acknowledged, and is manufactured confidence avoided? | `uncertainty-acknowledgement` (flagging only) | +| `recommendation_consistency` | Does every section support the same candidate the structured result selected? | `unsupported-claims` (name-based subset) | +| `pairing_usefulness` | Does the pair explanation say something about the *combination*, not two summaries side by side? | `pairing-integrity` (structural only) | +| `unsupported_claim_avoidance` | Are claims scoped to what was actually computed? | `unsupported-claims` (phrase-level subset) | + +Where a deterministic grader is named, it covers a **conservative subset** of +that dimension and never replaces your judgment. `unsupported-claims`, for +example, catches a short list of specific phrases and a name-based +contradiction; it cannot tell you whether an argument is sound. + +## How to review + +1. **Read the inputs first.** Open the case file in + `evals/datasets/decision-benchmark-v1/cases/`. Know what evidence actually + existed before you read what the system said about it. Grounding is + impossible to judge in the other order. +2. **Read the structured result before the prose.** Note the winner, the + ranking, and the pairing result. Then read the explanation and judge it + against what you already know the system computed. +3. **Score each dimension independently.** Resist letting a well-written + explanation lift `evidence_grounding`, or a thin case depress `clarity`. + These are separate questions, which is precisely why there are eight. +4. **Prefer a non-score over a guess.** An honest `cannot_determine` is more + useful than an invented 2. +5. **Write the note while you still remember why.** A bare 1 six weeks later is + not actionable. + +## Cases that need extra care + +- **`case-002` (close call).** Any of the three winning is acceptable. What you + are judging is whether the output acknowledges how close it is. A confident + single recommendation with no hedging should score low on + `uncertainty_handling` even though the winner is allowed. +- **`case-008` / `case-009` (thin and conflicting evidence).** The benchmark + makes no winner claim at all here. An output that manufactures specificity + the inputs never contained should score low on `evidence_grounding` + regardless of how confident or fluent it reads. +- **`case-010` (missing evidence).** The leading candidate has nothing on the + criteria that matter most. The right behaviour is to name the gap, not fill + it. +- **`case-015` (duplicate display names).** Two candidates are both called + "Alex Moreau" with distinct IDs. Check the pair explanation refers to the + right one. The automated contradiction check deliberately skips this case, + because a name-based check genuinely cannot distinguish them — so here you + are the only check. +- **`case-016` (pairing).** The two strongest individuals are the *worst* + combination. A pair explanation that just praises the top two has missed the + point. + +## Aggregation + +`aggregateHumanReview()` produces per-dimension statistics — scored count, +`not_applicable` count, `cannot_determine` count, mean, min, max — and these +are **always retained**. + +It also produces a single `aggregate_mean` for convenience, with two +protections: + +- it is `null` unless at least five dimensions were actually scored, because + below that a single number is noise; +- it always carries a caveat stating it must never be reported without the + per-dimension scores it came from. + +Collapsing eight dimensions into one opaque number hides which dimension was +weak, which is the only actionable part of the review. + +## Comparing reviews across runs + +`npm run eval:compare` compares rubric dimensions **only when both runs carry a +completed review with at least one real score**. Two blank templates are never +reported as agreement, and a rubric change never alters the improved/regressed +verdict — that is reserved for deterministic invariants. + +Two reviews by one reviewer are two data points, not a trend. No +inter-rater reliability work has been done in this project, and none should be +implied from these numbers. diff --git a/docs/evaluation/RUNBOOK.md b/docs/evaluation/RUNBOOK.md new file mode 100644 index 0000000..ee8d7e9 --- /dev/null +++ b/docs/evaluation/RUNBOOK.md @@ -0,0 +1,238 @@ +# Evaluation runbook + +Operating instructions for the ScenarioRank evaluation harness. Architecture is +in [`EVALUATION_ARCHITECTURE.md`](EVALUATION_ARCHITECTURE.md); the benchmark +itself is in [`BENCHMARK_V1.md`](BENCHMARK_V1.md). + +## Commands + +All four run from the repository root, support `--help`, reject unknown or +malformed options, emit no ANSI escape codes, and return nonzero on a required +failure. + +```bash +npm run eval:validate +``` + +```bash +npm run eval:fixtures +``` + +```bash +npm run eval:live +``` + +```bash +npm run eval:compare +``` + +Only `eval:live` accesses the network, and only after every guard below has +passed. + +## Validate + +Loads and fully validates a benchmark without executing anything — no pipeline +run, no provider, no artifacts. The cheapest way to find out that a benchmark +edit broke something. + +```bash +npm run eval:validate +``` + +Checks the supported schema version; the manifest, rubric, and every case file; +that the manifest's case list and the files on disk agree exactly; that every +case's decision input validates against the **production** request contract; +that every referenced rubric dimension and known-defect grader exists; that +every variant's `variant_of` resolves; and that the declared pipeline +generation matches this harness. + +## Fixtures + +```bash +npm run eval:fixtures +``` + +Runs the real production pipeline against offline fake providers. No network +access, no API key, no cost, deterministic decision content. Suitable for CI. + +Useful options: + +```bash +npm run eval:fixtures -- --case case-007 --case case-015 +``` + +```bash +npm run eval:fixtures -- --repetitions 3 +``` + +```bash +npm run eval:fixtures -- --no-write +``` + +`--profile ` overrides every case's fake-provider profile. It exists to +check that a grader catches a defect — for example, confirming that a missing +pair really does produce an honest "unavailable" rather than a partial best +pair: + +```bash +npm run eval:fixtures -- --case case-015 --profile missing-pair --no-write +``` + +Deliberately-invalid profiles are never used in the committed baseline. + +Exit status is `0` when every required grader passed, `1` otherwise. Known +defects do not affect it — see below. + +## Live + +Live mode spends real money. **No real OpenAI call was made during Phase 3A +implementation, and no automated test in this repository calls OpenAI.** + +```bash +npm run eval:live -- --live --case case-001 --max-budget-usd 0.25 +``` + +### Safeguards + +| Guard | Behaviour | +|---|---| +| `--live` required | without it nothing is sent; the provider module is never even imported | +| `OPENAI_API_KEY` required | checked before anything is constructed; never recorded in any artifact | +| CI refused by default | `--allow-ci` is required to override. `CI=false` still counts as CI declaring itself | +| Budget required | `--max-budget-usd ` or `EVAL_MAX_BUDGET_USD`; must be positive and finite | +| Plan displayed first | model, case count, repetitions, worst-case call count, worst-case cost, and budget, before the first request | +| Pre-flight refusal | if the worst case exceeds the budget, the run is refused **before** the first call, not stopped part-way | +| Unpriced model refused | a budget that cannot be computed cannot be enforced, and guessing a price would defeat the purpose | +| Between-execution guard | stops **before** starting any execution whose worst case would breach the limit | +| Default repetitions is 1 | never silently multiplied | +| Default case selection is nothing | `--case` is repeatable; the whole benchmark requires `--all-cases` | +| No extra retries | the production retry policy is used unchanged | +| Nothing auto-committed | artifacts go to git-ignored `.eval-runs/` | + +### Budget arithmetic + +The estimate is deliberately pessimistic, because under-estimating is the only +error that costs money. It includes two provider attempts for each context and +decision request; two integrity passes for scoring and, where applicable, +pairing, each with two provider attempts; and truncation-retry output headroom. +It also uses a generous fixed input size per attempt, then prices the result +through the same `server/ai/pricing/openaiPricing.js` table the application +uses. + +One honest caveat, stated in the stop message itself: reported spend excludes +attempts that failed before returning a response body, so true spend can exceed +the reported total. OpenAI's billing dashboard is the source of truth; this is a +budget guard, not an invoice. + +### Recommended first live run + +One case, one repetition, a small budget: + +```bash +npm run eval:live -- --live --case case-001 --max-budget-usd 0.05 +``` + +Read the plan it prints before letting it proceed. + +## Compare + +```bash +npm run eval:compare -- --baseline .eval-runs/run-a --candidate .eval-runs/run-b +``` + +Reads only artifacts already on disk. Writes `comparison.json` and +`comparison.md` into the candidate run directory (or `--out `). + +Verdicts: `improved`, `regressed`, `unchanged`, `inconclusive`. + +What it will not do: + +- claim statistical significance — cost, token, and duration deltas are raw and + marked `not_assessed`; +- compare different benchmarks or benchmark versions; +- compare rubric dimensions unless **both** runs carry a completed human review + with at least one real score; +- compare stability unless **both** runs used more than one repetition; +- call a changed winner a regression — several cases have more than one + defensible winner. + +`--fail-on-regressed` exits nonzero on a `regressed` verdict, for CI use. + +## Artifacts and privacy + +Runs write to `.eval-runs//`, which is git-ignored: + +```text +run-manifest.json identity, versions, commit, totals, grader versions +case-results.jsonl one JSON line per case, with full responses +summary.json counts, grader totals, stability, disclaimer +summary.md readable report +permutations.json variant-versus-original findings +human-review-template.json blank template for a reviewer +``` + +Recorded: run ID, timestamp, benchmark ID/version, rubric version, git commit +and branch, provider, model, case selection, repetition count, pairing case +count, logical stages, provider attempts, token totals, estimated cost, +duration, and grader versions. + +Never recorded: API keys, request headers, request or response bodies, or +machine-specific absolute paths. Every artifact is schema-validated and scanned +for secret- and absolute-path-shaped strings **before** it is written — +checking after writing would leave a leaked value on disk even if the command +then failed. + +The request observer keeps derived identifiers only (candidate IDs, canonical +pair keys, per-stage attempt counts), never prompt or response text. + +Run output is never committed. A run is reproducible from the committed +benchmark plus a commit hash. + +## Known defects in a run + +A case may declare a documented, pre-existing product defect it currently +reproduces. Those failures appear as `known defect` in the report, are listed +in their own prominent section of `summary.md`, and do **not** gate the exit +status — one real finding should not leave the baseline permanently red, which +would train everyone to ignore it. + +The safety catch: every record names its exact scenario indexes and stable +semantic finding code. If that exact known defect stops reproducing, a +**required** failure fires demanding the record be removed. A known-defect +record cannot outlive the defect it describes, and it cannot suppress an +unrelated failure from the same grader. + +If you see `known-defect-still-present:SR-XXX-NNN` fail, something was fixed. +Remove the `known_defects` entry from the case and update the referenced +documentation. + +## CI + +`npm run eval:fixtures` is CI-suitable: offline, free, deterministic, and it +exits nonzero on a required failure. `npm test` already includes +`npm run test:evals`, which covers the harness itself. + +`npm run eval:live` refuses to run in CI by default and should stay that way. + +## Troubleshooting + +**`Unsupported benchmark schema_version`** — the benchmark's file shape is +newer than this harness build. Do not edit the version to make it load; that is +the check working. + +**`Benchmark "..." failed validation`** — the error lists every issue. Nothing +runs until all are fixed; a partially-valid benchmark produces results that +look authoritative and are not. + +**`Refusing to write `** — a secret- or absolute-path-shaped string +reached an artifact. Investigate rather than relaxing the scanner. + +**`Refusing to compare benchmark versions`** — expected. Re-run the baseline at +the current benchmark version. + +**`no recorded pricing for model`** — the model is not in +`server/ai/pricing/openaiPricing.js`. Add it from OpenAI's own pricing page +before running live. +# Current reporting and live safeguards + +Fixture reports lead with run state and list clean cases, expected observations, affected executions, unexpected failures, and unexpected defect resolutions. The baseline is `PASS WITH KNOWN DEFECTS`: 12 clean cases, 8 observations, 4 affected executions, and 0 unexpected failures. Live runs are not part of this baseline and have not been run. A live plan is budget-verified before provider construction and before every execution; budgets are decimal-only and capped at $100, repetitions are decimal integers capped at 20. diff --git a/eslint.config.js b/eslint.config.js index a686807..da751b0 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -46,4 +46,17 @@ export default tseslint.config( globals: globals.node, }, }, + // Evaluation harness (Phase 3A). Linted on the same terms as the backend it + // exercises. It is deliberately a separate config block, not an extension of + // the backend one: evals/ is not production code, production code never + // imports it, and a repository-protection test enforces that direction. + { + extends: [js.configs.recommended], + files: ["evals/**/*.js", "evals/**/*.mjs"], + languageOptions: { + ecmaVersion: 2022, + sourceType: "module", + globals: globals.node, + }, + }, ); diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..111ae11 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,70 @@ +# ScenarioRank evaluation harness + +Local-first, offline-capable evaluation for the ScenarioRank decision pipeline. +Added in **Phase 3A**, which built the measurement infrastructure and changed +no prompt, model, schema, scoring formula, ranking rule, or pairing behaviour. + +## Quick start + +```bash +npm run eval:validate +``` + +```bash +npm run eval:fixtures +``` + +Both are offline. Neither reads an API key or makes a network request. + +## What lives here + +| Path | Purpose | +|---|---| +| `datasets/` | `decision-benchmark-v1` (manifest, rubric, 16 synthetic cases) and the strict loader | +| `schemas/` | benchmark case, manifest/rubric, run artifacts, and report schemas | +| `fixtures/` | seven offline fake-provider profiles | +| `graders/` | 11 deterministic graders, the human-review template, and review aggregation | +| `runners/` | case and benchmark execution, comparison, variants, live gating | +| `reporters/` | JSON run artifacts and markdown summaries | +| `cli/` | `validate`, `fixtures`, `live`, `compare` | + +## Rules this harness follows + +- **Production never imports it.** `evals/` imports the pipeline, the shared + contracts, and the deterministic scoring functions. Nothing under `server/`, + `src/`, `shared/`, or `scripts/` imports `evals/`, and a test enforces it. +- **No competing schema copies.** Decision output validates through the real + public contract. Evaluation schemas wrap it; they never restate it. +- **Fixture mode is offline and free.** No network, no API key, no cost. +- **Live mode is gated.** `--live`, an API key, an explicit budget, and a + deliberate case selection are all required; CI is refused by default. +- **Artifacts are git-ignored** and scanned for secrets and absolute paths + before they are written. +- **Nothing overclaims.** Every artifact carries the scope disclaimer. + +## Documentation + +- [`docs/evaluation/EVALUATION_ARCHITECTURE.md`](../docs/evaluation/EVALUATION_ARCHITECTURE.md) + — design, boundaries, graders, what a run does and does not prove +- [`docs/evaluation/BENCHMARK_V1.md`](../docs/evaluation/BENCHMARK_V1.md) + — cases, versioning policy, current baseline, known limitations +- [`docs/evaluation/HUMAN_REVIEW_GUIDE.md`](../docs/evaluation/HUMAN_REVIEW_GUIDE.md) + — the 0-4 anchored rubric and how to score it +- [`docs/evaluation/RUNBOOK.md`](../docs/evaluation/RUNBOOK.md) + — commands, safeguards, artifacts, troubleshooting +- [`docs/decisions/ADR-0009-local-first-evaluation-harness.md`](../docs/decisions/ADR-0009-local-first-evaluation-harness.md) + — why local-first, and the benchmark-versioning policy + +## Scope + +`decision-benchmark-v1` is a **development benchmark**. It is not +scientifically validated, not representative of real hiring decisions, not +evidence of fairness or demographic neutrality, not a legal-compliance test, +not a calibrated-confidence benchmark, and not a production service-level +objective. Every candidate, company, and record in it is invented. + +A passing fixture run means the orchestration, deterministic computation, and +graders behave as specified. It says nothing about prompt quality. +# Current baseline and safety caps + +The offline fixture baseline is `pass_with_known_defects`: 12 clean cases and 8 expected observations across 4 executions, with no unexpected failures. SR-P3A-001 remains unfixed. Live mode has a $100 per-command safety cap and at most 20 repetitions; it verifies the conservative plan before provider construction and before each execution. No live run has occurred. diff --git a/evals/cli/args.js b/evals/cli/args.js new file mode 100644 index 0000000..b2e322e --- /dev/null +++ b/evals/cli/args.js @@ -0,0 +1,140 @@ +/** + * @file Minimal CLI argument parsing (Phase 3A evaluation harness). + * + * Dependency-free by design: the repository already declines to add packages + * for jobs this small, and an evaluation harness that pulls in a CLI framework + * would add supply-chain surface for no capability. + * + * Output rules these commands follow (docs/evaluation/RUNBOOK.md): + * - no ANSI escape codes, so output stays greppable and diffable; + * - errors name the fix, not just the problem; + * - a required failure exits nonzero. + */ + +/** + * Parses `--flag`, `--key value`, and `--key=value`. Repeated keys collect + * into an array, which is how `--case a --case b` works. + * @param {string[]} argv + * @returns {{ flags: Record, flagCounts: Record, values: Record, positional: string[] }} + */ +export function parseArgs(argv) { + const flags = {}; + const flagCounts = {}; + const values = {}; + const positional = []; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith("--")) { + positional.push(token); + continue; + } + const body = token.slice(2); + const equals = body.indexOf("="); + if (equals !== -1) { + const key = body.slice(0, equals); + const value = body.slice(equals + 1); + (values[key] ??= []).push(value); + continue; + } + const next = argv[index + 1]; + if (next !== undefined && !next.startsWith("--")) { + (values[body] ??= []).push(next); + index += 1; + continue; + } + flags[body] = true; + flagCounts[body] = (flagCounts[body] ?? 0) + 1; + } + + return { flags, flagCounts, values, positional }; +} + +/** + * Rejects misspelled, positional, and value/flag-shape mistakes before a CLI + * performs any work. A harness must never quietly treat an unknown option as + * permission to run a different command (especially a whole benchmark run). + */ +export function assertAllowedArgs(parsed, { flags = [], values = [], singleValues = [] }) { + const allowedFlags = new Set(flags); + const allowedValues = new Set(values); + const unknown = [ + ...Object.keys(parsed.flags), + ...Object.keys(parsed.values), + ].filter((key) => !allowedFlags.has(key) && !allowedValues.has(key)); + if (unknown.length > 0) { + throw new Error(`Unknown option(s): ${unknown.map((key) => `--${key}`).join(", ")}. Run with --help to see supported options.`); + } + + const valuesUsedAsFlags = Object.keys(parsed.flags).filter((key) => allowedValues.has(key)); + if (valuesUsedAsFlags.length > 0) { + throw new Error(`Option(s) require a value: ${valuesUsedAsFlags.map((key) => `--${key}`).join(", ")}.`); + } + const flagsUsedAsValues = Object.keys(parsed.values).filter((key) => allowedFlags.has(key)); + if (flagsUsedAsValues.length > 0) { + throw new Error(`Option(s) do not accept a value: ${flagsUsedAsValues.map((key) => `--${key}`).join(", ")}.`); + } + const duplicateFlags = Object.entries(parsed.flagCounts ?? {}).filter(([, count]) => count > 1).map(([key]) => key); + if (duplicateFlags.length > 0) { + throw new Error(`Option(s) may be used only once: ${duplicateFlags.map((key) => `--${key}`).join(", ")}. Run with --help to see supported options.`); + } + const duplicateValues = singleValues.filter((key) => (parsed.values[key]?.length ?? 0) > 1); + if (duplicateValues.length > 0) { + throw new Error(`Option(s) may be supplied only once: ${duplicateValues.map((key) => `--${key}`).join(", ")}. Run with --help to see supported options.`); + } + if (parsed.positional.length > 0) { + throw new Error(`Unexpected positional argument(s): ${parsed.positional.join(", ")}. Run with --help to see supported options.`); + } +} + +/** @returns {string|undefined} the last value given for a key */ +export function single(values, key) { + const list = values[key]; + return list === undefined ? undefined : list[list.length - 1]; +} + +/** @returns {string[]} every value given for a key */ +export function many(values, key) { + return values[key] ?? []; +} + +/** + * Parses an integer option, rejecting anything that is not exactly an integer. + * `--repetitions 2.5` silently becoming 2 would be worse than an error. + * @param {string|undefined} raw + * @param {string} label + * @param {number} fallback + */ +export function integerOption(raw, label, fallback) { + if (raw === undefined) return fallback; + if (!/^\d+$/.test(raw)) { + throw new Error(`Invalid ${label} "${raw}". It must be a decimal integer.`); + } + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Invalid ${label} "${raw}". It must be a safe decimal integer.`); + } + return parsed; +} + +/** + * Prints help and exits 0. Help is always available and never requires a + * valid configuration to reach. + * @param {string} text + */ +export function showHelp(text) { + process.stdout.write(`${text.trimEnd()}\n`); + process.exit(0); +} + +/** + * Reports a failure the way the runbook promises: a plain message on stderr + * and a nonzero exit status. + * @param {unknown} error + * @param {number} [code] + */ +export function failWith(error, code = 1) { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`${message}\n`); + process.exit(code); +} diff --git a/evals/cli/compare.mjs b/evals/cli/compare.mjs new file mode 100644 index 0000000..e0db496 --- /dev/null +++ b/evals/cli/compare.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * `npm run eval:compare` + * + * Compares two recorded runs and reports one of four verdicts. Reads only + * artifacts already on disk: no pipeline runs, no provider, no network. + */ +import path from "node:path"; +import { writeFile } from "node:fs/promises"; + +import { parseArgs, assertAllowedArgs, single, showHelp, failWith } from "./args.js"; +import { readRunArtifacts } from "../reporters/jsonReporter.js"; +import { compareRuns } from "../runners/compareRuns.js"; +import { renderComparisonMarkdown } from "../reporters/markdownReporter.js"; +import { assertArtifactIsPolicyClean } from "../schemas/evaluationRun.js"; + +const HELP = ` +eval:compare — compare two recorded evaluation runs + +Usage: + npm run eval:compare -- --baseline .eval-runs/run-a --candidate .eval-runs/run-b + +Required: + --baseline Run directory to compare against + --candidate Run directory being assessed + +Options: + --out Write comparison.json and comparison.md here + (default: the candidate run directory) + --fail-on-regressed Exit nonzero when the verdict is "regressed" + --help Show this help + +Verdicts: + improved required-grader failures fell + regressed required-grader failures rose + unchanged no invariant, decision, or explanation difference + inconclusive output changed without any invariant change, or the runs + could not be meaningfully compared + +What it deliberately does not do: + - it never claims statistical significance; cost, token, and duration + deltas are reported raw and marked "not_assessed" + - it refuses to compare different benchmarks or benchmark versions + - it compares rubric dimensions only when both runs carry a completed + human review containing at least one real score + - it compares stability only when both runs used more than one repetition + +Exit status: + 0 the comparison completed + 1 the runs could not be compared, or --fail-on-regressed and the verdict + was "regressed" +`; + +async function main() { + const parsed = parseArgs(process.argv.slice(2)); + const { flags, values } = parsed; + assertAllowedArgs(parsed, { flags: ["help", "fail-on-regressed"], values: ["baseline", "candidate", "out"], singleValues: ["baseline", "candidate", "out"] }); + if (flags.help) showHelp(HELP); + + const baselineDir = single(values, "baseline"); + const candidateDir = single(values, "candidate"); + if (!baselineDir || !candidateDir) { + throw new Error( + "Both --baseline and --candidate are required. Each must be a run directory produced by eval:fixtures or eval:live.", + ); + } + + let baseline; + let candidate; + try { + baseline = await readRunArtifacts(path.resolve(baselineDir)); + candidate = await readRunArtifacts(path.resolve(candidateDir)); + } catch { + throw new Error("Could not read one or both run directories. Pass directories produced by eval:fixtures or eval:live."); + } + const report = compareRuns(baseline, candidate); + + const markdown = renderComparisonMarkdown(report); + process.stdout.write(`${markdown}\n`); + + const outDir = path.resolve(single(values, "out") ?? candidateDir); + const json = `${JSON.stringify(report, null, 2)}\n`; + assertArtifactIsPolicyClean(json, "comparison.json"); + assertArtifactIsPolicyClean(markdown, "comparison.md"); + await writeFile(path.join(outDir, "comparison.json"), json, "utf8"); + await writeFile(path.join(outDir, "comparison.md"), `${markdown}\n`, "utf8"); + + if (flags["fail-on-regressed"] && report.verdict === "regressed") { + process.exit(1); + } +} + +main().catch(failWith); diff --git a/evals/cli/fixtures.mjs b/evals/cli/fixtures.mjs new file mode 100644 index 0000000..349a0d1 --- /dev/null +++ b/evals/cli/fixtures.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node +/** + * `npm run eval:fixtures` + * + * Runs the benchmark against the real pipeline with offline fake providers. + * No network access, no API key, no cost, deterministic decision content. + * Suitable for CI, and the command a change to prompts, scoring, or ranking + * should be measured against — before and after. + */ +import { parseArgs, assertAllowedArgs, single, many, integerOption, showHelp, failWith } from "./args.js"; +import { loadBenchmark, DEFAULT_BENCHMARK_ID } from "../datasets/loadBenchmark.js"; +import { createEvalFakeProvider } from "../fixtures/fakeProviderProfiles.js"; +import { runBenchmark } from "../runners/runBenchmark.js"; +import { buildHumanReviewTemplate } from "../graders/rubricTemplate.js"; +import { writeRunArtifacts } from "../reporters/jsonReporter.js"; +import { renderRunMarkdown, renderConsoleSummary } from "../reporters/markdownReporter.js"; + +const HELP = ` +eval:fixtures — run the benchmark offline against the real pipeline + +Usage: + npm run eval:fixtures -- [options] + +Options: + --benchmark Benchmark to run (default: ${DEFAULT_BENCHMARK_ID}) + --case Run only this case; repeatable. Default: every case + --repetitions Executions per case+scenario (default: 1) + --profile Override every case's fake-provider profile. Intended + for deliberately-invalid profiles when checking that a + grader catches a defect + --no-write Grade and print, but write no run artifacts + --help Show this help + +Behaviour: + - executes the real production pipeline with an offline fake provider + - makes zero network requests and reads no API key + - writes artifacts to .eval-runs// (git-ignored) + - decision content is deterministic; timestamps, durations, and request_id + are not, and are excluded from every comparison + +Exit status: + 0 every required grader passed + 1 at least one required grader failed, or the run could not complete +`; + +async function main() { + const parsed = parseArgs(process.argv.slice(2)); + const { flags, values } = parsed; + assertAllowedArgs(parsed, { flags: ["help", "no-write"], values: ["benchmark", "case", "repetitions", "profile"], singleValues: ["benchmark", "repetitions", "profile"] }); + if (flags.help) showHelp(HELP); + + const benchmarkId = single(values, "benchmark") ?? DEFAULT_BENCHMARK_ID; + const repetitions = integerOption(single(values, "repetitions"), "--repetitions", 1); + if (repetitions < 1) throw new Error("--repetitions must be at least 1."); + + const benchmark = await loadBenchmark({ benchmarkId }); + const requested = many(values, "case"); + const known = new Set(benchmark.cases.map((entry) => entry.case_id)); + const unknown = requested.filter((caseId) => !known.has(caseId)); + if (unknown.length > 0) { + throw new Error( + `Unknown case id(s): ${unknown.join(", ")}. Run \`npm run eval:validate\` to see this benchmark's cases.`, + ); + } + + // Unlike live mode, running everything is the sensible default here: a + // fixture run is free, offline, and fast. + const caseIds = requested.length > 0 ? requested : benchmark.cases.map((entry) => entry.case_id); + const profileOverride = single(values, "profile"); + + const run = await runBenchmark({ + benchmark, + caseIds, + mode: "fixtures", + provider: "fake-eval", + model: profileOverride ? `fixture:${profileOverride}` : "fixture:per-case", + repetitions, + createProvider: ({ benchmarkCase, scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile: profileOverride }), + }); + + process.stdout.write(`${renderConsoleSummary(run)}\n`); + + if (!flags["no-write"]) { + const { runDir, files } = await writeRunArtifacts({ + run, + markdown: renderRunMarkdown(run), + humanReviewTemplate: buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: run.manifest, + caseResults: run.caseResults, + casesById: new Map(benchmark.cases.map((entry) => [entry.case_id, entry])), + }), + }); + // Relative, so the line is identical on every machine and safe to paste. + process.stdout.write( + `artifacts: .eval-runs/${run.manifest.run_id}/ (${files.join(", ")})\n`, + ); + void runDir; + } + + if (!run.passed) { + process.stderr.write( + `\n${run.summary.required_failures} required grader failure(s). See summary.md in the run directory for details.\n`, + ); + process.exit(1); + } +} + +main().catch(failWith); diff --git a/evals/cli/live.mjs b/evals/cli/live.mjs new file mode 100644 index 0000000..7ffd570 --- /dev/null +++ b/evals/cli/live.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +/** + * `npm run eval:live` + * + * Runs benchmark cases against the real OpenAI provider. Every guard in + * evals/runners/liveRunner.js applies, and the provider module is imported + * lazily so that merely loading this file — for `--help`, or for a test — + * never constructs an OpenAI client. + * + * No real OpenAI call was made by this command during Phase 3A + * implementation. Its refusal paths and budget arithmetic are covered by + * tests that inject values rather than calling the API. + */ +import { parseArgs, assertAllowedArgs, single, many, integerOption, showHelp, failWith } from "./args.js"; +import { loadBenchmark, DEFAULT_BENCHMARK_ID } from "../datasets/loadBenchmark.js"; +import { runBenchmark } from "../runners/runBenchmark.js"; +import { buildHumanReviewTemplate } from "../graders/rubricTemplate.js"; +import { writeRunArtifacts } from "../reporters/jsonReporter.js"; +import { renderRunMarkdown, renderConsoleSummary } from "../reporters/markdownReporter.js"; +import { + assertLiveModeAllowed, + assertBudgetCoversPlan, + estimateRunBudget, + createBudgetGuard, + renderLivePlan, +} from "../runners/liveRunner.js"; +import { resolveOpenAIModel } from "../../server/config/env.js"; + +const HELP = ` +eval:live — run benchmark cases against the real OpenAI provider (spends money) + +Usage: + npm run eval:live -- --live --case case-001 --max-budget-usd 0.25 + +Required: + --live Explicit opt-in. Without it, nothing is sent + --max-budget-usd Hard budget limit (or set EVAL_MAX_BUDGET_USD) + --case Case to run; repeatable + or --all-cases Run every case in the benchmark, deliberately + +Options: + --benchmark Benchmark to run (default: ${DEFAULT_BENCHMARK_ID}) + --repetitions Executions per case+scenario (default: 1) + --allow-ci Permit running inside CI (refused by default) + --help Show this help + +Safeguards: + - OPENAI_API_KEY must be set; it is never recorded in any artifact + - refuses to run in CI unless --allow-ci is passed + - refuses without an explicit, positive, finite budget limit + - refuses a model that has no recorded pricing, because an unpriced run + cannot be budget-enforced + - prints the model, case count, repetitions, worst-case call count, and + worst-case cost before the first request + - refuses to start if the worst case exceeds the budget + - stops before starting any execution that could exceed the budget + - default repetitions is 1, and no case is selected by default + +Cost note: + Reported spend excludes attempts that failed before returning a response + body, so true spend can exceed the reported total. OpenAI's billing + dashboard is the source of truth. + +Exit status: + 0 every required grader passed + 1 refused, failed to complete, or a required grader failed +`; + +async function main() { + const parsed = parseArgs(process.argv.slice(2)); + const { flags, values } = parsed; + assertAllowedArgs(parsed, { flags: ["help", "live", "all-cases", "allow-ci"], values: ["benchmark", "case", "repetitions", "max-budget-usd"], singleValues: ["benchmark", "repetitions", "max-budget-usd"] }); + if (flags.help) showHelp(HELP); + + const benchmarkId = single(values, "benchmark") ?? DEFAULT_BENCHMARK_ID; + const benchmark = await loadBenchmark({ benchmarkId }); + + const { budgetUsd, selectedIds, repetitions } = assertLiveModeAllowed({ + live: Boolean(flags.live), + allowCi: Boolean(flags["allow-ci"]), + caseIds: many(values, "case"), + allCases: Boolean(flags["all-cases"]), + repetitions: integerOption(single(values, "repetitions"), "--repetitions", 1), + maxBudgetUsd: single(values, "max-budget-usd"), + benchmarkCases: benchmark.cases, + }); + const selectedCases = selectedIds.map((caseId) => + benchmark.cases.find((entry) => entry.case_id === caseId), + ); + const model = resolveOpenAIModel(); + const estimate = estimateRunBudget(selectedCases, model, repetitions); + assertBudgetCoversPlan(estimate, budgetUsd); + process.stdout.write(`${renderLivePlan({ model, selectedIds, repetitions, estimate, budgetUsd })}\n\n`); + + // Imported and constructed only after the entire plan fits the declared + // budget, so a refused plan cannot instantiate an OpenAI client. + const { createProvider } = await import("../../server/ai/providerFactory.js"); + const provider = createProvider(); + + const guard = createBudgetGuard({ budgetUsd, model }); + const stopped = []; + + const run = await runBenchmark({ + benchmark, + caseIds: selectedIds, + mode: "live", + provider: provider.name, + model, + repetitions, + createProvider: () => provider, + onExecution: (execution) => { + guard.record(execution.response?.run_metadata.estimatedCostUsd ?? null); + }, + beforeExecution: ({ benchmarkCase }) => { + if (guard.canProceed(benchmarkCase)) return true; + stopped.push(benchmarkCase.case_id); + return false; + }, + }); + + process.stdout.write(`${renderConsoleSummary(run)}\n`); + process.stdout.write(`reported spend: $${guard.spentUsd.toFixed(6)} of $${budgetUsd.toFixed(6)} budget\n`); + if (stopped.length > 0) { + process.stdout.write(`not executed (budget): ${stopped.join(", ")}\n`); + } + if (guard.stoppedReason) { + process.stdout.write(`${guard.stoppedReason}\n`); + } + + const { files } = await writeRunArtifacts({ + run, + markdown: renderRunMarkdown(run), + humanReviewTemplate: buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: run.manifest, + caseResults: run.caseResults, + casesById: new Map(benchmark.cases.map((entry) => [entry.case_id, entry])), + }), + }); + process.stdout.write(`artifacts: .eval-runs/${run.manifest.run_id}/ (${files.join(", ")})\n`); + + if (!run.passed) process.exit(1); +} + +main().catch(failWith); diff --git a/evals/cli/update-integrity.mjs b/evals/cli/update-integrity.mjs new file mode 100644 index 0000000..d295e6d --- /dev/null +++ b/evals/cli/update-integrity.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +/** Deliberate, local-only release-integrity update with review provenance. */ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { parseArgs, assertAllowedArgs, single, showHelp, failWith } from "./args.js"; +import { DEFAULT_BENCHMARK_ID, REPO_ROOT } from "../datasets/loadBenchmark.js"; +import { benchmarkContentDigest, RELEASE_REGISTRY_PATH, readReleaseRegistry } from "../datasets/releasedBenchmarkIntegrity.js"; + +const HELP = ` +eval:update-integrity — deliberately record a reviewed benchmark release + +Usage: + npm run eval:update-integrity -- --benchmark decision-benchmark-v1 --reason "reviewed change" + +This command never commits or contacts a network. Confirm whether a change is +semantic or cosmetic before running it; normal validation never changes locks. +`; + +async function main() { + const parsed = parseArgs(process.argv.slice(2)); + assertAllowedArgs(parsed, { flags: ["help"], values: ["benchmark", "reason"], singleValues: ["benchmark", "reason"] }); + if (parsed.flags.help) showHelp(HELP); + const benchmarkId = single(parsed.values, "benchmark") ?? DEFAULT_BENCHMARK_ID; + const reason = single(parsed.values, "reason")?.trim(); + if (!reason) throw new Error("--reason is required and must be nonempty; confirm the release change with a reviewer first."); + const benchmarkDir = path.join(REPO_ROOT, "evals", "datasets", benchmarkId); + const manifest = JSON.parse(await readFile(path.join(benchmarkDir, "manifest.json"), "utf8")); + const registry = await readReleaseRegistry(); + const prior = registry.records + .filter((record) => record.benchmark_id === manifest.benchmark_id) + .at(-1); + if (prior && prior.benchmark_version === manifest.benchmark_version && prior.metadata_revision === manifest.metadata_revision) { + throw new Error("Refusing integrity update: change benchmark_version for semantic changes or metadata_revision for cosmetic changes first."); + } + const digest = await benchmarkContentDigest(benchmarkDir, manifest); + const timestamp = new Date().toISOString(); + const record = { + benchmark_id: manifest.benchmark_id, + benchmark_version: manifest.benchmark_version, + schema_version: manifest.schema_version, + metadata_revision: manifest.metadata_revision, + previous_digest: prior?.digest ?? null, + digest, + reason, + timestamp, + }; + const localPath = path.join(benchmarkDir, "release-integrity.json"); + const local = { + benchmark_id: manifest.benchmark_id, + benchmark_version: manifest.benchmark_version, + schema_version: manifest.schema_version, + metadata_revision: manifest.metadata_revision, + digest, + }; + process.stdout.write(`Will write:\n - evals/datasets/${benchmarkId}/release-integrity.json\n - evals/datasets/released-benchmark-registry.json\n`); + await writeFile(localPath, `${JSON.stringify(local, null, 2)}\n`); + await writeFile(RELEASE_REGISTRY_PATH, `${JSON.stringify({ records: [...registry.records, record] }, null, 2)}\n`); + process.stdout.write(`Updated ${benchmarkId}: ${prior?.digest ?? "none"} -> ${digest}\n`); +} + +main().catch(failWith); diff --git a/evals/cli/validate.mjs b/evals/cli/validate.mjs new file mode 100644 index 0000000..c330243 --- /dev/null +++ b/evals/cli/validate.mjs @@ -0,0 +1,94 @@ +#!/usr/bin/env node +/** + * `npm run eval:validate` + * + * Loads and fully validates a benchmark without executing anything. No + * pipeline runs, no provider is constructed, no network access, no artifacts + * written. This is the cheapest way to find out that a benchmark edit broke + * something. + */ +import { parseArgs, assertAllowedArgs, single, showHelp, failWith } from "./args.js"; +import { loadBenchmark, DEFAULT_BENCHMARK_ID } from "../datasets/loadBenchmark.js"; +import { FAKE_PROVIDER_PROFILES, VALID_BASELINE_PROFILES } from "../fixtures/fakeProviderProfiles.js"; +import { ALL_GRADERS, GRADER_SUITE_VERSION } from "../graders/deterministicGraders.js"; + +const HELP = ` +eval:validate — validate a benchmark's manifest, rubric, and cases + +Usage: + npm run eval:validate -- [options] + +Options: + --benchmark Benchmark to validate (default: ${DEFAULT_BENCHMARK_ID}) + --help Show this help + +What it checks: + - the manifest schema_version is one this harness supports + - the manifest, rubric, and every case file validate against their schemas + - the manifest's case list and the case files on disk agree exactly + - every case's decision input validates against the production public + request contract in shared/contracts/decisionApi.js + - every rubric dimension a case references exists + - every variant's variant_of points at a case that exists + - the benchmark's declared pipeline generation matches this harness + - every committed case declares synthetic-only data and a valid baseline + fake-provider profile + +Exit status: + 0 the benchmark is valid + 1 validation failed (details on stderr) + +No pipeline is executed and no network request is made. +`; + +async function main() { + const parsed = parseArgs(process.argv.slice(2)); + const { flags, values } = parsed; + assertAllowedArgs(parsed, { flags: ["help"], values: ["benchmark"], singleValues: ["benchmark"] }); + if (flags.help) showHelp(HELP); + + const benchmarkId = single(values, "benchmark") ?? DEFAULT_BENCHMARK_ID; + const { manifest, rubric, cases } = await loadBenchmark({ benchmarkId }); + + // Checked here rather than in the schema: which profiles are acceptable for + // a *committed baseline* case is a policy of this harness build, not part of + // the case file format. + const badProfiles = cases + .filter((entry) => !VALID_BASELINE_PROFILES.includes(entry.fake_provider_plan.profile)) + .map((entry) => `${entry.case_id}: "${entry.fake_provider_plan.profile}"`); + if (badProfiles.length > 0) { + throw new Error( + `Committed cases must declare a valid baseline fake-provider profile ` + + `(${VALID_BASELINE_PROFILES.join(", ")}). Invalid profiles found:\n - ${badProfiles.join("\n - ")}\n` + + "Deliberately-invalid profiles exist to prove the graders catch real defects, and belong in targeted tests, not in the committed baseline.", + ); + } + + const tagCounts = {}; + for (const entry of cases) { + for (const tag of entry.tags) tagCounts[tag] = (tagCounts[tag] ?? 0) + 1; + } + + const lines = [ + `benchmark: ${manifest.benchmark_id} v${manifest.benchmark_version} (schema ${manifest.schema_version}, metadata revision ${manifest.metadata_revision})`, + `rubric: v${rubric.rubric_version}, ${rubric.dimensions.length} dimension(s)`, + `cases: ${cases.length} valid`, + `pairing cases: ${cases.filter((entry) => entry.deterministic_expectations.pairing_enabled).length}`, + `variant cases: ${cases.filter((entry) => entry.variant_of !== null).length}`, + `multi-scenario: ${cases.filter((entry) => entry.input.scenarios.length > 1).length}`, + `scenarios total: ${cases.reduce((total, entry) => total + entry.input.scenarios.length, 0)}`, + `graders: ${ALL_GRADERS.length} (suite ${GRADER_SUITE_VERSION})`, + `fixture profiles: ${Object.keys(FAKE_PROVIDER_PROFILES).length} (${VALID_BASELINE_PROFILES.length} valid for committed cases)`, + `data policy: ${manifest.data_policy}`, + "", + "tags:", + ...Object.entries(tagCounts) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([tag, count]) => ` ${tag.padEnd(22)} ${count}`), + "", + manifest.scope_disclaimer, + ]; + process.stdout.write(`${lines.join("\n")}\n`); +} + +main().catch(failWith); diff --git a/evals/datasets/benchmarkDataset.test.js b/evals/datasets/benchmarkDataset.test.js new file mode 100644 index 0000000..902491e --- /dev/null +++ b/evals/datasets/benchmarkDataset.test.js @@ -0,0 +1,260 @@ +/** + * Committed-benchmark validation tests. + * + * These assert properties of the *actual dataset in the repository*, not of + * the schema. They are what stops a benchmark edit from quietly changing what + * the benchmark measures, or from introducing anything that is not synthetic. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync, readdirSync } from "node:fs"; +import path from "node:path"; + +import { loadBenchmark, toRepoRelative, BenchmarkValidationError } from "./loadBenchmark.js"; +import { VALID_BASELINE_PROFILES } from "../fixtures/fakeProviderProfiles.js"; +import { + readReleaseRegistry, + assertReleasedBenchmarkIntegrity, + benchmarkContentDigest, + canonicalJson, +} from "./releasedBenchmarkIntegrity.js"; +import { evaluationRequestSchema } from "../../shared/contracts/decisionApi.js"; + +const DATASET = path.resolve("evals/datasets/decision-benchmark-v1"); + +const benchmark = await loadBenchmark(); + +describe("committed benchmark", () => { + it("loads, validates, and matches its manifest", () => { + expect(benchmark.manifest.benchmark_id).toBe("decision-benchmark-v1"); + expect(benchmark.cases).toHaveLength(benchmark.manifest.case_count); + expect(benchmark.cases.map((entry) => entry.case_id)).toEqual(benchmark.manifest.case_ids); + }); + + it("matches the reviewed content lock for this released benchmark", async () => { + const registry = await readReleaseRegistry(); + const record = registry.records.find((entry) => + entry.benchmark_id === benchmark.manifest.benchmark_id && + entry.benchmark_version === benchmark.manifest.benchmark_version && + entry.metadata_revision === benchmark.manifest.metadata_revision, + ); + expect(record?.digest).toBeTruthy(); + await expect(benchmarkContentDigest(DATASET, benchmark.manifest)).resolves.toBe( + record.digest, + ); + }); + + it("refuses an unregistered version instead of letting a version bump bypass the lock", async () => { + await expect( + assertReleasedBenchmarkIntegrity(DATASET, { + ...benchmark.manifest, + benchmark_version: "999.0.0", + }), + ).rejects.toThrow(/invalid release-integrity/i); + }); + + it("canonicalizes object keys while preserving meaningful array order", () => { + expect(canonicalJson({ b: ["second", "first"], a: 1 })).toBe( + canonicalJson({ a: 1, b: ["second", "first"] }), + ); + expect(canonicalJson({ a: ["first", "second"] })).not.toBe( + canonicalJson({ a: ["second", "first"] }), + ); + }); + + it("contains between 12 and 16 cases", () => { + expect(benchmark.cases.length).toBeGreaterThanOrEqual(12); + expect(benchmark.cases.length).toBeLessThanOrEqual(16); + }); + + it("orders cases by the manifest, not by the filesystem", () => { + const onDisk = readdirSync(path.join(DATASET, "cases")).sort(); + expect(onDisk).toHaveLength(benchmark.cases.length); + expect(benchmark.cases.map((entry) => entry.case_id)).toEqual(benchmark.manifest.case_ids); + }); + + it("declares every case synthetic-only", () => { + for (const entry of benchmark.cases) { + expect(entry.synthetic, entry.case_id).toBe(true); + expect(entry.data_policy, entry.case_id).toBe("synthetic-only"); + } + }); + + it("uses only valid baseline fake-provider profiles", () => { + for (const entry of benchmark.cases) { + expect(VALID_BASELINE_PROFILES, entry.case_id).toContain(entry.fake_provider_plan.profile); + } + }); + + it("validates every scenario against the production request contract", () => { + for (const entry of benchmark.cases) { + for (const scenario of entry.input.scenarios) { + const result = evaluationRequestSchema.safeParse({ + role: entry.input.role, + scenario, + decision_mode: entry.input.decision_mode, + candidates: entry.input.candidates, + options: entry.input.options, + }); + expect(result.success, `${entry.case_id}: ${scenario}`).toBe(true); + } + } + }); + + it("covers every category the benchmark claims to cover", () => { + const tags = new Set(benchmark.cases.flatMap((entry) => entry.tags)); + for (const required of [ + "basic-ranking", + "multi-scenario", + "close-call", + "missing-evidence", + "conflicting-evidence", + "permutation", + "duplicate-name", + "pairing", + "uncertainty", + ]) { + expect(tags, required).toContain(required); + } + }); + + it("covers both pairing-disabled and pairing-enabled cases", () => { + const enabled = benchmark.cases.filter((e) => e.deterministic_expectations.pairing_enabled); + const disabled = benchmark.cases.filter((e) => !e.deterministic_expectations.pairing_enabled); + expect(enabled.length).toBeGreaterThan(0); + expect(disabled.length).toBeGreaterThan(0); + }); + + it("covers all four variant kinds with valid linkage", () => { + const variants = benchmark.cases.filter((entry) => entry.variant_of !== null); + const kinds = new Set(variants.map((entry) => entry.variant_kind)); + expect(kinds).toEqual( + new Set(["candidate-order", "scenario-order", "equivalent-wording", "irrelevant-text"]), + ); + + const byId = new Map(benchmark.cases.map((entry) => [entry.case_id, entry])); + for (const variant of variants) { + const original = byId.get(variant.variant_of); + expect(original, variant.case_id).toBeDefined(); + // Candidate IDs are the link between a variant and its original. Losing + // them would make the comparison meaningless. + expect( + [...variant.input.candidates.map((c) => c.id)].sort(), + variant.case_id, + ).toEqual([...original.input.candidates.map((c) => c.id)].sort()); + } + }); + + it("includes a case with duplicate display names and distinct IDs", () => { + const duplicate = benchmark.cases.find((entry) => entry.tags.includes("duplicate-name")); + expect(duplicate).toBeDefined(); + const names = duplicate.input.candidates.map((c) => c.name); + const ids = duplicate.input.candidates.map((c) => c.id); + expect(new Set(names).size).toBeLessThan(names.length); + expect(new Set(ids).size).toBe(ids.length); + }); + + it("includes single-scenario and multi-scenario cases, within contract limits", () => { + const counts = benchmark.cases.map((entry) => entry.input.scenarios.length); + expect(Math.min(...counts)).toBe(1); + expect(Math.max(...counts)).toBeGreaterThan(1); + expect(Math.max(...counts)).toBeLessThanOrEqual(5); + }); + + it("uses allowed_winner_ids with more than one option for close-call cases", () => { + const closeCalls = benchmark.cases.filter((entry) => entry.tags.includes("close-call")); + expect(closeCalls.length).toBeGreaterThan(0); + for (const entry of closeCalls) { + const allowed = entry.deterministic_expectations.allowed_winner_ids; + // Either several winners are defensible, or the case declines to make a + // winner claim at all. A single hard-coded winner would be dishonest for + // a case that exists because the decision is genuinely close. + expect(allowed === null || allowed.length > 1, entry.case_id).toBe(true); + } + }); + + it("expects human review wherever it claims thin or conflicting evidence", () => { + const uncertain = benchmark.cases.filter((entry) => entry.tags.includes("uncertainty")); + expect(uncertain.length).toBeGreaterThan(0); + for (const entry of uncertain) { + expect( + entry.deterministic_expectations.expect_human_review_for_candidate_ids.length, + entry.case_id, + ).toBeGreaterThan(0); + } + }); + + it("references only rubric dimensions that exist", () => { + const ids = new Set(benchmark.rubric.dimensions.map((dimension) => dimension.id)); + for (const entry of benchmark.cases) { + for (const dimensionId of entry.rubric_dimensions) { + expect(ids, `${entry.case_id}:${dimensionId}`).toContain(dimensionId); + } + } + }); + + it("asks for the pairing rubric dimension only where pairing is enabled", () => { + for (const entry of benchmark.cases) { + const asksPairing = entry.rubric_dimensions.includes("pairing_usefulness"); + expect(asksPairing, entry.case_id).toBe(entry.deterministic_expectations.pairing_enabled); + } + }); + + it("documents every known defect with a real reference", () => { + for (const entry of benchmark.cases) { + for (const defect of entry.known_defects) { + expect(defect.reference.length, `${entry.case_id}:${defect.id}`).toBeGreaterThan(0); + expect(defect.summary.length).toBeGreaterThan(20); + } + } + }); +}); + +describe("committed benchmark content contains no real-world data", () => { + const raw = readdirSync(path.join(DATASET, "cases")) + .map((name) => readFileSync(path.join(DATASET, "cases", name), "utf8")) + .join("\n"); + + it("contains no email address, phone number, or URL", () => { + expect(raw).not.toMatch(/[\w.+-]+@[\w-]+\.[\w.]+/); + expect(raw).not.toMatch(/\+?\d[\d\s().-]{8,}\d/); + expect(raw).not.toMatch(/https?:\/\//); + }); + + it("contains no absolute filesystem path", () => { + expect(raw).not.toMatch(/\/(?:Users|home|root)\//); + expect(raw).not.toMatch(/[A-Za-z]:\\\\/); + }); + + it("contains no secret-shaped string", () => { + expect(raw).not.toMatch(/\bsk-[A-Za-z0-9_-]{16,}/); + }); + + it("labels its people and organisations as invented", () => { + // Every case description states its content is fictional; this asserts the + // convention holds rather than trusting it. + for (const name of readdirSync(path.join(DATASET, "cases"))) { + const text = readFileSync(path.join(DATASET, "cases", name), "utf8"); + expect(text, name).toMatch(/fictional|invented|synthetic/i); + } + }); +}); + +describe("benchmark loading failures", () => { + it("throws BenchmarkValidationError for a benchmark that does not exist", async () => { + await expect(loadBenchmark({ benchmarkId: "decision-benchmark-v999" })).rejects.toThrow(); + }); + + it("reports issues as a readable list", () => { + const error = new BenchmarkValidationError("broken", ["one", "two"]); + expect(error.message).toContain("- one"); + expect(error.issues).toEqual(["one", "two"]); + }); +}); + +describe("path handling", () => { + it("converts absolute paths to repository-relative ones", () => { + const relative = toRepoRelative(path.join(process.cwd(), "evals", "datasets")); + expect(relative).toBe("evals/datasets"); + expect(relative.startsWith("/")).toBe(false); + }); +}); diff --git a/evals/datasets/decision-benchmark-v1/cases/case-001.json b/evals/datasets/decision-benchmark-v1/cases/case-001.json new file mode 100644 index 0000000..16b81c1 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-001.json @@ -0,0 +1,115 @@ +{ + "case_id": "case-001", + "schema_version": "1.0.0", + "title": "One clearly dominant candidate and one clearly weak candidate", + "description": "A single-scenario ranking problem in which one fictional candidate is stronger on every criterion and one is weaker on every criterion. The point is that an unambiguous ordering must come out unambiguously ordered.", + "tags": [ + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Director of Regional Logistics", + "description": "Owns three regional distribution centres and the plan to consolidate them into one automated hub. Accountable for service levels during the transition, for the depot workforce, and for the automation vendor relationship." + }, + "scenarios": [ + "Consolidating three regional warehouses into one automated hub within nine months while holding next-day delivery service levels." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "nadia-brookfield", + "name": "Nadia Brookfield", + "description": "Ran a four-depot consolidation at a fictional grocery wholesaler, delivering it two weeks early with no service-level breach. Rebuilt the shift model with the works council before automation went live. Previously ran the same firm's peak-season crisis desk." + }, + { + "id": "owen-kestrel", + "name": "Owen Kestrel", + "description": "Eight years in regional distribution management at an invented parcel operator, with steady but unremarkable results. Has supported one automation rollout as a workstream lead, not as the owner." + }, + { + "id": "priya-tallow", + "name": "Priya Tallow", + "description": "Moved into logistics eighteen months ago from a fictional retail buying team. Has not led a site consolidation, an automation programme, or a workforce transition, and has no direct depot P&L experience." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "nadia-brookfield", + "owen-kestrel", + "priya-tallow" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Consolidating three regional warehouses into one automated hub within nine months while holding next-day delivery service levels." + ], + "allowed_winner_ids": [ + "nadia-brookfield" + ], + "forbidden_winner_ids": [ + "priya-tallow", + "owen-kestrel" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "nadia-brookfield": { + "default": 9, + "confidence": 0.88, + "evidence_quality": "specific" + }, + "owen-kestrel": { + "default": 6, + "confidence": 0.82, + "evidence_quality": "specific" + }, + "priya-tallow": { + "default": 3, + "confidence": 0.8, + "evidence_quality": "specific" + } + } + }, + "known_defects": [ + { + "defect_id": "SR-P3A-001", + "title": "Negative risk-adjusted score violates public response contract", + "case_id": "case-001", + "execution_scope": { "execution_id": "case-001#s0#r1", "scenario_id": "scenario-1", "scenario_index": 0, "variant_id": null, "repetition": 1 }, + "expected_observations": [ + { "grader_id": "contract-validity", "signature": { "kind": "schema_issue", "path_pattern": "candidate_evaluations.*.risk_adjusted_score", "code": "too_small", "minimum": 0, "subject_candidate_id": "priya-tallow" } }, + { "grader_id": "score-integrity", "signature": { "kind": "score_bound_violation", "metric": "risk_adjusted_score", "operator": "lt", "bound": 0, "subject_candidate_id": "priya-tallow" } } + ], + "summary": "computeRiskAdjustedScore can return a negative value for a weak candidate, but the public completed-response contract bounds risk_adjusted_score to 0-100, so server/http/routes.js rejects its own response and returns a generic 500 after the model has already been paid for.", + "reference": "docs/architecture/KNOWN_LIMITATIONS.md (P0.7) and docs/evaluation/BENCHMARK_V1.md" + } + ], + "notes": "Deliberately the easiest case in the benchmark. If this one fails, the failure is structural (coverage, ranking agreement, contract validity) rather than a matter of judgment." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-002.json b/evals/datasets/decision-benchmark-v1/cases/case-002.json new file mode 100644 index 0000000..7510e9f --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-002.json @@ -0,0 +1,111 @@ +{ + "case_id": "case-002", + "schema_version": "1.0.0", + "title": "Balanced close call between three defensible candidates", + "description": "Three fictional candidates with different but comparably strong profiles. No single winner is objectively correct, so every candidate is listed as an allowed winner and the case tests whether the output acknowledges how close the decision is.", + "tags": [ + "basic-ranking", + "close-call" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Head of Customer Operations", + "description": "Leads a 120-person contact and fulfilment operation through a service-model change. Success is measured on resolution time, cost per contact, and staff retention across the change." + }, + "scenarios": [ + "Rolling out a new service model across a 120-person customer operation while protecting resolution times and staff retention." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "harlan-vex", + "name": "Harlan Vex", + "description": "Deep operational background in contact-centre management at a fictional utility, with the strongest technical grounding of the three but the least experience of leading a change programme." + }, + { + "id": "imani-solvay", + "name": "Imani Solvay", + "description": "Led two service-model changes at an invented insurer, strongest on stakeholder handling and change leadership, with less hands-on operational depth than her peers." + }, + { + "id": "jules-mirren", + "name": "Jules Mirren", + "description": "Consistently solid across operations, change, and stakeholder work at a fictional travel business, without a standout strength in any one of them." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "harlan-vex", + "imani-solvay", + "jules-mirren" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Rolling out a new service model across a 120-person customer operation while protecting resolution times and staff retention." + ], + "allowed_winner_ids": [ + "harlan-vex", + "imani-solvay", + "jules-mirren" + ], + "forbidden_winner_ids": [], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "harlan-vex": { + "default": 7, + "criteria": { + "domain_expertise": 8, + "operational_execution": 8, + "transformation_leadership": 6 + }, + "confidence": 0.72, + "evidence_quality": "specific" + }, + "imani-solvay": { + "default": 7, + "criteria": { + "stakeholder_management": 8, + "transformation_leadership": 8, + "operational_execution": 6 + }, + "confidence": 0.72, + "evidence_quality": "specific" + }, + "jules-mirren": { + "default": 7, + "confidence": 0.72, + "evidence_quality": "specific" + } + } + }, + "notes": "This case exists to stop the benchmark rewarding false confidence. Any of the three winning is acceptable; what is not acceptable is presenting the choice as clear-cut. Note a real gap this case exposed: ScenarioRank has no near-tie uncertainty signal at all. The deterministic confidence-and-evidence review keys only on reported confidence and evidence length, so a decision separated by noise is never flagged for human review. That is why this case is not tagged \"uncertainty\" — see docs/evaluation/BENCHMARK_V1.md, \"Known limitations\"." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-003.json b/evals/datasets/decision-benchmark-v1/cases/case-003.json new file mode 100644 index 0000000..1a71b75 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-003.json @@ -0,0 +1,133 @@ +{ + "case_id": "case-003", + "schema_version": "1.0.0", + "title": "Candidates with different but individually valid strengths", + "description": "Four fictional candidates, each genuinely strong in a different area. Two of those areas are central to the stated scenario and two are not, so the case constrains the winner set without pretending a single answer is correct.", + "tags": [ + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Programme Director, Payments Platform", + "description": "Accountable for delivering a replacement payments platform across four business units, including regulator engagement, vendor management, and the migration cutover itself." + }, + "scenarios": [ + "Delivering a regulated payments platform migration across four business units under a fixed regulatory deadline." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "kestrel-adeyemi", + "name": "Kestrel Adeyemi", + "description": "Strongest on regulated delivery and cutover execution, having run two fictional core-banking migrations to fixed regulatory dates." + }, + { + "id": "linnea-vasquez", + "name": "Linnea Vasquez", + "description": "Strongest on regulator and executive stakeholder handling, having led supervisory engagement for an invented payments provider through two remediation programmes." + }, + { + "id": "mikhail-orin", + "name": "Mikhail Orin", + "description": "Strongest on greenfield product innovation and developer-platform design at a fictional fintech, with limited exposure to regulated migration work." + }, + { + "id": "noor-halvorsen", + "name": "Noor Halvorsen", + "description": "Strongest on long-horizon organisational scaling at an invented marketplace business, with no payments or regulated-delivery background." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "kestrel-adeyemi", + "linnea-vasquez", + "mikhail-orin", + "noor-halvorsen" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Delivering a regulated payments platform migration across four business units under a fixed regulatory deadline." + ], + "allowed_winner_ids": [ + "kestrel-adeyemi", + "linnea-vasquez" + ], + "forbidden_winner_ids": [ + "mikhail-orin", + "noor-halvorsen" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "kestrel-adeyemi": { + "default": 6, + "criteria": { + "domain_expertise": 9, + "operational_execution": 9, + "crisis_management": 8 + }, + "confidence": 0.83, + "evidence_quality": "specific" + }, + "linnea-vasquez": { + "default": 6, + "criteria": { + "stakeholder_management": 9, + "transformation_leadership": 8, + "domain_expertise": 8 + }, + "confidence": 0.83, + "evidence_quality": "specific" + }, + "mikhail-orin": { + "default": 5, + "criteria": { + "innovation_digital": 9, + "strategic_scalability": 7, + "domain_expertise": 3 + }, + "confidence": 0.8, + "evidence_quality": "specific" + }, + "noor-halvorsen": { + "default": 5, + "criteria": { + "strategic_scalability": 9, + "transformation_leadership": 6, + "domain_expertise": 3 + }, + "confidence": 0.8, + "evidence_quality": "specific" + } + } + }, + "notes": "The two forbidden winners are genuinely strong people whose strengths are simply not what this scenario needs. A system that ranks raw strength rather than scenario fit will fail this case." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-004.json b/evals/datasets/decision-benchmark-v1/cases/case-004.json new file mode 100644 index 0000000..48b7ab3 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-004.json @@ -0,0 +1,131 @@ +{ + "case_id": "case-004", + "schema_version": "1.0.0", + "title": "Two scenarios that favour different skills", + "description": "The same three fictional candidates evaluated against two scenarios that genuinely reward different criteria. A scenario-sensitive system should not produce the same winner in both.", + "tags": [ + "multi-scenario" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Chief Operating Officer", + "description": "Accountable for both day-to-day service reliability and the medium-term commercial expansion of a fictional mid-sized services group." + }, + "scenarios": [ + "Responding to a sustained service outage that has triggered regulatory attention and requires day-by-day incident command.", + "Opening two new digital sales channels over eighteen months in a stable, low-incident operating environment." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "rowan-delacroix", + "name": "Rowan Delacroix", + "description": "Built and ran the incident command function at a fictional energy retailer, taking two major outages through to regulator sign-off. Hands-on operationally; describes himself as a poor fit for exploratory product work." + }, + { + "id": "sable-quintero", + "name": "Sable Quintero", + "description": "Led the digital reinvention of an invented equipment-rental business, launching three new channels in two years. Has never run a live incident bridge and avoids operational firefighting." + }, + { + "id": "tobias-nunn", + "name": "Tobias Nunn", + "description": "Steady general manager at a fictional distribution group, competent across the board without a pronounced strength or weakness in either crisis response or digital innovation." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "rowan-delacroix", + "sable-quintero", + "tobias-nunn" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Responding to a sustained service outage that has triggered regulatory attention and requires day-by-day incident command.", + "Opening two new digital sales channels over eighteen months in a stable, low-incident operating environment." + ], + "allowed_winner_ids": [ + "rowan-delacroix", + "sable-quintero" + ], + "forbidden_winner_ids": [ + "tobias-nunn" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "rowan-delacroix": { + "default": 6, + "criteria": { + "crisis_management": 9, + "operational_execution": 9, + "domain_expertise": 8, + "innovation_digital": 3, + "strategic_scalability": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "sable-quintero": { + "default": 6, + "criteria": { + "innovation_digital": 9, + "strategic_scalability": 9, + "transformation_leadership": 8, + "crisis_management": 3, + "operational_execution": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "tobias-nunn": { + "default": 6, + "confidence": 0.84, + "evidence_quality": "specific" + } + }, + "scenario_weight_deltas": { + "0": { + "crisis_management": 14, + "operational_execution": 8, + "innovation_digital": -10, + "strategic_scalability": -10 + }, + "1": { + "innovation_digital": 14, + "strategic_scalability": 10, + "crisis_management": -12, + "operational_execution": -8 + } + } + }, + "notes": "Scenario sensitivity is applied through criterion weight deltas, the same mechanism the production scenario-analysis stage uses. Candidate scores are identical in both scenarios, so any change in outcome comes from the scenario alone." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-005.json b/evals/datasets/decision-benchmark-v1/cases/case-005.json new file mode 100644 index 0000000..ff15efb --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-005.json @@ -0,0 +1,154 @@ +{ + "case_id": "case-005", + "schema_version": "1.0.0", + "title": "Three scenarios requiring genuine trade-offs, including a consistently moderate candidate", + "description": "Four fictional candidates across three scenarios, each scenario tuned toward a different specialism. One candidate is deliberately moderate everywhere: useful as a hedge, but never the best answer when a scenario points hard at one specialism.", + "tags": [ + "multi-scenario" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "General Manager, Industrial Services", + "description": "Runs a fictional industrial services division through a period in which the dominant pressure changes from quarter to quarter: delivery throughput, then a workforce dispute, then a platform modernisation." + }, + "scenarios": [ + "Recovering delivery throughput after two quarters of missed installation targets.", + "Rebuilding trust with a workforce and its representatives after a contested restructuring.", + "Modernising a legacy field-service platform while the division keeps operating on it." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "vesper-lange", + "name": "Vesper Lange", + "description": "Throughput and execution specialist at an invented installation contractor; recovered two failing delivery programmes but has limited change-communication experience." + }, + { + "id": "wren-castellan", + "name": "Wren Castellan", + "description": "Broad general manager at a fictional services group, consistently adequate across delivery, people, and technology work without a standout strength anywhere." + }, + { + "id": "xander-poole", + "name": "Xander Poole", + "description": "Workforce-relations specialist who rebuilt representative relationships at a fictional manufacturer after a disputed restructuring; weaker on delivery mechanics." + }, + { + "id": "yara-ilves", + "name": "Yara Ilves", + "description": "Platform modernisation specialist at an invented field-services company; strong on legacy replacement, limited experience of industrial workforce dynamics." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "vesper-lange", + "wren-castellan", + "xander-poole", + "yara-ilves" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Recovering delivery throughput after two quarters of missed installation targets.", + "Rebuilding trust with a workforce and its representatives after a contested restructuring.", + "Modernising a legacy field-service platform while the division keeps operating on it." + ], + "allowed_winner_ids": [ + "vesper-lange", + "xander-poole", + "yara-ilves" + ], + "forbidden_winner_ids": [ + "wren-castellan" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "vesper-lange": { + "default": 5, + "criteria": { + "operational_execution": 9, + "domain_expertise": 8, + "crisis_management": 8, + "stakeholder_management": 4 + }, + "confidence": 0.82, + "evidence_quality": "specific" + }, + "wren-castellan": { + "default": 6, + "confidence": 0.82, + "evidence_quality": "specific" + }, + "xander-poole": { + "default": 5, + "criteria": { + "stakeholder_management": 9, + "transformation_leadership": 8, + "operational_execution": 4 + }, + "confidence": 0.82, + "evidence_quality": "specific" + }, + "yara-ilves": { + "default": 5, + "criteria": { + "innovation_digital": 9, + "strategic_scalability": 8, + "transformation_leadership": 7, + "stakeholder_management": 4 + }, + "confidence": 0.82, + "evidence_quality": "specific" + } + }, + "scenario_weight_deltas": { + "0": { + "operational_execution": 15, + "domain_expertise": 8, + "innovation_digital": -10, + "strategic_scalability": -10 + }, + "1": { + "stakeholder_management": 16, + "transformation_leadership": 8, + "innovation_digital": -10, + "operational_execution": -8 + }, + "2": { + "innovation_digital": 15, + "strategic_scalability": 10, + "crisis_management": -10, + "stakeholder_management": -8 + } + } + }, + "notes": "The forbidden winner is the consistently moderate candidate. Ranking them first in a scenario that is explicitly tuned toward one specialism would indicate the scenario weighting is not doing real work." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-006.json b/evals/datasets/decision-benchmark-v1/cases/case-006.json new file mode 100644 index 0000000..0398eb0 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-006.json @@ -0,0 +1,148 @@ +{ + "case_id": "case-006", + "schema_version": "1.0.0", + "title": "A candidate strong in one scenario and weak in another", + "description": "Three fictional candidates over two scenarios, where one candidate's evidence supports one context strongly and the other barely at all. Scores differ per scenario, mirroring the fact that the production scoring prompt sees the scenario text.", + "tags": [ + "multi-scenario" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Director of Manufacturing Quality", + "description": "Owns quality across two fictional plants: one running a mature, tightly regulated line, and one commissioning an entirely new process." + }, + "scenarios": [ + "Holding quality and audit readiness on a mature, tightly regulated production line.", + "Commissioning an unproven new production process with no established quality baseline." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "aurelio-fenn", + "name": "Aurelio Fenn", + "description": "Twelve years of regulated quality management on mature lines at a fictional medical-device maker, with an unbroken audit record. Has never commissioned a new process." + }, + { + "id": "brielle-oyelaran", + "name": "Brielle Oyelaran", + "description": "Commissioned two new production processes at an invented speciality-chemicals firm, comfortable establishing a baseline from nothing; less experience of long-run regulated stability." + }, + { + "id": "caspian-drew", + "name": "Caspian Drew", + "description": "Quality engineer moving into leadership at a fictional components supplier; limited depth in either mature-line stewardship or new-process commissioning." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "aurelio-fenn", + "brielle-oyelaran", + "caspian-drew" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Holding quality and audit readiness on a mature, tightly regulated production line.", + "Commissioning an unproven new production process with no established quality baseline." + ], + "allowed_winner_ids": [ + "aurelio-fenn", + "brielle-oyelaran" + ], + "forbidden_winner_ids": [ + "caspian-drew" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "aurelio-fenn": { + "default": 8, + "criteria": { + "domain_expertise": 9, + "operational_execution": 9, + "innovation_digital": 4 + }, + "confidence": 0.85, + "evidence_quality": "specific" + }, + "brielle-oyelaran": { + "default": 6, + "criteria": { + "innovation_digital": 8, + "transformation_leadership": 7 + }, + "confidence": 0.83, + "evidence_quality": "specific" + }, + "caspian-drew": { + "default": 5, + "confidence": 0.8, + "evidence_quality": "specific" + } + }, + "scenario_overrides": { + "1": { + "aurelio-fenn": { + "default": 4, + "criteria": { + "domain_expertise": 4, + "operational_execution": 5, + "innovation_digital": 3 + }, + "confidence": 0.7 + }, + "brielle-oyelaran": { + "default": 8, + "criteria": { + "innovation_digital": 9, + "transformation_leadership": 8, + "domain_expertise": 8 + }, + "confidence": 0.85 + } + } + } + }, + "known_defects": [ + { + "defect_id": "SR-P3A-001", + "title": "Negative risk-adjusted score violates public response contract", + "case_id": "case-006", + "execution_scope": { "execution_id": "case-006#s1#r1", "scenario_id": "scenario-2", "scenario_index": 1, "variant_id": null, "repetition": 1 }, + "expected_observations": [ + { "grader_id": "contract-validity", "signature": { "kind": "schema_issue", "path_pattern": "candidate_evaluations.*.risk_adjusted_score", "code": "too_small", "minimum": 0, "subject_candidate_id": "aurelio-fenn" } }, + { "grader_id": "score-integrity", "signature": { "kind": "score_bound_violation", "metric": "risk_adjusted_score", "operator": "lt", "bound": 0, "subject_candidate_id": "aurelio-fenn" } } + ], + "summary": "computeRiskAdjustedScore can return a negative value for a weak candidate, but the public completed-response contract bounds risk_adjusted_score to 0-100, so server/http/routes.js rejects its own response and returns a generic 500 after the model has already been paid for.", + "reference": "docs/architecture/KNOWN_LIMITATIONS.md (P0.7) and docs/evaluation/BENCHMARK_V1.md" + } + ], + "notes": "Unlike case-004, scenario sensitivity here comes from the candidate scores themselves rather than the weights, so the two mechanisms are covered separately." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-007.json b/evals/datasets/decision-benchmark-v1/cases/case-007.json new file mode 100644 index 0000000..7e254b4 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-007.json @@ -0,0 +1,112 @@ +{ + "case_id": "case-007", + "schema_version": "1.0.0", + "title": "Strong, specific, verifiable evidence for every candidate", + "description": "Every fictional candidate description carries concrete, dated, measurable claims. This is the benchmark's control case for evidence grounding: there is no excuse for an unsupported claim here.", + "tags": [ + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Plant Operations Manager", + "description": "Runs a single fictional production site end to end, accountable for uptime, safety, and the site's ability to absorb unplanned disruption without missing customer commitments." + }, + "scenarios": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "dagny-holloway", + "name": "Dagny Holloway", + "description": "Cut unplanned downtime at a fictional bottling plant from 9.4% to 3.1% over fourteen months by rebuilding the preventive-maintenance schedule and retraining two shift teams. Ran the plant through a four-day flood recovery in 2021 with no missed customer order." + }, + { + "id": "esteban-rourke", + "name": "Esteban Rourke", + "description": "Managed a fictional packaging line for six years, holding output within 2% of plan every quarter, and led the site's move to a new maintenance planning system with documented before-and-after metrics." + }, + { + "id": "farrah-lindgren", + "name": "Farrah Lindgren", + "description": "Ran maintenance planning at an invented beverage site for three years, delivering a 12% reduction in emergency call-outs, with no experience of leading a site through a major disruption." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "dagny-holloway", + "esteban-rourke", + "farrah-lindgren" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "allowed_winner_ids": [ + "dagny-holloway" + ], + "forbidden_winner_ids": [ + "farrah-lindgren" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "dagny-holloway": { + "default": 8, + "criteria": { + "operational_execution": 9, + "crisis_management": 9, + "domain_expertise": 9 + }, + "confidence": 0.9, + "evidence_quality": "specific" + }, + "esteban-rourke": { + "default": 7, + "criteria": { + "operational_execution": 8, + "domain_expertise": 7 + }, + "confidence": 0.88, + "evidence_quality": "specific" + }, + "farrah-lindgren": { + "default": 6, + "criteria": { + "crisis_management": 4 + }, + "confidence": 0.86, + "evidence_quality": "specific" + } + } + }, + "notes": "Serves as the original for the two wording/irrelevant-text variants, case-013 and case-014." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-008.json b/evals/datasets/decision-benchmark-v1/cases/case-008.json new file mode 100644 index 0000000..3614dbd --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-008.json @@ -0,0 +1,115 @@ +{ + "case_id": "case-008", + "schema_version": "1.0.0", + "title": "Vague, unsupported claims with no verifiable detail", + "description": "Every fictional candidate description is confident but empty: no dates, no numbers, no named outcomes. The deterministic confidence-and-evidence review must flag this, and the explanation must not manufacture specificity that the inputs never contained.", + "tags": [ + "missing-evidence", + "uncertainty" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Director of Commercial Strategy", + "description": "Sets commercial direction for a fictional B2B services group, with accountability for pricing, portfolio mix, and the group's largest client relationships." + }, + "scenarios": [ + "Rebuilding a commercial strategy after two years of margin erosion across the portfolio." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "galen-morrow", + "name": "Galen Morrow", + "description": "A results-driven commercial leader with a track record of transformational impact and a passion for excellence. Known for strategic thinking and driving change at pace." + }, + { + "id": "hester-quill", + "name": "Hester Quill", + "description": "A visionary strategist who consistently exceeds expectations and brings world-class commercial judgement to complex situations." + }, + { + "id": "ivo-strand", + "name": "Ivo Strand", + "description": "An accomplished senior leader with deep experience and a strong reputation for delivering outstanding commercial outcomes." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "galen-morrow", + "hester-quill", + "ivo-strand" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Rebuilding a commercial strategy after two years of margin erosion across the portfolio." + ], + "allowed_winner_ids": null, + "forbidden_winner_ids": [], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [ + "galen-morrow", + "hester-quill", + "ivo-strand" + ] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "galen-morrow": { + "default": 6, + "confidence": 0.5, + "evidence_quality": "vague" + }, + "hester-quill": { + "default": 6, + "confidence": 0.48, + "evidence_quality": "vague" + }, + "ivo-strand": { + "default": 5, + "confidence": 0.48, + "evidence_quality": "vague" + } + } + }, + "known_defects": [ + { + "defect_id": "SR-P3A-001", + "title": "Negative risk-adjusted score violates public response contract", + "case_id": "case-008", + "execution_scope": { "execution_id": "case-008#s0#r1", "scenario_id": "scenario-1", "scenario_index": 0, "variant_id": null, "repetition": 1 }, + "expected_observations": [ + { "grader_id": "contract-validity", "signature": { "kind": "schema_issue", "path_pattern": "candidate_evaluations.*.risk_adjusted_score", "code": "too_small", "minimum": 0, "subject_candidate_id": "ivo-strand" } }, + { "grader_id": "score-integrity", "signature": { "kind": "score_bound_violation", "metric": "risk_adjusted_score", "operator": "lt", "bound": 0, "subject_candidate_id": "ivo-strand" } } + ], + "summary": "computeRiskAdjustedScore can return a negative value for a weak candidate, but the public completed-response contract bounds risk_adjusted_score to 0-100, so server/http/routes.js rejects its own response and returns a generic 500 after the model has already been paid for.", + "reference": "docs/architecture/KNOWN_LIMITATIONS.md (P0.7) and docs/evaluation/BENCHMARK_V1.md" + } + ], + "notes": "No winner claim is made at all: with evidence this thin, asserting that one of these candidates should win would be exactly the overclaiming this benchmark is meant to detect. What is checked is that every candidate is flagged for human review." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-009.json b/evals/datasets/decision-benchmark-v1/cases/case-009.json new file mode 100644 index 0000000..5e8839e --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-009.json @@ -0,0 +1,101 @@ +{ + "case_id": "case-009", + "schema_version": "1.0.0", + "title": "Internally conflicting evidence within candidate descriptions", + "description": "Each fictional description contains claims that contradict each other. The output should surface the contradiction rather than silently picking whichever half supports a confident conclusion.", + "tags": [ + "conflicting-evidence", + "uncertainty" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Head of Engineering", + "description": "Leads a fictional 60-person engineering function through a reliability programme, accountable for delivery pace, system stability, and engineer retention at the same time." + }, + "scenarios": [ + "Improving system reliability without slowing delivery, in a team that has lost a third of its engineers in a year." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "jonas-albrecht", + "name": "Jonas Albrecht", + "description": "Described as having rebuilt a platform team's reliability practice, and elsewhere in the same profile as having had reliability owned entirely by a separate SRE group he did not manage. Retention is reported both as a personal strength and as the reason his previous team was restructured." + }, + { + "id": "kalinda-obi", + "name": "Kalinda Obi", + "description": "Credited with shipping a major replatform on schedule, while the same profile states the replatform was descoped twice and finished a quarter late. Reported as both hands-on technically and as having left engineering practice five years ago." + }, + { + "id": "lorcan-pryce", + "name": "Lorcan Pryce", + "description": "Described as a calm, stabilising presence, and separately as having been brought in twice specifically to force pace against team objections. No dates or measurable outcomes are given for either characterisation." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "jonas-albrecht", + "kalinda-obi", + "lorcan-pryce" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Improving system reliability without slowing delivery, in a team that has lost a third of its engineers in a year." + ], + "allowed_winner_ids": null, + "forbidden_winner_ids": [], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [ + "jonas-albrecht", + "kalinda-obi", + "lorcan-pryce" + ] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "jonas-albrecht": { + "default": 6, + "confidence": 0.55, + "evidence_quality": "conflicting" + }, + "kalinda-obi": { + "default": 6, + "confidence": 0.54, + "evidence_quality": "conflicting" + }, + "lorcan-pryce": { + "default": 5, + "confidence": 0.52, + "evidence_quality": "conflicting" + } + } + }, + "notes": "Conflicting evidence differs from missing evidence: the evidence strings here are long and detailed, so the weak-evidence length heuristic will not fire. Low reported confidence is what should drive the human-review recommendation." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-010.json b/evals/datasets/decision-benchmark-v1/cases/case-010.json new file mode 100644 index 0000000..170410a --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-010.json @@ -0,0 +1,111 @@ +{ + "case_id": "case-010", + "schema_version": "1.0.0", + "title": "Decision-critical evidence missing for the leading candidate", + "description": "Two fictional candidates are close on everything that is documented, but the leading one has nothing at all on the criteria the scenario cares about most. The right behaviour is to name the gap, not to fill it.", + "tags": [ + "missing-evidence", + "uncertainty" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Regional Safety Director", + "description": "Accountable for safety performance across eleven fictional sites, including incident investigation, regulator relationships, and the safety culture programme." + }, + "scenarios": [ + "Turning around safety performance across eleven sites following a serious incident and a regulator improvement notice." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "marisol-vantree", + "name": "Marisol Vantree", + "description": "Strong general leadership record at a fictional infrastructure group. The profile says nothing about incident investigation, regulator engagement, or any prior safety accountability." + }, + { + "id": "nikolai-ashgrove", + "name": "Nikolai Ashgrove", + "description": "Led a safety turnaround across seven sites at an invented civil-engineering firm following an improvement notice, with documented incident-rate reduction and regulator sign-off." + }, + { + "id": "orla-benedetti", + "name": "Orla Benedetti", + "description": "Site safety manager at a fictional logistics operator with solid single-site results and no multi-site or regulator-facing experience." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "marisol-vantree", + "nikolai-ashgrove", + "orla-benedetti" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Turning around safety performance across eleven sites following a serious incident and a regulator improvement notice." + ], + "allowed_winner_ids": [ + "nikolai-ashgrove" + ], + "forbidden_winner_ids": [ + "orla-benedetti" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [ + "marisol-vantree" + ] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "marisol-vantree": { + "default": 7, + "criteria": { + "crisis_management": 6, + "domain_expertise": 5 + }, + "confidence": 0.5, + "evidence_quality": "missing" + }, + "nikolai-ashgrove": { + "default": 7, + "criteria": { + "crisis_management": 9, + "domain_expertise": 8 + }, + "confidence": 0.87, + "evidence_quality": "specific" + }, + "orla-benedetti": { + "default": 6, + "confidence": 0.84, + "evidence_quality": "specific" + } + } + }, + "notes": "The candidate with the evidence gap must be flagged for human review even though their headline scores are competitive. This is the case that most directly tests whether missing evidence is treated as missing rather than neutral." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-011.json b/evals/datasets/decision-benchmark-v1/cases/case-011.json new file mode 100644 index 0000000..413c02c --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-011.json @@ -0,0 +1,116 @@ +{ + "case_id": "case-011", + "schema_version": "1.0.0", + "title": "Candidate-order permutation of case-001", + "description": "Byte-identical to case-001 except that the candidate list is submitted in reverse order. The winner, the ranking, and the structured evidence should all be unchanged; if they are not, submission order is influencing the result.", + "tags": [ + "permutation", + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Director of Regional Logistics", + "description": "Owns three regional distribution centres and the plan to consolidate them into one automated hub. Accountable for service levels during the transition, for the depot workforce, and for the automation vendor relationship." + }, + "scenarios": [ + "Consolidating three regional warehouses into one automated hub within nine months while holding next-day delivery service levels." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "priya-tallow", + "name": "Priya Tallow", + "description": "Moved into logistics eighteen months ago from a fictional retail buying team. Has not led a site consolidation, an automation programme, or a workforce transition, and has no direct depot P&L experience." + }, + { + "id": "owen-kestrel", + "name": "Owen Kestrel", + "description": "Eight years in regional distribution management at an invented parcel operator, with steady but unremarkable results. Has supported one automation rollout as a workstream lead, not as the owner." + }, + { + "id": "nadia-brookfield", + "name": "Nadia Brookfield", + "description": "Ran a four-depot consolidation at a fictional grocery wholesaler, delivering it two weeks early with no service-level breach. Rebuilt the shift model with the works council before automation went live. Previously ran the same firm's peak-season crisis desk." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "priya-tallow", + "owen-kestrel", + "nadia-brookfield" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Consolidating three regional warehouses into one automated hub within nine months while holding next-day delivery service levels." + ], + "allowed_winner_ids": [ + "nadia-brookfield" + ], + "forbidden_winner_ids": [ + "priya-tallow", + "owen-kestrel" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": "case-001", + "variant_kind": "candidate-order", + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "priya-tallow": { + "default": 3, + "confidence": 0.8, + "evidence_quality": "specific" + }, + "owen-kestrel": { + "default": 6, + "confidence": 0.82, + "evidence_quality": "specific" + }, + "nadia-brookfield": { + "default": 9, + "confidence": 0.88, + "evidence_quality": "specific" + } + } + }, + "known_defects": [ + { + "defect_id": "SR-P3A-001", + "title": "Negative risk-adjusted score violates public response contract", + "case_id": "case-011", + "execution_scope": { "execution_id": "case-011#s0#r1", "scenario_id": "scenario-1", "scenario_index": 0, "variant_id": "candidate-order", "repetition": 1 }, + "expected_observations": [ + { "grader_id": "contract-validity", "signature": { "kind": "schema_issue", "path_pattern": "candidate_evaluations.*.risk_adjusted_score", "code": "too_small", "minimum": 0, "subject_candidate_id": "priya-tallow" } }, + { "grader_id": "score-integrity", "signature": { "kind": "score_bound_violation", "metric": "risk_adjusted_score", "operator": "lt", "bound": 0, "subject_candidate_id": "priya-tallow" } } + ], + "summary": "computeRiskAdjustedScore can return a negative value for a weak candidate, but the public completed-response contract bounds risk_adjusted_score to 0-100, so server/http/routes.js rejects its own response and returns a generic 500 after the model has already been paid for.", + "reference": "docs/architecture/KNOWN_LIMITATIONS.md (P0.7) and docs/evaluation/BENCHMARK_V1.md" + } + ], + "notes": "Scores are keyed by candidate ID, never by position, so the offline provider returns identical scores for both cases. Any divergence between case-001 and case-011 therefore comes from the pipeline, not the fixture." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-012.json b/evals/datasets/decision-benchmark-v1/cases/case-012.json new file mode 100644 index 0000000..74403c4 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-012.json @@ -0,0 +1,132 @@ +{ + "case_id": "case-012", + "schema_version": "1.0.0", + "title": "Scenario-order permutation of case-004", + "description": "The same two scenarios as case-004, submitted in the opposite order with their weight deltas moved with them. Each scenario's own result should be identical to its result in case-004.", + "tags": [ + "permutation", + "multi-scenario" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Chief Operating Officer", + "description": "Accountable for both day-to-day service reliability and the medium-term commercial expansion of a fictional mid-sized services group." + }, + "scenarios": [ + "Opening two new digital sales channels over eighteen months in a stable, low-incident operating environment.", + "Responding to a sustained service outage that has triggered regulatory attention and requires day-by-day incident command." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "rowan-delacroix", + "name": "Rowan Delacroix", + "description": "Built and ran the incident command function at a fictional energy retailer, taking two major outages through to regulator sign-off. Hands-on operationally; describes himself as a poor fit for exploratory product work." + }, + { + "id": "sable-quintero", + "name": "Sable Quintero", + "description": "Led the digital reinvention of an invented equipment-rental business, launching three new channels in two years. Has never run a live incident bridge and avoids operational firefighting." + }, + { + "id": "tobias-nunn", + "name": "Tobias Nunn", + "description": "Steady general manager at a fictional distribution group, competent across the board without a pronounced strength or weakness in either crisis response or digital innovation." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "rowan-delacroix", + "sable-quintero", + "tobias-nunn" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Opening two new digital sales channels over eighteen months in a stable, low-incident operating environment.", + "Responding to a sustained service outage that has triggered regulatory attention and requires day-by-day incident command." + ], + "allowed_winner_ids": [ + "rowan-delacroix", + "sable-quintero" + ], + "forbidden_winner_ids": [ + "tobias-nunn" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": "case-004", + "variant_kind": "scenario-order", + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "rowan-delacroix": { + "default": 6, + "criteria": { + "crisis_management": 9, + "operational_execution": 9, + "domain_expertise": 8, + "innovation_digital": 3, + "strategic_scalability": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "sable-quintero": { + "default": 6, + "criteria": { + "innovation_digital": 9, + "strategic_scalability": 9, + "transformation_leadership": 8, + "crisis_management": 3, + "operational_execution": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "tobias-nunn": { + "default": 6, + "confidence": 0.84, + "evidence_quality": "specific" + } + }, + "scenario_weight_deltas": { + "0": { + "innovation_digital": 14, + "strategic_scalability": 10, + "crisis_management": -12, + "operational_execution": -8 + }, + "1": { + "crisis_management": 14, + "operational_execution": 8, + "innovation_digital": -10, + "strategic_scalability": -10 + } + } + }, + "notes": "Because each scenario is executed as its own pipeline run, scenario order should have no effect whatsoever. This case makes that assumption checkable rather than assumed." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-013.json b/evals/datasets/decision-benchmark-v1/cases/case-013.json new file mode 100644 index 0000000..9149500 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-013.json @@ -0,0 +1,113 @@ +{ + "case_id": "case-013", + "schema_version": "1.0.0", + "title": "Semantically equivalent wording variant of case-007", + "description": "Each fictional candidate description from case-007 is rewritten to say the same thing in different words, with the same numbers, dates, and outcomes. The result should be unchanged.", + "tags": [ + "permutation", + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Plant Operations Manager", + "description": "Runs a single fictional production site end to end, accountable for uptime, safety, and the site's ability to absorb unplanned disruption without missing customer commitments." + }, + "scenarios": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "dagny-holloway", + "name": "Dagny Holloway", + "description": "Over fourteen months, reduced unplanned downtime at a fictional bottling plant from 9.4% to 3.1%, achieved by reworking the preventive-maintenance schedule and putting two shift teams through retraining. In 2021 she led the plant through four days of flood recovery without a single customer order being missed." + }, + { + "id": "esteban-rourke", + "name": "Esteban Rourke", + "description": "Spent six years running a fictional packaging line, keeping quarterly output inside 2% of plan throughout, and headed the site's transition to a replacement maintenance planning system, with the before-and-after metrics documented." + }, + { + "id": "farrah-lindgren", + "name": "Farrah Lindgren", + "description": "Spent three years leading maintenance planning at an invented beverage site, where emergency call-outs fell by 12%. Has not led a site through a significant disruption." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "dagny-holloway", + "esteban-rourke", + "farrah-lindgren" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "allowed_winner_ids": [ + "dagny-holloway" + ], + "forbidden_winner_ids": [ + "farrah-lindgren" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": "case-007", + "variant_kind": "equivalent-wording", + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "dagny-holloway": { + "default": 8, + "criteria": { + "operational_execution": 9, + "crisis_management": 9, + "domain_expertise": 9 + }, + "confidence": 0.9, + "evidence_quality": "specific" + }, + "esteban-rourke": { + "default": 7, + "criteria": { + "operational_execution": 8, + "domain_expertise": 7 + }, + "confidence": 0.88, + "evidence_quality": "specific" + }, + "farrah-lindgren": { + "default": 6, + "criteria": { + "crisis_management": 4 + }, + "confidence": 0.86, + "evidence_quality": "specific" + } + } + }, + "notes": "Only the candidate descriptions are reworded. The role and scenario text are byte-identical to case-007, so the reworded descriptions are the single changed variable. Under the offline fake provider this case is guaranteed to match case-007, because the fixture scores by candidate ID and ignores description text. Its real value is against a live model, where wording genuinely can move scores — see the limitations section of docs/evaluation/BENCHMARK_V1.md." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-014.json b/evals/datasets/decision-benchmark-v1/cases/case-014.json new file mode 100644 index 0000000..d8250e2 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-014.json @@ -0,0 +1,113 @@ +{ + "case_id": "case-014", + "schema_version": "1.0.0", + "title": "Irrelevant-sentence variant of case-007", + "description": "Identical to case-007 except that one decision-irrelevant sentence is appended to each fictional candidate description. Nothing decision-relevant has changed, so nothing about the result should change either.", + "tags": [ + "permutation", + "basic-ranking" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Plant Operations Manager", + "description": "Runs a single fictional production site end to end, accountable for uptime, safety, and the site's ability to absorb unplanned disruption without missing customer commitments." + }, + "scenarios": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "dagny-holloway", + "name": "Dagny Holloway", + "description": "Cut unplanned downtime at a fictional bottling plant from 9.4% to 3.1% over fourteen months by rebuilding the preventive-maintenance schedule and retraining two shift teams. Ran the plant through a four-day flood recovery in 2021 with no missed customer order. She commutes by bicycle and organises the site's annual charity quiz." + }, + { + "id": "esteban-rourke", + "name": "Esteban Rourke", + "description": "Managed a fictional packaging line for six years, holding output within 2% of plan every quarter, and led the site's move to a new maintenance planning system with documented before-and-after metrics. He keeps bees at home and once appeared briefly in a regional radio interview about packaging." + }, + { + "id": "farrah-lindgren", + "name": "Farrah Lindgren", + "description": "Ran maintenance planning at an invented beverage site for three years, delivering a 12% reduction in emergency call-outs, with no experience of leading a site through a major disruption. She plays in a local five-a-side football league on Thursdays." + } + ], + "options": { + "enable_pair_simulation": false + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "dagny-holloway", + "esteban-rourke", + "farrah-lindgren" + ], + "pairing_enabled": false, + "expected_pair_count": null, + "expected_best_pair_ids": null, + "required_stage_count": 3, + "required_scenario_coverage": [ + "Restoring plant uptime and disruption resilience after a year of unplanned downtime and one major site incident." + ], + "allowed_winner_ids": [ + "dagny-holloway" + ], + "forbidden_winner_ids": [ + "farrah-lindgren" + ], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 8, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance" + ], + "variant_of": "case-007", + "variant_kind": "irrelevant-text", + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "dagny-holloway": { + "default": 8, + "criteria": { + "operational_execution": 9, + "crisis_management": 9, + "domain_expertise": 9 + }, + "confidence": 0.9, + "evidence_quality": "specific" + }, + "esteban-rourke": { + "default": 7, + "criteria": { + "operational_execution": 8, + "domain_expertise": 7 + }, + "confidence": 0.88, + "evidence_quality": "specific" + }, + "farrah-lindgren": { + "default": 6, + "criteria": { + "crisis_management": 4 + }, + "confidence": 0.86, + "evidence_quality": "specific" + } + } + }, + "notes": "The injected sentences are deliberately harmless and non-demographic. This case checks robustness to irrelevant content, not fairness — the benchmark makes no demographic claims of any kind." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-015.json b/evals/datasets/decision-benchmark-v1/cases/case-015.json new file mode 100644 index 0000000..d3fb4de --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-015.json @@ -0,0 +1,184 @@ +{ + "case_id": "case-015", + "schema_version": "1.0.0", + "title": "Pairing enabled with duplicate display names and one clearly complementary pair", + "description": "Four fictional candidates, two of whom share a display name while having distinct IDs, with pairing enabled. One pair is clearly the strongest combination. Every expected pair must be evaluated exactly once and the duplicate names must remain unambiguous throughout.", + "tags": [ + "duplicate-name", + "pairing" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Head of Clinical Operations", + "description": "Leads clinical delivery across a fictional network of outpatient sites, working as one half of a two-person leadership pairing with a clinical governance counterpart." + }, + "scenarios": [ + "Standing up a two-person operational leadership pairing to run an outpatient network through a capacity expansion." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "alex-moreau-a", + "name": "Alex Moreau", + "description": "Operational delivery lead at a fictional outpatient network, strongest on capacity planning and throughput; comfortable being the operational half of a leadership pairing." + }, + { + "id": "alex-moreau-b", + "name": "Alex Moreau", + "description": "A different fictional candidate who happens to share a display name: a clinical governance specialist with an audit and standards background, weaker on capacity mechanics." + }, + { + "id": "dara-lindqvist", + "name": "Dara Lindqvist", + "description": "Clinical quality lead at an invented care group, strong on governance and staff engagement, with limited capacity-planning experience." + }, + { + "id": "esme-tanaka", + "name": "Esme Tanaka", + "description": "Operations manager at a fictional diagnostics provider, broadly similar in profile to the first Alex Moreau, with overlapping rather than complementary strengths." + } + ], + "options": { + "enable_pair_simulation": true + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "alex-moreau-a", + "alex-moreau-b", + "dara-lindqvist", + "esme-tanaka" + ], + "pairing_enabled": true, + "expected_pair_count": 6, + "expected_best_pair_ids": [ + "alex-moreau-a", + "dara-lindqvist" + ], + "required_stage_count": 4, + "required_scenario_coverage": [ + "Standing up a two-person operational leadership pairing to run an outpatient network through a capacity expansion." + ], + "allowed_winner_ids": [ + "alex-moreau-a" + ], + "forbidden_winner_ids": [], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 10, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance", + "pairing_usefulness" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "alex-moreau-a": { + "default": 7, + "criteria": { + "operational_execution": 9, + "domain_expertise": 8, + "stakeholder_management": 5 + }, + "confidence": 0.85, + "evidence_quality": "specific" + }, + "alex-moreau-b": { + "default": 6, + "criteria": { + "stakeholder_management": 8, + "domain_expertise": 7, + "operational_execution": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "dara-lindqvist": { + "default": 6, + "criteria": { + "stakeholder_management": 9, + "transformation_leadership": 7, + "operational_execution": 4 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "esme-tanaka": { + "default": 6, + "criteria": { + "operational_execution": 8, + "domain_expertise": 7, + "stakeholder_management": 5 + }, + "confidence": 0.83, + "evidence_quality": "specific" + } + }, + "pair_overrides": { + "alex-moreau-a::dara-lindqvist": { + "scenario_coverage": 0.93, + "complementarity": 0.92, + "overlap_risk": 0.12, + "conflict_risk": 0.08, + "execution_cohesion": 0.9, + "pair_adaptability": 0.85 + }, + "alex-moreau-a::esme-tanaka": { + "scenario_coverage": 0.62, + "complementarity": 0.3, + "overlap_risk": 0.85, + "conflict_risk": 0.3, + "execution_cohesion": 0.6, + "pair_adaptability": 0.45 + }, + "alex-moreau-a::alex-moreau-b": { + "scenario_coverage": 0.78, + "complementarity": 0.72, + "overlap_risk": 0.3, + "conflict_risk": 0.2, + "execution_cohesion": 0.72, + "pair_adaptability": 0.66 + }, + "alex-moreau-b::dara-lindqvist": { + "scenario_coverage": 0.55, + "complementarity": 0.28, + "overlap_risk": 0.88, + "conflict_risk": 0.32, + "execution_cohesion": 0.55, + "pair_adaptability": 0.42 + }, + "alex-moreau-b::esme-tanaka": { + "scenario_coverage": 0.7, + "complementarity": 0.62, + "overlap_risk": 0.35, + "conflict_risk": 0.25, + "execution_cohesion": 0.66, + "pair_adaptability": 0.6 + }, + "dara-lindqvist::esme-tanaka": { + "scenario_coverage": 0.72, + "complementarity": 0.68, + "overlap_risk": 0.3, + "conflict_risk": 0.22, + "execution_cohesion": 0.68, + "pair_adaptability": 0.62 + } + } + }, + "notes": "The two candidates sharing the display name 'Alex Moreau' are the point of the case: every pair identity, pair display name, and ranking entry must stay resolvable by ID." +} diff --git a/evals/datasets/decision-benchmark-v1/cases/case-016.json b/evals/datasets/decision-benchmark-v1/cases/case-016.json new file mode 100644 index 0000000..dc9b398 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/cases/case-016.json @@ -0,0 +1,181 @@ +{ + "case_id": "case-016", + "schema_version": "1.0.0", + "title": "Pairing where the two strongest individuals are not the best pair", + "description": "Four fictional candidates with pairing enabled. The top two individually overlap heavily and clash; the best combination pairs the strongest candidate with a lower-ranked complement. A system that simply pairs the top two will get this wrong.", + "tags": [ + "pairing" + ], + "synthetic": true, + "data_policy": "synthetic-only", + "input": { + "role": { + "title": "Co-Director, Field Operations", + "description": "One half of a fictional co-director pairing running field operations across four territories, where the two directors must divide territory coverage and escalation duty between them." + }, + "scenarios": [ + "Forming a co-director pairing to run field operations across four territories with shared escalation duty." + ], + "decision_mode": "best_fit", + "candidates": [ + { + "id": "finnegan-adler", + "name": "Finnegan Adler", + "description": "Strongest individual profile: an invented field-operations director with deep execution and crisis credentials across three territories." + }, + { + "id": "giselle-varga", + "name": "Giselle Varga", + "description": "Second strongest individually, with a profile that closely mirrors Finnegan Adler's — same strengths, same blind spots, and a documented history of friction with similarly-profiled peers." + }, + { + "id": "hollis-nakamura", + "name": "Hollis Nakamura", + "description": "Ranked lower individually, but strong exactly where the two leaders are weak: stakeholder handling, territory relationships, and the long-horizon coverage model." + }, + { + "id": "isolde-marchetti", + "name": "Isolde Marchetti", + "description": "A capable generalist at a fictional services operator, neither strongly complementary to nor strongly overlapping with the others." + } + ], + "options": { + "enable_pair_simulation": true + } + }, + "deterministic_expectations": { + "expected_candidate_ids": [ + "finnegan-adler", + "giselle-varga", + "hollis-nakamura", + "isolde-marchetti" + ], + "pairing_enabled": true, + "expected_pair_count": 6, + "expected_best_pair_ids": [ + "finnegan-adler", + "hollis-nakamura" + ], + "required_stage_count": 4, + "required_scenario_coverage": [ + "Forming a co-director pairing to run field operations across four territories with shared escalation duty." + ], + "allowed_winner_ids": [ + "finnegan-adler" + ], + "forbidden_winner_ids": [], + "required_not_measured_fields": [ + "outcome_models[].cross_scenario_consistency", + "adaptability_profiles[].best_scenario", + "adaptability_profiles[].worst_scenario" + ], + "maximum_provider_attempts": 10, + "expect_human_review_for_candidate_ids": [] + }, + "rubric_dimensions": [ + "evidence_grounding", + "scenario_relevance", + "tradeoff_clarity", + "clarity", + "uncertainty_handling", + "recommendation_consistency", + "unsupported_claim_avoidance", + "pairing_usefulness" + ], + "variant_of": null, + "variant_kind": null, + "fake_provider_plan": { + "profile": "valid-standard", + "candidate_scores": { + "finnegan-adler": { + "default": 8, + "criteria": { + "operational_execution": 9, + "crisis_management": 9, + "domain_expertise": 9, + "stakeholder_management": 5 + }, + "confidence": 0.86, + "evidence_quality": "specific" + }, + "giselle-varga": { + "default": 7, + "criteria": { + "operational_execution": 9, + "crisis_management": 8, + "domain_expertise": 8, + "stakeholder_management": 4 + }, + "confidence": 0.85, + "evidence_quality": "specific" + }, + "hollis-nakamura": { + "default": 6, + "criteria": { + "stakeholder_management": 9, + "transformation_leadership": 8, + "strategic_scalability": 8, + "operational_execution": 5 + }, + "confidence": 0.84, + "evidence_quality": "specific" + }, + "isolde-marchetti": { + "default": 6, + "confidence": 0.83, + "evidence_quality": "specific" + } + }, + "pair_overrides": { + "finnegan-adler::giselle-varga": { + "scenario_coverage": 0.6, + "complementarity": 0.18, + "overlap_risk": 0.95, + "conflict_risk": 0.75, + "execution_cohesion": 0.45, + "pair_adaptability": 0.35 + }, + "finnegan-adler::hollis-nakamura": { + "scenario_coverage": 0.94, + "complementarity": 0.93, + "overlap_risk": 0.1, + "conflict_risk": 0.07, + "execution_cohesion": 0.91, + "pair_adaptability": 0.88 + }, + "finnegan-adler::isolde-marchetti": { + "scenario_coverage": 0.74, + "complementarity": 0.6, + "overlap_risk": 0.4, + "conflict_risk": 0.22, + "execution_cohesion": 0.7, + "pair_adaptability": 0.6 + }, + "giselle-varga::hollis-nakamura": { + "scenario_coverage": 0.82, + "complementarity": 0.8, + "overlap_risk": 0.18, + "conflict_risk": 0.14, + "execution_cohesion": 0.78, + "pair_adaptability": 0.74 + }, + "giselle-varga::isolde-marchetti": { + "scenario_coverage": 0.7, + "complementarity": 0.55, + "overlap_risk": 0.42, + "conflict_risk": 0.25, + "execution_cohesion": 0.66, + "pair_adaptability": 0.58 + }, + "hollis-nakamura::isolde-marchetti": { + "scenario_coverage": 0.66, + "complementarity": 0.5, + "overlap_risk": 0.45, + "conflict_risk": 0.3, + "execution_cohesion": 0.6, + "pair_adaptability": 0.55 + } + } + }, + "notes": "The individually-best pair is deliberately the worst combination. This case is the reason pair scoring is deterministic and separate from individual ranking." +} diff --git a/evals/datasets/decision-benchmark-v1/manifest.json b/evals/datasets/decision-benchmark-v1/manifest.json new file mode 100644 index 0000000..e9bfb2f --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/manifest.json @@ -0,0 +1,49 @@ +{ + "benchmark_id": "decision-benchmark-v1", + "benchmark_version": "1.0.0", + "schema_version": "1.0.0", + "metadata_revision": 0, + "created_at": "2026-08-02T00:00:00.000Z", + "description": "The first ScenarioRank development benchmark: 16 fully synthetic decision cases covering basic ranking, multi-scenario behaviour, evidence quality, robustness to input permutation, and pairing. It exists to make regressions visible before any prompt, model, or scoring change is attempted. It is a development benchmark, not a validation of the product.", + "case_count": 16, + "case_ids": [ + "case-001", + "case-002", + "case-003", + "case-004", + "case-005", + "case-006", + "case-007", + "case-008", + "case-009", + "case-010", + "case-011", + "case-012", + "case-013", + "case-014", + "case-015", + "case-016" + ], + "rubric_version": "1.0.0", + "supported_modes": ["fixtures", "live"], + "required_pipeline_version": "v2-phase-2d", + "tag_catalog": [ + "basic-ranking", + "multi-scenario", + "close-call", + "missing-evidence", + "conflicting-evidence", + "permutation", + "duplicate-name", + "pairing", + "uncertainty" + ], + "data_policy": "synthetic-only", + "scope_disclaimer": "This is a development benchmark. It is not scientifically validated, not representative of real hiring decisions, not evidence of fairness or demographic neutrality, not a legal-compliance test, not a calibrated-confidence benchmark, and not a production service-level objective. Every candidate, company, and record in it is invented.", + "versioning_policy": { + "case_ids_immutable": true, + "meaning_change_requires_new_version": true, + "cosmetic_change_increments_metadata_revision": true, + "reports_record_benchmark_version_and_commit": true + } +} diff --git a/evals/datasets/decision-benchmark-v1/release-integrity.json b/evals/datasets/decision-benchmark-v1/release-integrity.json new file mode 100644 index 0000000..f353837 --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/release-integrity.json @@ -0,0 +1,7 @@ +{ + "benchmark_id": "decision-benchmark-v1", + "benchmark_version": "1.0.0", + "schema_version": "1.0.0", + "metadata_revision": 0, + "digest": "c59afa0c0362a69e7f03f3c6ef9511e9c8987dd766085b096061d6fc8efa60f8" +} diff --git a/evals/datasets/decision-benchmark-v1/rubric.json b/evals/datasets/decision-benchmark-v1/rubric.json new file mode 100644 index 0000000..342d97c --- /dev/null +++ b/evals/datasets/decision-benchmark-v1/rubric.json @@ -0,0 +1,170 @@ +{ + "rubric_version": "1.0.0", + "schema_version": "1.0.0", + "description": "Qualitative review dimensions for ScenarioRank decision explanations. These dimensions are scored by a human reviewer, never automatically. Where a deterministic grader covers part of a dimension, it covers a conservative subset only and is named explicitly.", + "interpretation_warning": "These scores are structured human opinion, not objective measurements. They are not calibrated, not inter-rater validated, and not evidence that ScenarioRank is fair, unbiased, or production-ready. A single reviewer's scores describe one reviewer's judgment of one run.", + "allowed_non_scores": ["not_applicable", "cannot_determine"], + "dimensions": [ + { + "id": "evidence_grounding", + "label": "Evidence grounding", + "what_is_judged": "Whether each claim about a candidate can be traced back to something actually present in that candidate's supplied description, rather than invented, assumed, or imported from outside the case.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "Claims about candidates are largely invented; the text describes attributes that appear nowhere in the inputs.", + "1": "Several substantive claims have no basis in the supplied descriptions.", + "2": "Mostly grounded, but at least one meaningful claim is unsupported or overstated relative to the evidence.", + "3": "Every substantive claim traces to supplied evidence; minor phrasing is looser than the evidence strictly supports.", + "4": "Every claim traces to supplied evidence, and the strength of each claim matches the strength of the evidence behind it." + }, + "failure_examples": [ + "Attributing years of experience, a former employer, or a certification that the description never mentions.", + "Describing a candidate as a proven crisis leader when the description only says they are calm under pressure." + ], + "human_review_required": true, + "deterministic_automation_possible": false, + "deterministic_grader_id": null + }, + { + "id": "scenario_relevance", + "label": "Relevance to role and scenarios", + "what_is_judged": "Whether the explanation actually engages with the specific role and the specific scenario supplied, rather than producing generic executive-hiring prose that would read identically for any role or scenario.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "The explanation is entirely generic; nothing ties it to this role or this scenario.", + "1": "The scenario is mentioned but never affects the reasoning.", + "2": "The scenario influences part of the reasoning but is ignored where it should matter most.", + "3": "The role and scenario shape the reasoning throughout, with a minor generic passage.", + "4": "Every substantive judgment is visibly conditioned on this role and this scenario." + }, + "failure_examples": [ + "Recommendation text that would be unchanged if the scenario were replaced with an unrelated one.", + "Discussing crisis leadership at length in a scenario that is explicitly about steady-state scaling." + ], + "human_review_required": true, + "deterministic_automation_possible": true, + "deterministic_grader_id": "scenario-coverage" + }, + { + "id": "tradeoff_clarity", + "label": "Quality of trade-off explanation", + "what_is_judged": "Whether the explanation states what is genuinely given up by choosing the recommended candidate, in concrete terms a decision-maker could act on, rather than listing costless positives.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "No trade-off is acknowledged; the recommendation is presented as costless.", + "1": "A trade-off is named but is vacuous or is actually another benefit in disguise.", + "2": "A real trade-off is named but not explained well enough to act on.", + "3": "Real trade-offs are named and explained, with the runner-up comparison mostly clear.", + "4": "Real trade-offs are named, quantified against the runner-up where possible, and clearly attributed to specific evidence." + }, + "failure_examples": [ + "Listing 'may be too thorough' as the sole trade-off.", + "Recommending a candidate over a close runner-up without saying what the runner-up would have brought." + ], + "human_review_required": true, + "deterministic_automation_possible": false, + "deterministic_grader_id": null + }, + { + "id": "clarity", + "label": "Clarity", + "what_is_judged": "Whether a non-specialist decision-maker can read the output once and correctly state who was recommended, on what basis, and with what reservations.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "The output is confusing or self-contradictory; the recommendation is hard to identify.", + "1": "The recommendation is identifiable but the reasoning is muddled or padded with jargon.", + "2": "Understandable with effort; structure or wording obscures the reasoning in places.", + "3": "Clear and readable; a reader would get the recommendation and reasoning right on one pass.", + "4": "Clear, concise, well-structured, and free of filler; every sentence carries decision-relevant content." + }, + "failure_examples": [ + "Executive summary and detailed explanation that emphasise different reasons for the same choice.", + "Heavy use of unexplained internal terminology such as raw criterion keys." + ], + "human_review_required": true, + "deterministic_automation_possible": false, + "deterministic_grader_id": null + }, + { + "id": "uncertainty_handling", + "label": "Uncertainty handling", + "what_is_judged": "Whether the output acknowledges thin, missing, or conflicting evidence where it genuinely exists, and refrains from manufacturing confidence the evidence does not support.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "Presents a confident conclusion despite obviously missing or conflicting evidence.", + "1": "Uncertainty is mentioned only in boilerplate that does not reflect this case's actual gaps.", + "2": "Some real gaps are acknowledged; others that clearly matter are not.", + "3": "The material evidence gaps are acknowledged and their effect on the recommendation is stated.", + "4": "Gaps are acknowledged, their effect on the recommendation is stated, and what additional evidence would resolve them is identified." + }, + "failure_examples": [ + "A confident single recommendation in a case where two candidates are separated by noise.", + "Describing model confidence as a probability that the recommendation is correct." + ], + "human_review_required": true, + "deterministic_automation_possible": true, + "deterministic_grader_id": "uncertainty-acknowledgement" + }, + { + "id": "recommendation_consistency", + "label": "Recommendation consistency", + "what_is_judged": "Whether the narrative text agrees with the structured result: the candidate the prose recommends is the candidate the deterministic ranking selected, and no section implies a different choice.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "The prose recommends a different candidate than the structured result.", + "1": "A section of the prose clearly implies a different candidate should be chosen.", + "2": "The prose is consistent but hedges enough that the recommendation is ambiguous.", + "3": "The prose consistently supports the structured recommendation.", + "4": "Every section — summary, winner reason, trade-offs, alternative — consistently supports the structured recommendation and correctly identifies the runner-up." + }, + "failure_examples": [ + "Executive summary naming the runner-up as the recommendation.", + "A trade-off section arguing the alternative is the safer choice without acknowledging the recommendation stands." + ], + "human_review_required": true, + "deterministic_automation_possible": true, + "deterministic_grader_id": "unsupported-claims" + }, + { + "id": "pairing_usefulness", + "label": "Pairing explanation quality", + "what_is_judged": "When pairing is enabled, whether the pair explanation says something genuinely about the combination — complementarity, overlap, friction — rather than restating each individual's strengths side by side.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "The pair explanation is wrong about who is in the pair, or pairing is claimed when unavailable.", + "1": "The explanation simply concatenates two individual summaries.", + "2": "Some combination-level reasoning, but overlap or friction is ignored.", + "3": "Real combination-level reasoning covering both complementarity and overlap.", + "4": "Combination-level reasoning that also explains why this pair beats the obvious alternative pairing." + }, + "failure_examples": [ + "Naming a pair whose display names do not match its candidate IDs.", + "Recommending the two highest-ranked individuals as a pair with no reasoning about how they work together." + ], + "human_review_required": true, + "deterministic_automation_possible": true, + "deterministic_grader_id": "pairing-integrity" + }, + { + "id": "unsupported_claim_avoidance", + "label": "Unsupported-claim avoidance", + "what_is_judged": "Whether the output avoids claims the system cannot support: measured fairness, demographic neutrality, calibrated confidence, observed cross-scenario performance, or statistical stability from a single run.", + "scale": { "min": 0, "max": 4 }, + "anchors": { + "0": "Makes an explicit fairness, bias-freedom, or calibration claim.", + "1": "Strongly implies measured properties the system never measured.", + "2": "Contains at least one overstated claim about what was measured.", + "3": "No unsupported measurement claims; some confident phrasing sits close to the line.", + "4": "Claims are precisely scoped to what was actually computed, and unmeasured concepts are named as unmeasured." + }, + "failure_examples": [ + "Describing the result as an objective or bias-free assessment.", + "Reporting cross-scenario consistency as observed when the field is 'not_measured'.", + "Calling model confidence a calibrated probability." + ], + "human_review_required": true, + "deterministic_automation_possible": true, + "deterministic_grader_id": "unsupported-claims" + } + ] +} diff --git a/evals/datasets/loadBenchmark.js b/evals/datasets/loadBenchmark.js new file mode 100644 index 0000000..4b550b4 --- /dev/null +++ b/evals/datasets/loadBenchmark.js @@ -0,0 +1,250 @@ +/** + * @file Benchmark loading and validation (Phase 3A evaluation harness). + * + * Loading is deliberately strict and fail-closed. A benchmark that does not + * fully validate is never partially executed: a half-valid benchmark produces + * results that look authoritative and are not. + */ +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { evaluationRequestSchema } from "../../shared/contracts/decisionApi.js"; +import { CRITERIA_KEYS } from "../../server/ai/schemas/criteriaKeys.js"; +import { parseBenchmarkCase } from "../schemas/benchmarkCase.js"; +import { ALL_GRADERS } from "../graders/deterministicGraders.js"; +import { + benchmarkManifestSchema, + rubricSchema, + assertSupportedSchemaVersion, + assertPipelineCompatibility, +} from "../schemas/benchmarkManifest.js"; +import { assertReleasedBenchmarkIntegrity } from "./releasedBenchmarkIntegrity.js"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); + +/** Repository root, derived from this module's own location. */ +export const REPO_ROOT = path.resolve(HERE, "..", ".."); + +/** The maximum logical model-backed stages the production pipeline can use. */ +const PRODUCTION_MAX_LOGICAL_STAGES = 4; + +export const DEFAULT_BENCHMARK_ID = "decision-benchmark-v1"; + +/** + * Converts an absolute path to a repository-relative one. Every path that + * reaches an artifact goes through this — recording `/Users//...` in + * a committed or shared report leaks the machine layout for no benefit. + * @param {string} absolutePath + */ +export function toRepoRelative(absolutePath) { + const relative = path.relative(REPO_ROOT, absolutePath); + return relative.split(path.sep).join("/"); +} + +export class BenchmarkValidationError extends Error { + /** @param {string} message @param {string[]} issues */ + constructor(message, issues) { + super(`${message}\n${issues.map((issue) => ` - ${issue}`).join("\n")}`); + this.name = "BenchmarkValidationError"; + this.issues = issues; + } +} + +async function readJson(filePath) { + let raw; + try { + raw = await readFile(filePath, "utf8"); + } catch (error) { + throw new BenchmarkValidationError(`Could not read ${toRepoRelative(filePath)}.`, [ + error && typeof error === "object" && "code" in error ? `filesystem error: ${error.code}` : "filesystem error", + ]); + } + try { + return JSON.parse(raw); + } catch (error) { + throw new BenchmarkValidationError(`Could not parse ${toRepoRelative(filePath)} as JSON.`, [ + error.message, + ]); + } +} + +function formatIssues(zodError) { + return zodError.issues.map( + (issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`, + ); +} + +/** + * Loads, validates, and cross-checks a benchmark directory. + * + * Every one of these checks exists because its absence would let a silently + * broken benchmark produce confident-looking numbers: + * - schema version must be one this build understands; + * - the manifest, the rubric, and every case must validate; + * - the manifest's case list and the case files on disk must agree exactly; + * - every case's decision input must validate against the *production* + * public request contract, not an evaluation-only copy of it; + * - every rubric dimension a case references must exist; + * - every variant's `variant_of` must point at a case that exists; + * - the benchmark's declared pipeline generation must match this harness. + * + * @param {{ benchmarkId?: string, datasetsDir?: string }} [options] + */ +export async function loadBenchmark({ + benchmarkId = DEFAULT_BENCHMARK_ID, + datasetsDir = HERE, +} = {}) { + const benchmarkDir = path.join(datasetsDir, benchmarkId); + const manifestRaw = await readJson(path.join(benchmarkDir, "manifest.json")); + + assertSupportedSchemaVersion(manifestRaw?.schema_version); + + const manifestResult = benchmarkManifestSchema.safeParse(manifestRaw); + if (!manifestResult.success) { + throw new BenchmarkValidationError( + `Invalid manifest for benchmark "${benchmarkId}".`, + formatIssues(manifestResult.error), + ); + } + const manifest = manifestResult.data; + + if (manifest.benchmark_id !== benchmarkId) { + throw new BenchmarkValidationError(`Benchmark identity mismatch.`, [ + `Directory is "${benchmarkId}" but manifest declares "${manifest.benchmark_id}".`, + ]); + } + + const rubricResult = rubricSchema.safeParse( + await readJson(path.join(benchmarkDir, "rubric.json")), + ); + if (!rubricResult.success) { + throw new BenchmarkValidationError( + `Invalid rubric for benchmark "${benchmarkId}".`, + formatIssues(rubricResult.error), + ); + } + const rubric = rubricResult.data; + + if (rubric.rubric_version !== manifest.rubric_version) { + throw new BenchmarkValidationError(`Rubric version mismatch.`, [ + `Manifest declares rubric ${manifest.rubric_version}, rubric file is ${rubric.rubric_version}.`, + ]); + } + + assertPipelineCompatibility({ + criteriaKeys: CRITERIA_KEYS, + maxLogicalStages: PRODUCTION_MAX_LOGICAL_STAGES, + declaredVersion: manifest.required_pipeline_version, + }); + + const casesDir = path.join(benchmarkDir, "cases"); + const caseFiles = (await readdir(casesDir)) + .filter((name) => name.endsWith(".json")) + .sort(); + + const issues = []; + const cases = []; + const rubricIds = new Set(rubric.dimensions.map((dimension) => dimension.id)); + const graderIds = new Set(ALL_GRADERS.map((grader) => grader.id)); + + for (const fileName of caseFiles) { + const parsed = parseBenchmarkCase(await readJson(path.join(casesDir, fileName))); + if (!parsed.ok) { + issues.push(...parsed.issues.map((issue) => `${fileName}: ${issue}`)); + continue; + } + const benchmarkCase = parsed.data; + + if (`${benchmarkCase.case_id}.json` !== fileName) { + issues.push(`${fileName}: file name must match case_id "${benchmarkCase.case_id}".`); + } + if (benchmarkCase.schema_version !== manifest.schema_version) { + issues.push( + `${fileName}: schema_version "${benchmarkCase.schema_version}" does not match the manifest's "${manifest.schema_version}".`, + ); + } + for (const tag of benchmarkCase.tags) { + if (!manifest.tag_catalog.includes(tag)) { + issues.push(`${fileName}: tag "${tag}" is not in the manifest tag catalog.`); + } + } + for (const dimensionId of benchmarkCase.rubric_dimensions) { + if (!rubricIds.has(dimensionId)) { + issues.push(`${fileName}: unknown rubric dimension "${dimensionId}".`); + } + } + + // A known-defect record that names a grader which does not exist would + // silently suppress nothing while looking like an acknowledged issue. + for (const defect of benchmarkCase.known_defects) { + for (const observation of defect.expected_observations) { + if (!graderIds.has(observation.grader_id)) { + issues.push( + `${fileName}: known defect ${defect.defect_id} references unknown grader "${observation.grader_id}".`, + ); + } + } + } + + // Every scenario must produce a request the *production* contract accepts. + // If the benchmark can describe a request the server would reject, the + // benchmark is measuring something the product cannot actually do. + benchmarkCase.input.scenarios.forEach((scenario, index) => { + const requestResult = evaluationRequestSchema.safeParse({ + role: benchmarkCase.input.role, + scenario, + decision_mode: benchmarkCase.input.decision_mode, + candidates: benchmarkCase.input.candidates, + options: benchmarkCase.input.options, + }); + if (!requestResult.success) { + issues.push( + ...formatIssues(requestResult.error).map( + (issue) => `${fileName}: scenario[${index}] fails the production request contract: ${issue}`, + ), + ); + } + }); + + cases.push(benchmarkCase); + } + + const foundIds = cases.map((benchmarkCase) => benchmarkCase.case_id); + const declaredIds = manifest.case_ids; + const missing = declaredIds.filter((id) => !foundIds.includes(id)); + const unexpected = foundIds.filter((id) => !declaredIds.includes(id)); + if (missing.length > 0) { + issues.push(`manifest lists case(s) with no file on disk: ${missing.join(", ")}`); + } + if (unexpected.length > 0) { + issues.push(`case file(s) not listed in the manifest: ${unexpected.join(", ")}`); + } + + const byId = new Map(cases.map((benchmarkCase) => [benchmarkCase.case_id, benchmarkCase])); + for (const benchmarkCase of cases) { + if (benchmarkCase.variant_of && !byId.has(benchmarkCase.variant_of)) { + issues.push( + `${benchmarkCase.case_id}: variant_of references unknown case "${benchmarkCase.variant_of}".`, + ); + } + } + + if (issues.length > 0) { + throw new BenchmarkValidationError( + `Benchmark "${benchmarkId}" failed validation.`, + issues, + ); + } + + // Only the repository's released corpus is locked. Temporary datasets in + // schema tests remain free to model invalid inputs deliberately. + if (path.resolve(datasetsDir) === HERE) { + await assertReleasedBenchmarkIntegrity(benchmarkDir, manifest); + } + + // Ordered by the manifest, so a run's case order is a property of the + // benchmark rather than of the filesystem. + const ordered = declaredIds.map((id) => byId.get(id)); + return { manifest, rubric, cases: ordered, benchmarkDir }; +} diff --git a/evals/datasets/released-benchmark-registry.json b/evals/datasets/released-benchmark-registry.json new file mode 100644 index 0000000..419c576 --- /dev/null +++ b/evals/datasets/released-benchmark-registry.json @@ -0,0 +1,14 @@ +{ + "records": [ + { + "benchmark_id": "decision-benchmark-v1", + "benchmark_version": "1.0.0", + "schema_version": "1.0.0", + "metadata_revision": 0, + "previous_digest": null, + "digest": "c59afa0c0362a69e7f03f3c6ef9511e9c8987dd766085b096061d6fc8efa60f8", + "reason": "Initial reviewed Phase 3A release record.", + "timestamp": "2026-08-02T14:04:51.000Z" + } + ] +} diff --git a/evals/datasets/releasedBenchmarkIntegrity.js b/evals/datasets/releasedBenchmarkIntegrity.js new file mode 100644 index 0000000..c1759ec --- /dev/null +++ b/evals/datasets/releasedBenchmarkIntegrity.js @@ -0,0 +1,89 @@ +/** + * Content locks for released development benchmarks. The lock deliberately + * lives in the repository-level registry: changing a released case and its + * local integrity declaration together still fails until the explicit update + * command records reviewed provenance in that external release record. + */ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +/** Filled from the reviewed v1 corpus; new benchmark versions require a new key. */ +const HERE = path.dirname(fileURLToPath(import.meta.url)); +export const RELEASE_REGISTRY_PATH = path.join(HERE, "released-benchmark-registry.json"); + +export async function readReleaseRegistry() { + const registry = JSON.parse(await readFile(RELEASE_REGISTRY_PATH, "utf8")); + if (!Array.isArray(registry.records)) throw new Error("Released benchmark registry is invalid."); + return registry; +} + +/** Canonical JSON: object-key order and whitespace never affect a release. */ +export function canonicalJson(value) { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +export async function benchmarkContentDigest(benchmarkDir, manifest) { + const files = ["manifest.json", "rubric.json", ...manifest.case_ids.map((id) => `cases/${id}.json`)]; + const hash = createHash("sha256"); + for (const relativePath of files) { + hash.update(relativePath); + hash.update("\0"); + const parsed = JSON.parse(await readFile(path.join(benchmarkDir, relativePath), "utf8")); + hash.update(canonicalJson(parsed)); + hash.update("\0"); + } + return hash.digest("hex"); +} + +export async function assertReleasedBenchmarkIntegrity(benchmarkDir, manifest) { + const key = `${manifest.benchmark_id}@${manifest.benchmark_version}`; + const releasePath = path.join(benchmarkDir, "release-integrity.json"); + let release; + try { + release = JSON.parse(await readFile(releasePath, "utf8")); + } catch { + throw new Error(`Released benchmark ${key} is missing a valid release-integrity.json file.`); + } + const registry = await readReleaseRegistry(); + const external = registry.records.find((record) => + record.benchmark_id === manifest.benchmark_id && + record.benchmark_version === manifest.benchmark_version && + record.schema_version === manifest.schema_version && + record.metadata_revision === manifest.metadata_revision, + ); + const expected = release.digest; + if ( + release.benchmark_id !== manifest.benchmark_id || + release.benchmark_version !== manifest.benchmark_version || + release.schema_version !== manifest.schema_version || + release.metadata_revision !== manifest.metadata_revision || + typeof expected !== "string" || !/^[a-f0-9]{64}$/.test(expected) + ) { + throw new Error(`Released benchmark ${key} has an invalid release-integrity.json declaration.`); + } + if (!expected) { + throw new Error( + `Released benchmark ${key} has no reviewed content lock. ` + + "Add the reviewed version and digest to the release registry before it can run.", + ); + } + if (!external || external.digest !== expected) { + throw new Error( + `Released benchmark ${key} does not match the repository-level release registry. ` + + "Use the explicit eval:update-integrity command after reviewer confirmation; normal validation never updates release records.", + ); + } + const actual = await benchmarkContentDigest(benchmarkDir, manifest); + if (actual !== expected) { + throw new Error( + `Released benchmark ${key} does not match its reviewed content lock. ` + + "Do not edit released case meaning in place: create a new benchmark version and add its reviewed digest to the release registry.", + ); + } +} diff --git a/evals/fixtures/fakeProviderProfiles.js b/evals/fixtures/fakeProviderProfiles.js new file mode 100644 index 0000000..d39124f --- /dev/null +++ b/evals/fixtures/fakeProviderProfiles.js @@ -0,0 +1,347 @@ +/** + * @file Offline fake-provider profiles (Phase 3A evaluation harness). + * + * These build a provider that satisfies the real provider-neutral contract + * (server/ai/types.js) and is driven entirely by a benchmark case's + * `fake_provider_plan`. The point is to exercise the *real* pipeline — + * real prompts, real schemas, real deterministic scoring, real batch-identity + * validation, real metadata accounting — with zero network access and zero + * API cost. + * + * Two properties matter more than realism here: + * + * 1. **Order independence.** Every score is looked up by candidate ID, never + * by array position. A candidate-order permutation of a case therefore + * receives byte-identical scoring input, which is what makes case-011 a + * genuine test of the pipeline rather than a test of the fixture. + * + * 2. **Determinism.** No randomness, no clock reads, no environment reads. + * The same case executed twice produces the same decision content. + * Timestamps, durations, and `request_id` are produced by the pipeline + * itself and are excluded from every comparison. + * + * What this is NOT: evidence about how a real model behaves. A fixture run + * proves the orchestration, the deterministic scoring, and the graders work. + * It proves nothing about prompt quality. See docs/evaluation/EVALUATION_ARCHITECTURE.md. + */ +import { CRITERIA_KEYS } from "../../server/ai/schemas/criteriaKeys.js"; +import { canonicalPairKey } from "../schemas/benchmarkCase.js"; + +/** + * The fake role analysis always returns these baseline criterion weights. + * They sum to exactly 100, so the pipeline's renormalisation guard is a no-op + * and a case's declared weight deltas are the only thing shifting emphasis. + */ +export const BASELINE_WEIGHTS = Object.freeze({ + domain_expertise: 18, + transformation_leadership: 16, + operational_execution: 15, + stakeholder_management: 14, + crisis_management: 13, + innovation_digital: 12, + strategic_scalability: 12, +}); + +/** + * Every profile this harness can run. Invalid profiles exist so the graders + * can be proven to catch real defects; they are used in targeted tests, never + * in the committed fixture baseline (docs/evaluation/RUNBOOK.md). + */ +export const FAKE_PROVIDER_PROFILES = Object.freeze({ + "valid-standard": { valid: true, description: "Complete, well-formed responses at every stage." }, + "valid-close-call": { + valid: true, + description: "Complete responses that compress candidate scores toward each other, so ranking margins are small.", + }, + "valid-pairing": { + valid: true, + description: "Complete responses including full, valid coverage of every expected pair.", + }, + "malformed-once-then-success": { + valid: false, + description: "Batch candidate scoring omits one candidate on the first attempt, then returns a complete set on the corrective retry. Exercises attempt accounting without changing the logical stage count.", + }, + "missing-pair": { + valid: false, + description: "Batch pairing analysis always omits one expected pair, so pairing must honestly report itself unavailable rather than presenting a partial best pair.", + }, + "unknown-candidate": { + valid: false, + description: "Batch candidate scoring returns a candidate ID that was never submitted, on every attempt. The scoring stage must fail rather than accept it.", + }, + "contradictory-explanation": { + valid: false, + description: "Structurally valid responses whose narrative recommends the runner-up instead of the deterministically ranked winner.", + }, +}); + +/** Profiles that a committed benchmark case is allowed to declare. */ +export const VALID_BASELINE_PROFILES = Object.freeze( + Object.entries(FAKE_PROVIDER_PROFILES) + .filter(([, meta]) => meta.valid) + .map(([id]) => id), +); + +const EVIDENCE_TEXT = Object.freeze({ + specific: + "The supplied description names a dated, measurable outcome for this criterion.", + vague: "Unclear.", + conflicting: + "The supplied description makes two claims about this criterion that cannot both be true, and gives no dates or figures to resolve them.", + missing: "", +}); + +/** + * Resolves one candidate's effective score plan for a given scenario index. + * A scenario override is a shallow merge over the base plan — the base plan + * stays the single place a candidate's default profile is stated. + */ +function resolveScorePlan(plan, candidateId, scenarioIndex) { + const base = plan.candidate_scores[candidateId]; + const override = plan.scenario_overrides?.[String(scenarioIndex)]?.[candidateId]; + if (!override) return base; + return { + ...base, + ...override, + criteria: { ...(base.criteria ?? {}), ...(override.criteria ?? {}) }, + }; +} + +/** + * Compresses a score toward the midpoint so ranking margins shrink without + * ever producing an exact tie. Exact ties are avoided deliberately: the + * production ranking resolves them by submission order (a stable sort over + * the submitted candidate array), which would make a candidate-order + * permutation legitimately change the winner. That behaviour is documented as + * a limitation rather than exercised as a baseline expectation. + */ +function compressTowardMidpoint(score, index) { + const compressed = 6.5 + (score - 6.5) * 0.15; + const separation = index * 0.01; + return Math.min(10, Math.max(1, Math.round((compressed + separation) * 100) / 100)); +} + +function buildCriteriaScores(scorePlan, { closeCall, candidateIndex }) { + const confidence = scorePlan.confidence ?? 0.8; + const evidence = EVIDENCE_TEXT[scorePlan.evidence_quality ?? "specific"]; + return Object.fromEntries( + CRITERIA_KEYS.map((key) => { + const raw = scorePlan.criteria?.[key] ?? scorePlan.default; + const score = closeCall ? compressTowardMidpoint(raw, candidateIndex) : raw; + return [ + key, + { + score, + confidence, + evidence, + reasoning: `Derived from the ${scorePlan.evidence_quality ?? "specific"} evidence supplied for ${key.replace(/_/g, " ")}.`, + }, + ]; + }), + ); +} + +const DEFAULT_PAIR_METRICS = Object.freeze({ + scenario_coverage: 0.72, + complementarity: 0.6, + overlap_risk: 0.35, + conflict_risk: 0.25, + execution_cohesion: 0.68, + pair_adaptability: 0.6, +}); + +/** + * Candidate IDs appear in the scoring prompt as `candidate_id: X\nName:`. + * Parsing them back out (rather than closing over the case's candidate list) + * means the fixture responds to what the pipeline actually asked for, so a + * pipeline bug that sends the wrong candidate set is visible instead of + * masked. + */ +function candidateIdsFromPrompt(prompt) { + return [...prompt.matchAll(/candidate_id: (\S+)\nName:/g)].map((match) => match[1]); +} + +function pairsFromPrompt(prompt) { + return [...prompt.matchAll(/candidate_id_a: ([^,\s]+), candidate_id_b: ([^,)\s]+)/g)].map( + (match) => [match[1], match[2]], + ); +} + +/** The decision prompt lists `Rank N: | ...`; used only by the contradictory profile. */ +function rankedNamesFromPrompt(prompt) { + return [...prompt.matchAll(/Rank \d+: ([^|\n]+?) \|/g)].map((match) => match[1].trim()); +} + +/** + * Builds a deterministic, offline provider for one execution of one case. + * + * @param {object} options + * @param {object} options.benchmarkCase validated benchmark case + * @param {number} options.scenarioIndex zero-based scenario index + * @param {string} [options.profile] overrides the case's declared profile + * @returns {{ name: string, model: string, generateStructured: Function, calls: object[] }} + */ +export function createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile }) { + const activeProfile = profile ?? benchmarkCase.fake_provider_plan.profile; + if (!(activeProfile in FAKE_PROVIDER_PROFILES)) { + throw new Error( + `Unknown fake provider profile "${activeProfile}". Known profiles: ${Object.keys(FAKE_PROVIDER_PROFILES).join(", ")}.`, + ); + } + + const plan = benchmarkCase.fake_provider_plan; + const closeCall = activeProfile === "valid-close-call"; + const candidateOrder = benchmarkCase.input.candidates.map((candidate) => candidate.id); + const calls = []; + const attemptsByPrompt = new Map(); + + function nextAttempt(promptId) { + const attempt = (attemptsByPrompt.get(promptId) ?? 0) + 1; + attemptsByPrompt.set(promptId, attempt); + return attempt; + } + + function contextAnalysis() { + const deltas = plan.scenario_weight_deltas?.[String(scenarioIndex)] ?? {}; + return { + role_analysis: { + criteria: [...CRITERIA_KEYS], + baseline_weights: { ...BASELINE_WEIGHTS }, + must_have_criteria: ["domain_expertise"], + role_success_definition: "The role succeeds when the stated scenario outcome is achieved without a service or safety breach.", + complexity_rating: "high", + }, + scenario_analysis: { + priority_shifts: Object.entries(deltas).map( + ([key, value]) => `${key.replace(/_/g, " ")} weighted ${value >= 0 ? "up" : "down"} by ${Math.abs(value)}.`, + ), + weight_deltas: Object.fromEntries( + CRITERIA_KEYS.map((key) => [key, deltas[key] ?? 0]), + ), + scenario_success_definition: "The scenario succeeds when its stated objective is met within the stated constraint.", + scenario_failure_definition: "The scenario fails when the stated constraint is breached.", + scenario_risks: ["Constraint breach", "Loss of key capability"], + key_pressures: ["Time", "Capability fit"], + weight_rationale: Object.keys(deltas).length > 0 + ? "Criterion weights were adjusted to reflect what this specific scenario actually demands." + : "This scenario does not shift criterion emphasis away from the role baseline.", + }, + }; + } + + function batchCandidateScoring(request) { + const attempt = nextAttempt("batch-candidate-scoring"); + const requestedIds = candidateIdsFromPrompt(request.prompt); + const results = requestedIds.map((id) => { + const scorePlan = resolveScorePlan(plan, id, scenarioIndex); + if (!scorePlan) { + throw new Error(`Benchmark case ${benchmarkCase.case_id} has no score plan for "${id}".`); + } + return { + candidate_id: id, + criteria_scores: buildCriteriaScores(scorePlan, { + closeCall, + candidateIndex: candidateOrder.indexOf(id), + }), + strengths: ["Strength stated in the supplied description."], + weaknesses: ["Gap stated in, or absent from, the supplied description."], + best_fit_contexts: ["The context described in this scenario."], + }; + }); + + if (activeProfile === "unknown-candidate") { + // Always invalid: the scoring stage must reject a candidate that was + // never submitted, on the corrective retry as well as the first call. + return { results: [...results.slice(1), { ...results[0], candidate_id: "ghost-candidate" }] }; + } + if (activeProfile === "malformed-once-then-success" && attempt === 1) { + // Incomplete on the first attempt only. The pipeline's batch-integrity + // corrective retry should recover, spending a second real attempt + // without entering a second logical stage. + return { results: results.slice(1) }; + } + return { results }; + } + + function batchPairingAnalysis(request) { + nextAttempt("batch-pairing-analysis"); + const requestedPairs = pairsFromPrompt(request.prompt); + const results = requestedPairs.map(([a, b]) => { + const metrics = plan.pair_overrides?.[canonicalPairKey(a, b)] ?? DEFAULT_PAIR_METRICS; + return { + candidate_id_a: a, + candidate_id_b: b, + ...metrics, + explanation: "Combination assessed on coverage, complementarity, overlap, and cohesion.", + }; + }); + if (activeProfile === "missing-pair") { + return { results: results.slice(1) }; + } + return { results }; + } + + function decisionExplanation(request) { + nextAttempt("decision-explanation"); + const rankedNames = rankedNamesFromPrompt(request.prompt); + const winnerName = rankedNames[0] ?? "the top-ranked candidate"; + const runnerUpName = rankedNames[1] ?? winnerName; + // The contradictory profile deliberately names the runner-up as the + // recommendation while the structured result still names the winner. It + // exists so the unsupported-claims grader can be proven to catch a real + // narrative/structure disagreement. + const namedChoice = activeProfile === "contradictory-explanation" ? runnerUpName : winnerName; + + return { + final_label: "Best Fit", + key_reason: `${namedChoice} scored highest on the criteria this scenario weights most heavily.`, + executive_interpretation: `${namedChoice} is the recommended candidate for this scenario, based on the evidence supplied.`, + winner_reason: `${namedChoice} leads on the criteria this scenario prioritises.`, + runner_up_trade_off: `${runnerUpName} brings comparable strengths in adjacent areas but is weaker on the scenario's primary criterion.`, + trade_offs: [ + { + title: "Depth over breadth", + description: "The recommendation favours depth in the scenario's primary criterion over broader coverage.", + type: "sacrifice", + severity: "medium", + }, + ], + executive_summary: { + recommendation: `${namedChoice} is recommended.`, + reason: "Highest deterministically computed score under the selected decision mode.", + trade_off: "Breadth across secondary criteria is lower than for the runner-up.", + opportunity_cost: "The runner-up's adjacent strengths are not obtained.", + adaptability: "Adaptability is a heuristic from this run's criteria only; cross-scenario resilience was not measured.", + alternative: runnerUpName, + }, + }; + } + + const handlers = { + "context-analysis": contextAnalysis, + "batch-candidate-scoring": batchCandidateScoring, + "batch-pairing-analysis": batchPairingAnalysis, + "decision-explanation": decisionExplanation, + }; + + return { + name: "fake-eval", + model: `fixture:${activeProfile}`, + profile: activeProfile, + async generateStructured(request) { + calls.push({ promptId: request.promptId }); + const handler = handlers[request.promptId]; + if (!handler) { + throw new Error(`No fixture handler registered for promptId "${request.promptId}".`); + } + // Validating through the production schema here — exactly as the real + // adapter does — means a fixture that drifts out of contract fails + // loudly rather than quietly feeding invalid data into the pipeline. + const data = request.schema.parse(handler(request)); + return { data, meta: { provider: "fake-eval", model: `fixture:${activeProfile}`, latencyMs: 0, attempts: 1 } }; + }, + get calls() { + return calls; + }, + }; +} diff --git a/evals/graders/deterministicGraders.js b/evals/graders/deterministicGraders.js new file mode 100644 index 0000000..47e2534 --- /dev/null +++ b/evals/graders/deterministicGraders.js @@ -0,0 +1,1039 @@ +/** + * @file Deterministic graders (Phase 3A evaluation harness). + * + * A deterministic grader answers a question that is objectively true or false + * about a pipeline response. It never scores writing quality, never judges + * whether a recommendation was *wise*, and never stands in for the human + * rubric. Where a check cannot be made honestly from the available data, the + * grader returns `skip` with the reason rather than a confident pass. + * + * Severity: + * - `required` — a failure is a defect. The run fails and the CLI exits + * nonzero. + * - `advisory` — a signal worth reading. It does not gate the exit status, + * because the check is either heuristic or observational. + * + * The unsupported-claim checks in particular are deliberately conservative. + * Keyword matching is not a reliable way to detect overclaiming, and treating + * it as authoritative would be its own form of overclaiming. They target a + * small set of specific, high-confidence phrases and are scoped to + * model-authored narrative fields only, so the pipeline's own honest + * "has not been measured" wording can never trip them. + */ +import { + completedPipelineResponseSchema, + pipelineStageProgressEventSchema, + runMetadataSchema, +} from "../../shared/contracts/decisionApi.js"; +import { + computeExecutionRisk, + computeCultureRisk, + computeTimeRisk, + computeAdaptabilityScore, + computeExpectedOutcomeScore, + computeRiskAdjustedScore, +} from "../../server/domain/scoring.js"; +import { canonicalPairKey } from "../schemas/benchmarkCase.js"; + +/** Bumped when a grader's meaning changes, so old reports stay interpretable. */ +export const GRADER_SUITE_VERSION = "1.0.0"; + +const EPSILON = 1e-9; + +function outcome(status, summary, details = [], findingCodes = [], observations = []) { + // Findings are the source of truth. Existing graders still supply their + // concise detail strings, but every one is represented exactly once here. + // A structured observation is assigned only to its corresponding detail; + // extra details become explicit unmatched findings and cannot be suppressed. + const findings = details.map((message, index) => ({ + ...(observations[index] ?? { kind: "detail", code: "unclassified" }), + message, + })); + return { + status, + summary, + finding_codes: findingCodes, + observations, + findings, + details: findings.map((finding) => finding.message), + }; +} +const pass = (summary, details = [], findingCodes = [], observations = []) => outcome("pass", summary, details, findingCodes, observations); +const fail = (summary, details = [], findingCodes = [], observations = []) => outcome("fail", summary, details, findingCodes, observations); +const skip = (summary, details = [], findingCodes = [], observations = []) => outcome("skip", summary, details, findingCodes, observations); + +function near(actual, expected) { + return typeof actual === "number" && Math.abs(actual - expected) <= EPSILON; +} + +/** The response field the deterministic ranking actually sorted on. */ +function sortFieldForMode(decisionMode) { + if (decisionMode === "best_fit") return "weighted_fit_score"; + if (decisionMode === "lowest_risk") return "risk_adjusted_score"; + return "expected_outcome_score"; +} + +/** Model-authored narrative fields only — never deterministic pipeline text. */ +function modelAuthoredText(response) { + const decision = response.decision_result; + const summary = response.executive_summary; + return [ + decision.key_reason, + decision.executive_interpretation, + decision.final_label, + ...response.candidate_evaluations.flatMap((candidate) => [ + candidate.winner_reason ?? "", + candidate.trade_off_note ?? "", + ]), + ...response.trade_offs.flatMap((tradeOff) => [tradeOff.title, tradeOff.description]), + summary.recommendation, + summary.reason, + summary.trade_off, + summary.opportunity_cost, + summary.adaptability, + summary.alternative, + ].filter((text) => typeof text === "string" && text.length > 0); +} + +// ===== EXECUTION-SCOPE GRADERS ===== + +const contractValidity = { + id: "contract-validity", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "The response and every stage event validate against the public production contract.", + run({ response, stageSnapshots }) { + const details = []; + const findingCodes = []; + const observations = []; + + const responseResult = completedPipelineResponseSchema.safeParse(response); + if (!responseResult.success) { + details.push( + ...responseResult.error.issues.map( + (issue) => `response.${issue.path.join(".") || "(root)"}: ${issue.message}`, + ), + ); + } + + const metadataResult = runMetadataSchema.safeParse(response.run_metadata); + if (!metadataResult.success) { + details.push( + ...metadataResult.error.issues.map( + (issue) => `run_metadata.${issue.path.join(".")}: ${issue.message}`, + ), + ); + } + + stageSnapshots.forEach((snapshot, index) => { + const stageResult = pipelineStageProgressEventSchema.safeParse(snapshot); + if (!stageResult.success) { + details.push( + ...stageResult.error.issues.map( + (issue) => `stage event ${index}.${issue.path.join(".")}: ${issue.message}`, + ), + ); + } + }); + + // A number that serialises but is NaN/Infinity passes many schemas and + // then poisons every downstream calculation, so it is checked explicitly. + const malformed = []; + const walk = (value, path) => { + if (typeof value === "number" && !Number.isFinite(value)) { + malformed.push(path); + return; + } + if (Array.isArray(value)) { + value.forEach((item, index) => walk(item, `${path}[${index}]`)); + return; + } + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value)) walk(child, `${path}.${key}`); + } + }; + walk(response, "response"); + details.push(...malformed.map((path) => `non-finite number at ${path}`)); + + for (const candidate of response.candidate_evaluations ?? []) { + if (candidate.risk_adjusted_score < 0) { + findingCodes.push("negative-risk-adjusted-score"); + observations.push({ + kind: "schema_issue", + path_pattern: "candidate_evaluations.*.risk_adjusted_score", + code: "too_small", + minimum: 0, + subject_candidate_id: candidate.candidate_id, + }); + } + } + + return details.length === 0 + ? pass("Response, run metadata, and every stage event validate against the public contract.") + : fail(`${details.length} contract violation(s).`, details, findingCodes, observations); + }, +}; + +const candidateCoverage = { + id: "candidate-coverage", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Every submitted candidate is evaluated exactly once, and no unknown candidate appears.", + run({ benchmarkCase, response, trace }) { + const expected = benchmarkCase.deterministic_expectations.expected_candidate_ids; + const evaluated = response.candidate_evaluations.map((candidate) => candidate.candidate_id); + const details = []; + + const seen = new Map(); + for (const id of evaluated) seen.set(id, (seen.get(id) ?? 0) + 1); + + for (const id of expected) { + const count = seen.get(id) ?? 0; + if (count === 0) details.push(`candidate "${id}" is missing from candidate_evaluations`); + if (count > 1) details.push(`candidate "${id}" appears ${count} times`); + } + for (const id of seen.keys()) { + if (!expected.includes(id)) details.push(`unknown candidate "${id}" appears in the response`); + } + + const ranks = response.candidate_evaluations.map((candidate) => candidate.rank).sort((a, b) => a - b); + const expectedRanks = expected.map((_, index) => index + 1); + if (JSON.stringify(ranks) !== JSON.stringify(expectedRanks)) { + details.push(`ranks are ${ranks.join(", ")}; expected a contiguous 1..${expected.length}`); + } + + // Duplicate display names must stay distinguishable by ID. This is why + // case-015 exists: names are labels, IDs are identity. + const nameCounts = new Map(); + for (const candidate of response.candidate_evaluations) { + nameCounts.set(candidate.candidate_name, (nameCounts.get(candidate.candidate_name) ?? 0) + 1); + } + for (const [name, count] of nameCounts) { + if (count > 1) { + const ids = response.candidate_evaluations + .filter((candidate) => candidate.candidate_name === name) + .map((candidate) => candidate.candidate_id); + if (new Set(ids).size !== ids.length) { + details.push(`display name "${name}" is shared by entries that are not distinguishable by ID`); + } + } + } + + if (trace?.requestedCandidateIds) { + const requested = [...trace.requestedCandidateIds].sort(); + if (JSON.stringify(requested) !== JSON.stringify([...expected].sort())) { + details.push( + `the scoring stage requested [${requested.join(", ")}] but the case submitted [${[...expected].sort().join(", ")}]`, + ); + } + } + + return details.length === 0 + ? pass(`All ${expected.length} candidates evaluated exactly once, with no unknown candidate.`) + : fail(`${details.length} candidate-coverage problem(s).`, details); + }, +}; + +const rankingConsistency = { + id: "ranking-consistency", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "The reported winner is the highest deterministic score, and ranking order agrees with those scores.", + run({ benchmarkCase, response }) { + const details = []; + const sortField = sortFieldForMode(benchmarkCase.input.decision_mode); + const ranked = [...response.candidate_evaluations].sort((a, b) => a.rank - b.rank); + const submissionOrder = benchmarkCase.input.candidates.map((candidate) => candidate.id); + + for (let index = 1; index < ranked.length; index += 1) { + const previous = ranked[index - 1]; + const current = ranked[index]; + if (current[sortField] > previous[sortField] + EPSILON) { + details.push( + `rank ${current.rank} (${current.candidate_id}, ${sortField}=${current[sortField]}) outscores rank ${previous.rank} (${previous.candidate_id}, ${sortField}=${previous[sortField]})`, + ); + } else if (near(current[sortField], previous[sortField])) { + // Documented tie-break: the production ranking is a stable sort over + // the submitted candidate array, so an exact tie keeps submission + // order. This is observed behaviour, not a designed guarantee — see + // docs/evaluation/BENCHMARK_V1.md, "Known limitations". + const previousPosition = submissionOrder.indexOf(previous.candidate_id); + const currentPosition = submissionOrder.indexOf(current.candidate_id); + if (currentPosition < previousPosition) { + details.push( + `tie on ${sortField} between "${previous.candidate_id}" and "${current.candidate_id}" was not resolved by submission order`, + ); + } + } + } + + const topRanked = ranked[0]; + if (response.decision_result.recommended_candidate_id !== topRanked.candidate_id) { + details.push( + `decision_result recommends "${response.decision_result.recommended_candidate_id}" but rank 1 is "${topRanked.candidate_id}"`, + ); + } + if (response.decision_result.recommended_candidate_name !== topRanked.candidate_name) { + details.push( + `decision_result names "${response.decision_result.recommended_candidate_name}" but rank 1 is "${topRanked.candidate_name}"`, + ); + } + + const best = ranked.reduce( + (bestSoFar, candidate) => (candidate[sortField] > bestSoFar[sortField] ? candidate : bestSoFar), + ranked[0], + ); + if (!near(best[sortField], topRanked[sortField])) { + details.push( + `rank 1 (${topRanked.candidate_id}) does not hold the highest ${sortField}; "${best.candidate_id}" does`, + ); + } + + return details.length === 0 + ? pass(`Ranking agrees with deterministic ${sortField}, and the winner is rank 1.`) + : fail(`${details.length} ranking-consistency problem(s).`, details); + }, +}; + +const scoreIntegrity = { + id: "score-integrity", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Scores stay in range and every recomputable deterministic value matches a fresh recomputation.", + run({ response }) { + const details = []; + const findingCodes = []; + const observations = []; + + for (const candidate of response.candidate_evaluations) { + const label = candidate.candidate_id; + const scores = Object.fromEntries( + Object.entries(candidate.criteria_scores).map(([key, value]) => [key, value.score]), + ); + const confidences = Object.fromEntries( + Object.entries(candidate.criteria_scores).map(([key, value]) => [key, value.confidence]), + ); + + for (const [key, criterion] of Object.entries(candidate.criteria_scores)) { + if (criterion.score < 1 || criterion.score > 10) { + details.push(`${label}.${key}.score=${criterion.score} is outside 1-10`); + } + if (criterion.confidence < 0 || criterion.confidence > 1) { + details.push(`${label}.${key}.confidence=${criterion.confidence} is outside 0-1`); + } + } + for (const field of ["weighted_fit_score", "risk_adjusted_score", "expected_outcome_score"]) { + if (candidate[field] < 0 || candidate[field] > 100) { + details.push(`${label}.${field}=${candidate[field]} is outside 0-100`); + if (field === "risk_adjusted_score" && candidate[field] < 0) { + findingCodes.push("negative-risk-adjusted-score"); + observations.push({ + kind: "score_bound_violation", + metric: "risk_adjusted_score", + operator: "lt", + bound: 0, + subject_candidate_id: label, + }); + } + } + } + + // Recomputation. `weighted_fit_score` is deliberately excluded: the + // normalised criterion weights are not part of the public response, so + // it cannot be recomputed from the response alone. Everything derived + // *from* it can be, and is. + const wfs = candidate.weighted_fit_score; + const overallConfidence = candidate.overall_confidence; + const executionRisk = computeExecutionRisk(scores); + const cultureRisk = computeCultureRisk(scores, confidences); + const timeRisk = computeTimeRisk(scores, wfs); + const confidenceRisk = Math.round((1 - overallConfidence) * 100 * 100) / 100; + const adaptabilityScore = computeAdaptabilityScore(scores); + const opportunityCostRisk = + Math.round(((executionRisk + cultureRisk + timeRisk) / 3) * 100) / 100; + + const recomputed = { + "risk_profile.execution_risk": executionRisk / 100, + "risk_profile.culture_risk": cultureRisk / 100, + "risk_profile.time_risk": timeRisk / 100, + "risk_profile.confidence_risk": confidenceRisk / 100, + "risk_profile.adaptability_risk": (100 - adaptabilityScore) / 100, + "risk_profile.opportunity_cost_risk": opportunityCostRisk / 100, + "outcome_model.adaptability_score": adaptabilityScore / 100, + }; + for (const [path, expected] of Object.entries(recomputed)) { + const [group, field] = path.split("."); + const actual = candidate[group][field]; + if (!near(actual, expected)) { + details.push(`${label}.${path}=${actual} but recomputation gives ${expected}`); + } + } + + const expectedOutcome = computeExpectedOutcomeScore({ + wfs, + adapt: adaptabilityScore, + exec: executionRisk, + cult: cultureRisk, + time: timeRisk, + conf: overallConfidence, + }); + if (!near(candidate.expected_outcome_score, expectedOutcome)) { + details.push( + `${label}.expected_outcome_score=${candidate.expected_outcome_score} but recomputation gives ${expectedOutcome}`, + ); + } + + const riskAdjusted = computeRiskAdjustedScore({ + wfs, + exec: executionRisk, + cult: cultureRisk, + time: timeRisk, + conf: overallConfidence, + adapt: adaptabilityScore, + opp: opportunityCostRisk, + }); + if (!near(candidate.risk_adjusted_score, riskAdjusted)) { + details.push( + `${label}.risk_adjusted_score=${candidate.risk_adjusted_score} but recomputation gives ${riskAdjusted}`, + ); + } + } + + return details.length === 0 + ? pass("All scores in range; every recomputable deterministic value matches a fresh recomputation.") + : fail(`${details.length} score-integrity problem(s).`, details, [...new Set(findingCodes)], observations); + }, +}; + +const pairingIntegrity = { + id: "pairing-integrity", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Pairing covers every expected pair exactly once, is canonicalised, and is never fabricated when disabled.", + run({ benchmarkCase, response, trace }) { + const expectations = benchmarkCase.deterministic_expectations; + const pairing = response.pairing_result; + const details = []; + + if (!expectations.pairing_enabled) { + if (pairing !== undefined) { + details.push(`pairing is disabled for this case but pairing_result is present (status "${pairing.status}")`); + } + return details.length === 0 + ? pass("Pairing is disabled and no pair result was fabricated.") + : fail("A pair result appeared for a case with pairing disabled.", details); + } + + if (pairing === undefined) { + return fail("Pairing is enabled for this case but the response has no pairing_result.", []); + } + + // Requested-pair coverage is checked from the provider request trace: the + // response only exposes the top three pairs, so completeness of the + // evaluated set cannot be read from the response alone. + if (trace?.requestedPairKeys) { + const requested = trace.requestedPairKeys; + const unique = new Set(requested); + if (unique.size !== requested.length) { + details.push("the pairing stage requested the same unordered pair more than once"); + } + if (unique.size !== expectations.expected_pair_count) { + details.push( + `the pairing stage requested ${unique.size} unique pair(s); the case expects ${expectations.expected_pair_count}`, + ); + } + } else { + details.push("no pair request trace was available, so pair coverage could not be verified"); + } + + if (pairing.status !== "ok") { + return fail( + `Pairing reported "${pairing.status}" for a case that expects complete pair coverage.`, + [...details, `reason: ${pairing.reason}`], + ); + } + + const candidatesById = new Map( + response.candidate_evaluations.map((candidate) => [candidate.candidate_id, candidate]), + ); + const seenKeys = new Set(); + for (const pair of pairing.top_pairs) { + const key = canonicalPairKey(pair.candidate_id_a, pair.candidate_id_b); + if (seenKeys.has(key)) { + details.push(`top_pairs contains a duplicate or reversed duplicate of pair ${key}`); + } + seenKeys.add(key); + + for (const [candidateId, displayName] of [ + [pair.candidate_id_a, pair.pair[0]], + [pair.candidate_id_b, pair.pair[1]], + ]) { + const candidate = candidatesById.get(candidateId); + if (!candidate) { + details.push(`pair ${key} references candidate "${candidateId}", which is not in candidate_evaluations`); + } else if (candidate.candidate_name !== displayName) { + details.push( + `pair ${key} labels "${candidateId}" as "${displayName}" but that ID belongs to "${candidate.candidate_name}"`, + ); + } + } + } + + const bestKey = canonicalPairKey(pairing.best_pair.candidate_id_a, pairing.best_pair.candidate_id_b); + if (!seenKeys.has(bestKey)) { + details.push(`best_pair ${bestKey} does not appear in top_pairs`); + } + + for (let index = 1; index < pairing.top_pairs.length; index += 1) { + if (pairing.top_pairs[index].pair_score > pairing.top_pairs[index - 1].pair_score + EPSILON) { + details.push("top_pairs is not ordered by descending pair_score"); + } + } + if (pairing.best_pair.pair_score < pairing.top_pairs[0].pair_score - EPSILON) { + details.push("best_pair does not hold the highest pair_score in top_pairs"); + } + + if (expectations.expected_best_pair_ids) { + const expectedKey = canonicalPairKey(...expectations.expected_best_pair_ids); + if (bestKey !== expectedKey) { + details.push(`best pair is ${bestKey}; the case expects ${expectedKey}`); + } + } + + return details.length === 0 + ? pass(`Pairing covered ${expectations.expected_pair_count} pair(s) with canonical, consistent identities.`) + : fail(`${details.length} pairing-integrity problem(s).`, details); + }, +}; + +const pipelineAccounting = { + id: "pipeline-accounting", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Logical stage count, provider attempts, and token/cost metadata are internally coherent.", + run({ benchmarkCase, response }) { + const expectations = benchmarkCase.deterministic_expectations; + const metadata = response.run_metadata; + const details = []; + + if (metadata.logicalProviderStageCount !== expectations.required_stage_count) { + details.push( + `logicalProviderStageCount=${metadata.logicalProviderStageCount}; this case requires ${expectations.required_stage_count} (${expectations.pairing_enabled ? "pairing enabled" : "pairing disabled"})`, + ); + } + if (metadata.providerAttemptCount < metadata.logicalProviderStageCount) { + details.push( + `providerAttemptCount=${metadata.providerAttemptCount} is below logicalProviderStageCount=${metadata.logicalProviderStageCount}; every logical stage makes at least one attempt`, + ); + } + if (metadata.providerAttemptCount > expectations.maximum_provider_attempts) { + details.push( + `providerAttemptCount=${metadata.providerAttemptCount} exceeds this case's maximum of ${expectations.maximum_provider_attempts}`, + ); + } + + const attemptSum = Object.values(metadata.attempts).reduce((total, value) => total + value, 0); + if (attemptSum !== metadata.providerAttemptCount) { + details.push( + `per-stage attempts sum to ${attemptSum} but providerAttemptCount is ${metadata.providerAttemptCount}`, + ); + } + + if (metadata.reasoningTokens > metadata.outputTokens) { + details.push( + `reasoningTokens=${metadata.reasoningTokens} exceeds outputTokens=${metadata.outputTokens}; reasoning tokens are a subset of output tokens`, + ); + } + if (metadata.cachedInputTokens > metadata.inputTokens) { + details.push( + `cachedInputTokens=${metadata.cachedInputTokens} exceeds inputTokens=${metadata.inputTokens}`, + ); + } + if (metadata.totalTokens > 0 && metadata.totalTokens < metadata.inputTokens + metadata.outputTokens) { + details.push( + `totalTokens=${metadata.totalTokens} is below inputTokens+outputTokens=${metadata.inputTokens + metadata.outputTokens}`, + ); + } + if (metadata.estimatedCostUsd !== null && metadata.estimatedCostUsd < 0) { + details.push(`estimatedCostUsd=${metadata.estimatedCostUsd} is negative`); + } + if (metadata.totalTokens === 0 && metadata.estimatedCostUsd !== null && metadata.estimatedCostUsd > 0) { + details.push("a nonzero cost was estimated for a run that reported no tokens"); + } + + return details.length === 0 + ? pass( + `${metadata.logicalProviderStageCount} logical stage(s), ${metadata.providerAttemptCount} provider attempt(s), coherent token accounting.`, + ) + : fail(`${details.length} pipeline-accounting problem(s).`, details); + }, +}; + +const notMeasuredFields = { + id: "not-measured-fields", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Concepts the pipeline never measures are reported as not_measured, not as a number or a claim.", + run({ benchmarkCase, response }) { + const details = []; + const required = benchmarkCase.deterministic_expectations.required_not_measured_fields; + + const checks = { + "outcome_models[].cross_scenario_consistency": () => + response.outcome_models.map((model, index) => [ + `outcome_models[${index}].cross_scenario_consistency`, + model.cross_scenario_consistency, + ]), + "adaptability_profiles[].best_scenario": () => + response.adaptability_profiles.map((profile, index) => [ + `adaptability_profiles[${index}].best_scenario`, + profile.best_scenario, + ]), + "adaptability_profiles[].worst_scenario": () => + response.adaptability_profiles.map((profile, index) => [ + `adaptability_profiles[${index}].worst_scenario`, + profile.worst_scenario, + ]), + }; + + for (const field of required) { + const check = checks[field]; + if (!check) { + details.push(`no check is implemented for required_not_measured_field "${field}"`); + continue; + } + for (const [path, value] of check()) { + if (value !== "not_measured") { + details.push(`${path}=${JSON.stringify(value)}; expected the literal "not_measured"`); + } + } + } + + for (const [index, candidate] of response.candidate_evaluations.entries()) { + const value = candidate.outcome_model.cross_scenario_consistency; + if (value !== "not_measured") { + details.push( + `candidate_evaluations[${index}].outcome_model.cross_scenario_consistency=${JSON.stringify(value)}; expected "not_measured"`, + ); + } + } + + return details.length === 0 + ? pass("Every unmeasured concept is reported as not_measured.") + : fail(`${details.length} not_measured violation(s).`, details); + }, +}; + +const winnerExpectation = { + id: "winner-expectation", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "The winner is one the case considers defensible, and is not one it rules out.", + run({ benchmarkCase, response }) { + const expectations = benchmarkCase.deterministic_expectations; + const winnerId = response.decision_result.recommended_candidate_id; + const details = []; + + if (expectations.forbidden_winner_ids.includes(winnerId)) { + details.push(`"${winnerId}" is listed as a forbidden winner for this case`); + } + if (expectations.allowed_winner_ids === null) { + return details.length === 0 + ? skip("This case deliberately makes no winner claim; only forbidden winners are checked.") + : fail("A forbidden winner was selected.", details); + } + if (!expectations.allowed_winner_ids.includes(winnerId)) { + details.push( + `winner "${winnerId}" is not among the allowed winners [${expectations.allowed_winner_ids.join(", ")}]`, + ); + } + + return details.length === 0 + ? pass(`Winner "${winnerId}" is an allowed outcome for this case.`) + : fail(`${details.length} winner-expectation problem(s).`, details); + }, +}; + +/** + * Conservative, phrase-level checks for claims the system cannot support. + * Scoped to model-authored narrative only. Each pattern targets a specific + * overclaim seen in practice; the list is intentionally short, because a long + * keyword list produces false positives that make the grader untrustworthy. + */ +const UNSUPPORTED_CLAIM_PATTERNS = Object.freeze([ + { id: "fairness", pattern: /\b(bias[-\s]free|unbiased|free from bias|objectively fair|proven fair|demographically neutral)\b/i, why: "asserts a fairness or bias property that was never measured" }, + { id: "validation", pattern: /\b(scientifically|empirically|statistically)\s+(validated|proven|significant)\b/i, why: "asserts empirical validation that has not been performed" }, + { id: "calibration", pattern: /\bcalibrated\s+(probability|probabilities|confidence)\b/i, why: "asserts calibrated confidence; model confidence in this system is not calibrated" }, + { id: "guarantee", pattern: /\b(guarantees?|guaranteed)\s+(the\s+)?(best|correct|optimal|right)\b/i, why: "asserts a guarantee the system cannot make" }, +]); + +const CROSS_SCENARIO_CLAIM_PATTERNS = Object.freeze([ + { id: "cross-scenario", pattern: /\bcross[-\s]scenario\s+(consistency|performance|results?)\s+(is|was|shows?|demonstrates?)\b/i }, + { id: "every-scenario", pattern: /\b(?:performs?|performed|ranked|scored)\s+\w*\s*(?:across|in)\s+(?:all|every)\s+scenarios?\b/i }, +]); + +const STABILITY_CLAIM_PATTERNS = Object.freeze([ + { id: "stability", pattern: /\b(stable|consistent|reproducible)\s+across\s+(runs|repetitions|executions)\b/i }, + { id: "variance", pattern: /\b(low|minimal|no)\s+(run[-\s]to[-\s]run\s+)?variance\b/i }, +]); + +const unsupportedClaims = { + id: "unsupported-claims", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Narrative text avoids fairness, calibration, cross-scenario, and stability claims the system cannot support, and does not contradict the structured result.", + run({ response, repetitions }) { + const details = []; + const texts = modelAuthoredText(response); + const joined = texts.join("\n"); + + for (const { pattern, why } of UNSUPPORTED_CLAIM_PATTERNS) { + const match = joined.match(pattern); + if (match) details.push(`"${match[0]}" ${why}`); + } + + const crossScenarioUnmeasured = response.outcome_models.every( + (model) => model.cross_scenario_consistency === "not_measured", + ); + if (crossScenarioUnmeasured) { + for (const { pattern } of CROSS_SCENARIO_CLAIM_PATTERNS) { + const match = joined.match(pattern); + if (match) { + details.push( + `"${match[0]}" claims observed cross-scenario behaviour while cross_scenario_consistency is "not_measured"`, + ); + } + } + } + + if (repetitions < 2) { + for (const { pattern } of STABILITY_CLAIM_PATTERNS) { + const match = joined.match(pattern); + if (match) { + details.push(`"${match[0]}" claims measured stability, but this run executed a single repetition`); + } + } + } + + // Narrative/structure contradiction. Name-based, so it is deliberately + // skipped where the winner's display name is shared by another candidate: + // in that case the text genuinely cannot distinguish them, and reporting a + // confident pass or fail would both be wrong. + const winner = response.candidate_evaluations.find( + (candidate) => candidate.candidate_id === response.decision_result.recommended_candidate_id, + ); + const winnerNameIsShared = + response.candidate_evaluations.filter( + (candidate) => candidate.candidate_name === winner?.candidate_name, + ).length > 1; + + let contradictionChecked = false; + if (winner && !winnerNameIsShared) { + contradictionChecked = true; + const others = response.candidate_evaluations + .filter((candidate) => candidate.candidate_id !== winner.candidate_id) + .map((candidate) => candidate.candidate_name) + .filter((name) => name !== winner.candidate_name); + + const recommendationFields = { + "decision_result.key_reason": response.decision_result.key_reason, + "decision_result.executive_interpretation": response.decision_result.executive_interpretation, + "executive_summary.recommendation": response.executive_summary.recommendation, + }; + for (const [path, text] of Object.entries(recommendationFields)) { + if (typeof text !== "string" || text.length === 0) continue; + const namesOther = others.some((name) => text.includes(name)); + const namesWinner = text.includes(winner.candidate_name); + if (namesOther && !namesWinner) { + details.push( + `${path} presents a candidate other than the ranked winner "${winner.candidate_name}" as the recommendation`, + ); + } + } + } + + if (details.length > 0) { + return fail(`${details.length} unsupported or contradictory claim(s).`, details); + } + const note = contradictionChecked + ? "No unsupported claims; narrative agrees with the structured recommendation." + : "No unsupported claims. Narrative/structure contradiction was not checked: the winner's display name is shared by another candidate."; + return pass(note); + }, +}; + +const uncertaintyAcknowledgement = { + id: "uncertainty-acknowledgement", + version: "1.0.0", + severity: "required", + scope: "execution", + description: "Candidates whose evidence the case marks as thin or conflicting are flagged for human review.", + run({ benchmarkCase, response }) { + const expected = benchmarkCase.deterministic_expectations.expect_human_review_for_candidate_ids; + if (expected.length === 0) { + return skip("This case makes no uncertainty claim, so no human-review flag is required."); + } + + const reviewsById = new Map( + response.confidence_evidence_reviews.map((review) => [review.candidate_id, review]), + ); + const details = []; + for (const candidateId of expected) { + const review = reviewsById.get(candidateId); + if (!review) { + details.push(`no confidence/evidence review was produced for "${candidateId}"`); + continue; + } + if (!review.recommend_human_review) { + details.push( + `"${candidateId}" was not flagged for human review despite thin or conflicting evidence (overall_confidence=${review.overall_confidence}, weak evidence on ${review.weak_evidence_flags.length} criteria)`, + ); + } + } + + return details.length === 0 + ? pass(`${expected.length} candidate(s) with thin or conflicting evidence were flagged for human review.`) + : fail(`${details.length} uncertainty-acknowledgement problem(s).`, details); + }, +}; + +// ===== CASE-SCOPE GRADERS ===== + +const scenarioCoverage = { + id: "scenario-coverage", + version: "1.0.0", + severity: "required", + scope: "case", + description: "Every committed scenario is executed and correctly reflected in its response; none is silently ignored.", + run({ benchmarkCase, executions, repetitions }) { + const scenarios = benchmarkCase.deterministic_expectations.required_scenario_coverage; + const details = []; + + scenarios.forEach((scenario, index) => { + const matching = executions.filter((execution) => execution.scenario_index === index); + if (matching.length !== repetitions) { + details.push( + `scenario ${index} produced ${matching.length} execution(s); ${repetitions} expected for this repetition count`, + ); + } + for (const execution of matching) { + if (execution.status === "skipped") { + details.push(`scenario ${index} was not executed: ${execution.skip_reason ?? "unknown reason"}`); + continue; + } + if (execution.scenario !== scenario) { + details.push(`scenario ${index} executed with the wrong scenario text`); + } + if (execution.status !== "completed" || !execution.response) continue; + if (execution.response.scenario_analysis.scenario !== scenario) { + details.push(`scenario ${index}: scenario_analysis.scenario does not match the submitted scenario`); + } + if (execution.response.decision_result.scenario !== scenario) { + details.push(`scenario ${index}: decision_result.scenario does not match the submitted scenario`); + } + } + }); + + const unexpected = executions.filter( + (execution) => execution.scenario_index >= scenarios.length, + ); + if (unexpected.length > 0) { + details.push(`${unexpected.length} execution(s) reference a scenario index this case does not declare`); + } + + return details.length === 0 + ? pass(`All ${scenarios.length} scenario(s) executed and correctly reflected in their responses.`) + : fail(`${details.length} scenario-coverage problem(s).`, details); + }, +}; + +// ===== REGISTRY ===== + +export const EXECUTION_GRADERS = Object.freeze([ + contractValidity, + candidateCoverage, + rankingConsistency, + scoreIntegrity, + pairingIntegrity, + pipelineAccounting, + notMeasuredFields, + winnerExpectation, + unsupportedClaims, + uncertaintyAcknowledgement, +]); + +export const CASE_GRADERS = Object.freeze([scenarioCoverage]); + +export const ALL_GRADERS = Object.freeze([...EXECUTION_GRADERS, ...CASE_GRADERS]); + +/** grader_id -> grader_version, recorded in every run manifest. */ +export function graderVersions() { + return Object.fromEntries(ALL_GRADERS.map((grader) => [grader.id, grader.version])); +} + +/** + * Runs a grader list, converting an unexpected throw into an `error` result + * rather than losing the whole run. A grader that crashes is itself a defect, + * so `error` is treated exactly like `fail` for exit-status purposes. + * @param {readonly object[]} graders + * @param {object} context + */ +export function runGraders(graders, context) { + return graders.map((grader) => { + let outcome; + try { + outcome = grader.run(context); + } catch (error) { + outcome = { + status: "error", + summary: `Grader "${grader.id}" threw while evaluating this result.`, + finding_codes: [], + observations: [], + findings: [{ kind: "grader_error", code: "threw", message: error.message }], + details: [error.message], + }; + } + return { + grader_id: grader.id, + grader_version: grader.version, + severity: grader.severity, + status: outcome.status, + summary: outcome.summary, + finding_codes: outcome.finding_codes ?? [], + observations: outcome.observations ?? [], + findings: outcome.findings ?? (outcome.details ?? []).map((message) => ({ kind: "detail", code: "unclassified", message })), + details: (outcome.findings ?? (outcome.details ?? []).map((message) => ({ message }))).map((finding) => finding.message), + }; + }); +} + +/** A required grader that failed or errored is what makes a run fail. */ +export function countFailures(graderResults) { + let required = 0; + let advisory = 0; + for (const result of graderResults) { + if (result.status !== "fail" && result.status !== "error") continue; + if (result.severity === "required") required += 1; + else advisory += 1; + } + return { required, advisory }; +} + +/** + * Reclassifies a single execution's grader results against a case's documented + * known defects. + * + * A failure that matches a known defect becomes `expected_failure` and stops + * gating the exit status — the finding stays visible in every report, but a + * documented pre-existing product defect does not hold the whole baseline red. + * + * This function only ever downgrades. Whether a defect has *stopped* + * reproducing is deliberately not decided here: a defect can legitimately + * reproduce in one scenario of a case and not another (case-006 is exactly + * that shape), so judging it per execution would raise a false alarm on every + * execution that happens not to trigger it. That judgment belongs to + * `checkKnownDefectsStillReproduce`, which sees the whole case. + * + * @param {object[]} graderResults + * @param {Array<{id: string, grader_id: string, summary: string, reference: string}>} knownDefects + */ +export function applyKnownDefects(graderResults, knownDefects, { execution, benchmarkCase } = {}) { + if (!knownDefects || knownDefects.length === 0) return graderResults; + + return graderResults.map((result) => { + if (result.status !== "fail" || !execution || !benchmarkCase) return result; + const matches = knownDefects.flatMap((defect) => { + const scope = defect.execution_scope; + const scoped = defect.case_id === benchmarkCase.case_id && + scope.execution_id === execution.execution_id && + scope.scenario_id === `scenario-${execution.scenario_index + 1}` && + scope.scenario_index === execution.scenario_index && + scope.variant_id === benchmarkCase.variant_kind && + scope.repetition === execution.repetition; + if (!scoped) return []; + return defect.expected_observations + .map((expected, index) => ({ defect, expected, index })) + .filter(({ expected }) => expected.grader_id === result.grader_id && result.findings?.some((finding) => { + const actual = { ...finding }; + delete actual.message; + return JSON.stringify(actual) === JSON.stringify(expected.signature); + })); + }); + const actualFindings = result.findings ?? []; + const derivedDetails = actualFindings.map((finding) => finding.message); + // A result may be downgraded only when findings are a faithful, complete + // source for the human details and every actual failure finding is named + // by the scoped defect record. + if (matches.length === 0 || matches.length !== actualFindings.length || JSON.stringify(result.details) !== JSON.stringify(derivedDetails)) return result; + const defectIds = [...new Set(matches.map(({ defect }) => defect.defect_id))]; + if (defectIds.length !== 1) return result; + const defect = matches[0].defect; + return { + ...result, + status: "expected_failure", + known_defect_id: defect.defect_id, + known_defect_observation_ids: matches.map(({ expected, index }) => `${defect.defect_id}:${execution.execution_id}:${expected.grader_id}:${index}`), + summary: `Known defect ${defect.defect_id}: ${result.summary}`, + details: [ + ...result.details, + `known defect ${defect.defect_id} — ${defect.summary} (see ${defect.reference})`, + ], + findings: [ + ...actualFindings, + { kind: "known_defect_reference", code: defect.defect_id, message: `known defect ${defect.defect_id} — ${defect.summary} (see ${defect.reference})` }, + ], + }; + }); +} + +/** + * Case-level check that every documented known defect still reproduces + * somewhere in the case. + * + * This is the half that makes the known-defect mechanism safe to use at all. + * Without it, a `known_defects` entry would be an ordinary suppression: it + * would keep hiding a grader long after the underlying problem was fixed, and + * nobody would find out. Here, a defect that has stopped reproducing raises a + * required failure demanding the record be removed. + * + * @param {object[]} executions + * @param {object[]} caseGraderResults + * @param {Array<{id: string, grader_id: string, summary: string, reference: string}>} knownDefects + */ +export function checkKnownDefectsStillReproduce(executions, caseGraderResults, knownDefects) { + if (!knownDefects || knownDefects.length === 0) return []; + + return knownDefects.flatMap((defect) => defect.expected_observations + .map((expected, index) => ({ defect, expected, index })) + .filter(({ defect, expected, index }) => !executions.some((execution) => + execution.execution_id === defect.execution_scope.execution_id && + execution.grader_results.some((result) => result.known_defect_observation_ids?.includes(`${defect.defect_id}:${execution.execution_id}:${expected.grader_id}:${index}`)), + )) + .map(({ defect, expected }) => ({ + grader_id: `unexpected-defect-resolution:${defect.defect_id}:${expected.grader_id}`, + grader_version: "1.0.0", + severity: "required", + status: "fail", + summary: `Expected known defect ${defect.defect_id} no longer reproduces via "${expected.grader_id}" in ${defect.execution_scope.execution_id}.`, + details: [ + `${defect.summary} (see ${defect.reference})`, + "If this was fixed deliberately, remove the known_defects entry from this benchmark case and update the referenced documentation. A known-defect record must never outlive the defect it describes.", + ], + finding_codes: [], + observations: [], + findings: [ + { kind: "unexpected_defect_resolution", code: defect.defect_id, message: defect.summary }, + { kind: "remediation", code: "review_required", message: "If this was fixed deliberately, remove the known_defects entry from this benchmark case and update the referenced documentation. A known-defect record must never outlive the defect it describes." }, + ], + unexpected_defect_resolution: true, + }))); +} diff --git a/evals/graders/deterministicGraders.test.js b/evals/graders/deterministicGraders.test.js new file mode 100644 index 0000000..9e57685 --- /dev/null +++ b/evals/graders/deterministicGraders.test.js @@ -0,0 +1,662 @@ +/** + * Deterministic grader tests. + * + * Every grader is proven twice: once that it passes a genuinely good result, + * and once that it catches a specific, realistic defect. A grader that has + * only ever been seen to pass is not evidence of anything. + * + * Good results come from executing the real pipeline with the offline fixture + * provider, so the "pass" side is never a hand-built object that happens to + * satisfy the grader. + */ +import { describe, it, expect, beforeAll } from "vitest"; + +import { loadBenchmark } from "../datasets/loadBenchmark.js"; +import { createEvalFakeProvider } from "../fixtures/fakeProviderProfiles.js"; +import { runCase } from "../runners/runCase.js"; +import { + EXECUTION_GRADERS, + CASE_GRADERS, + ALL_GRADERS, + runGraders, + countFailures, + applyKnownDefects, + checkKnownDefectsStillReproduce, + graderVersions, +} from "./deterministicGraders.js"; + +const graderById = new Map(ALL_GRADERS.map((grader) => [grader.id, grader])); + +let benchmark; +let caseById; + +beforeAll(async () => { + benchmark = await loadBenchmark(); + caseById = new Map(benchmark.cases.map((entry) => [entry.case_id, entry])); +}); + +/** Executes a case with the fixture provider and returns a grading context. */ +async function contextFor(caseId, { profile } = {}) { + const benchmarkCase = caseById.get(caseId); + const result = await runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile }), + }); + const execution = result.executions[0]; + return { + benchmarkCase, + caseResult: result, + execution, + response: structuredClone(execution.response), + // The trace is not carried on the execution record, so it is rebuilt here + // from the case's own expectations for the graders that consume it. + trace: { + requestedCandidateIds: benchmarkCase.input.candidates.map((candidate) => candidate.id), + requestedPairKeys: benchmarkCase.deterministic_expectations.expected_pair_count + ? buildExpectedPairKeys(benchmarkCase) + : null, + }, + stageSnapshots: [], + repetitions: 1, + }; +} + +function buildExpectedPairKeys(benchmarkCase) { + const ids = benchmarkCase.input.candidates.slice(0, 4).map((candidate) => candidate.id); + const keys = []; + for (let i = 0; i < ids.length; i += 1) { + for (let j = i + 1; j < ids.length; j += 1) keys.push([ids[i], ids[j]].sort().join("::")); + } + return keys; +} + +const run = (graderId, context) => graderById.get(graderId).run(context); + +describe("grader registry", () => { + it("registers every grader exactly once with a version", () => { + const ids = ALL_GRADERS.map((grader) => grader.id); + expect(new Set(ids).size).toBe(ids.length); + for (const grader of ALL_GRADERS) { + expect(grader.version, grader.id).toMatch(/^\d+\.\d+\.\d+$/); + expect(["required", "advisory"], grader.id).toContain(grader.severity); + } + }); + + it("exposes grader versions for the run manifest", () => { + expect(Object.keys(graderVersions())).toHaveLength(ALL_GRADERS.length); + }); + + it("converts a throwing grader into an error result instead of losing the run", () => { + const results = runGraders( + [{ id: "boom", version: "1.0.0", severity: "required", run: () => { throw new Error("kaboom"); } }], + {}, + ); + expect(results[0].status).toBe("error"); + expect(results[0].details[0]).toContain("kaboom"); + }); + + it("counts only required failures and errors toward the gate", () => { + const counts = countFailures([ + { status: "fail", severity: "required" }, + { status: "error", severity: "required" }, + { status: "fail", severity: "advisory" }, + { status: "pass", severity: "required" }, + { status: "skip", severity: "required" }, + { status: "expected_failure", severity: "required" }, + ]); + expect(counts).toEqual({ required: 2, advisory: 1 }); + }); +}); + +describe("contract-validity", () => { + it("passes a real, well-formed response", async () => { + const context = await contextFor("case-007"); + expect(run("contract-validity", context).status).toBe("pass"); + }); + + it("catches a response that violates the public contract", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].overall_confidence = 5; + const result = run("contract-validity", context); + expect(result.status).toBe("fail"); + expect(result.details.join(" ")).toContain("overall_confidence"); + }); + + it("catches a non-finite number that a schema would accept", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].weighted_fit_score = Number.NaN; + expect(run("contract-validity", context).details.join(" ")).toContain("non-finite"); + }); + + it("catches a malformed stage event", async () => { + const context = await contextFor("case-007"); + context.stageSnapshots = [[{ id: "input", label: "Input", status: "not-a-status" }]]; + expect(run("contract-validity", context).status).toBe("fail"); + }); +}); + +describe("candidate-coverage", () => { + it("passes complete coverage", async () => { + const context = await contextFor("case-007"); + expect(run("candidate-coverage", context).status).toBe("pass"); + }); + + it("catches a missing candidate", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations.pop(); + const result = run("candidate-coverage", context); + expect(result.status).toBe("fail"); + expect(result.details.join(" ")).toContain("is missing from candidate_evaluations"); + }); + + it("catches an unknown candidate", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].candidate_id = "ghost"; + expect(run("candidate-coverage", context).details.join(" ")).toContain('unknown candidate "ghost"'); + }); + + it("catches a duplicated candidate", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[1].candidate_id = + context.response.candidate_evaluations[0].candidate_id; + expect(run("candidate-coverage", context).details.join(" ")).toContain("appears 2 times"); + }); + + it("catches non-contiguous ranks", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[2].rank = 9; + expect(run("candidate-coverage", context).details.join(" ")).toContain("contiguous"); + }); + + it("catches a scoring stage that requested the wrong candidate set", async () => { + const context = await contextFor("case-007"); + context.trace.requestedCandidateIds = ["dagny-holloway"]; + expect(run("candidate-coverage", context).details.join(" ")).toContain("scoring stage requested"); + }); + + it("accepts duplicate display names that remain distinct by ID", async () => { + const context = await contextFor("case-015"); + expect(run("candidate-coverage", context).status).toBe("pass"); + }); +}); + +describe("ranking-consistency", () => { + it("passes a correctly ordered ranking", async () => { + const context = await contextFor("case-007"); + expect(run("ranking-consistency", context).status).toBe("pass"); + }); + + it("catches a ranking that disagrees with the deterministic score", async () => { + const context = await contextFor("case-007"); + const [first, second] = context.response.candidate_evaluations; + [first.rank, second.rank] = [second.rank, first.rank]; + const result = run("ranking-consistency", context); + expect(result.status).toBe("fail"); + expect(result.details.join(" ")).toContain("outscores rank"); + }); + + it("catches a winner that is not rank 1", async () => { + const context = await contextFor("case-007"); + const other = context.response.candidate_evaluations.find((c) => c.rank !== 1); + context.response.decision_result.recommended_candidate_id = other.candidate_id; + context.response.decision_result.recommended_candidate_name = other.candidate_name; + expect(run("ranking-consistency", context).details.join(" ")).toContain("but rank 1 is"); + }); + + it("catches a winner name that disagrees with the winner ID", async () => { + const context = await contextFor("case-007"); + context.response.decision_result.recommended_candidate_name = "Someone Else"; + expect(run("ranking-consistency", context).details.join(" ")).toContain("but rank 1 is"); + }); + + it("accepts a tie resolved by submission order, the documented behaviour", async () => { + const context = await contextFor("case-007"); + const ranked = [...context.response.candidate_evaluations].sort((a, b) => a.rank - b.rank); + // Force an exact tie between rank 1 and rank 2, keeping submission order. + ranked[1].weighted_fit_score = ranked[0].weighted_fit_score; + const submission = context.benchmarkCase.input.candidates.map((c) => c.id); + if (submission.indexOf(ranked[0].candidate_id) < submission.indexOf(ranked[1].candidate_id)) { + expect(run("ranking-consistency", context).status).toBe("pass"); + } + }); + + it("catches a tie resolved against submission order", async () => { + const context = await contextFor("case-007"); + const ranked = [...context.response.candidate_evaluations].sort((a, b) => a.rank - b.rank); + ranked[1].weighted_fit_score = ranked[0].weighted_fit_score; + // Swap identities so the later-submitted candidate holds the better rank. + const submission = context.benchmarkCase.input.candidates.map((c) => c.id); + const firstPos = submission.indexOf(ranked[0].candidate_id); + const secondPos = submission.indexOf(ranked[1].candidate_id); + if (firstPos < secondPos) { + [ranked[0].candidate_id, ranked[1].candidate_id] = [ranked[1].candidate_id, ranked[0].candidate_id]; + context.response.decision_result.recommended_candidate_id = ranked[0].candidate_id; + expect(run("ranking-consistency", context).details.join(" ")).toContain("submission order"); + } + }); +}); + +describe("score-integrity", () => { + it("passes real pipeline output", async () => { + const context = await contextFor("case-007"); + expect(run("score-integrity", context).status).toBe("pass"); + }); + + it("catches an out-of-range criterion score", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].criteria_scores.domain_expertise.score = 42; + expect(run("score-integrity", context).details.join(" ")).toContain("outside 1-10"); + }); + + it("catches an out-of-range confidence", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].criteria_scores.domain_expertise.confidence = 4; + expect(run("score-integrity", context).details.join(" ")).toContain("outside 0-1"); + }); + + it("catches a deterministic value that does not survive recomputation", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].risk_profile.execution_risk = 0.999; + const result = run("score-integrity", context); + expect(result.status).toBe("fail"); + expect(result.details.join(" ")).toContain("recomputation gives"); + }); + + it("catches a model-authored value replacing a deterministic outcome score", async () => { + const context = await contextFor("case-007"); + context.response.candidate_evaluations[0].expected_outcome_score = 99; + expect(run("score-integrity", context).details.join(" ")).toContain("expected_outcome_score"); + }); +}); + +describe("pairing-integrity", () => { + it("passes complete, canonical pair coverage", async () => { + const context = await contextFor("case-015"); + expect(run("pairing-integrity", context).status).toBe("pass"); + }); + + it("passes a pairing-disabled case with no pairing result", async () => { + const context = await contextFor("case-007"); + expect(run("pairing-integrity", context).status).toBe("pass"); + }); + + it("catches a fabricated pair result when pairing is disabled", async () => { + const context = await contextFor("case-007"); + context.response.pairing_result = { status: "ok", best_pair: {}, top_pairs: [] }; + expect(run("pairing-integrity", context).status).toBe("fail"); + }); + + it("catches an incomplete requested pair set", async () => { + const context = await contextFor("case-015"); + context.trace.requestedPairKeys = context.trace.requestedPairKeys.slice(0, 3); + expect(run("pairing-integrity", context).details.join(" ")).toContain("requested 3 unique pair(s)"); + }); + + it("catches a reversed duplicate pair", async () => { + const context = await contextFor("case-015"); + const pairs = context.response.pairing_result.top_pairs; + pairs[1].candidate_id_a = pairs[0].candidate_id_b; + pairs[1].candidate_id_b = pairs[0].candidate_id_a; + pairs[1].pair = [pairs[0].pair[1], pairs[0].pair[0]]; + expect(run("pairing-integrity", context).details.join(" ")).toContain("duplicate or reversed duplicate"); + }); + + it("catches a best pair missing from top_pairs", async () => { + const context = await contextFor("case-015"); + context.response.pairing_result.top_pairs = context.response.pairing_result.top_pairs.slice(1); + expect(run("pairing-integrity", context).details.join(" ")).toContain("does not appear in top_pairs"); + }); + + it("catches a pair display name that disagrees with its candidate ID", async () => { + const context = await contextFor("case-015"); + context.response.pairing_result.top_pairs[0].pair[0] = "Wrong Name"; + expect(run("pairing-integrity", context).details.join(" ")).toContain("but that ID belongs to"); + }); + + it("catches an unexpected best pair", async () => { + const context = await contextFor("case-016"); + context.benchmarkCase = { + ...context.benchmarkCase, + deterministic_expectations: { + ...context.benchmarkCase.deterministic_expectations, + expected_best_pair_ids: ["giselle-varga", "isolde-marchetti"], + }, + }; + expect(run("pairing-integrity", context).details.join(" ")).toContain("the case expects"); + }); + + it("fails when pairing reports unavailable for a case expecting coverage", async () => { + const context = await contextFor("case-015"); + context.response.pairing_result = { + status: "unavailable", + reason: "Complete pair analysis was unavailable.", + best_pair: null, + top_pairs: [], + }; + expect(run("pairing-integrity", context).status).toBe("fail"); + }); +}); + +describe("pipeline-accounting", () => { + it("passes 3 logical stages without pairing", async () => { + const context = await contextFor("case-007"); + expect(context.response.run_metadata.logicalProviderStageCount).toBe(3); + expect(run("pipeline-accounting", context).status).toBe("pass"); + }); + + it("passes 4 logical stages with pairing", async () => { + const context = await contextFor("case-015"); + expect(context.response.run_metadata.logicalProviderStageCount).toBe(4); + expect(run("pipeline-accounting", context).status).toBe("pass"); + }); + + it("catches a stage count that disagrees with the case", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.logicalProviderStageCount = 4; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("this case requires 3"); + }); + + it("catches per-stage attempts that do not sum to the total", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.providerAttemptCount = 99; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("attempts sum to"); + }); + + it("catches reasoning tokens exceeding output tokens", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.outputTokens = 10; + context.response.run_metadata.reasoningTokens = 50; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("reasoning tokens are a subset"); + }); + + it("catches cached input tokens exceeding input tokens", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.inputTokens = 5; + context.response.run_metadata.cachedInputTokens = 50; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("cachedInputTokens"); + }); + + it("catches a cost estimated for a run that reported no tokens", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.estimatedCostUsd = 1.5; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("no tokens"); + }); + + it("catches attempts beyond the case's ceiling", async () => { + const context = await contextFor("case-007"); + context.response.run_metadata.attempts = { context: 40 }; + context.response.run_metadata.providerAttemptCount = 40; + expect(run("pipeline-accounting", context).details.join(" ")).toContain("exceeds this case's maximum"); + }); +}); + +describe("not-measured-fields", () => { + it("passes real pipeline output", async () => { + const context = await contextFor("case-007"); + expect(run("not-measured-fields", context).status).toBe("pass"); + }); + + it("catches a fabricated cross-scenario consistency value", async () => { + const context = await contextFor("case-007"); + context.response.outcome_models[0].cross_scenario_consistency = 75; + expect(run("not-measured-fields", context).details.join(" ")).toContain("cross_scenario_consistency"); + }); + + it("catches a fabricated best_scenario claim", async () => { + const context = await contextFor("case-007"); + context.response.adaptability_profiles[0].best_scenario = "Rapid crisis/pivot scenario"; + expect(run("not-measured-fields", context).details.join(" ")).toContain("best_scenario"); + }); +}); + +describe("winner-expectation", () => { + it("passes an allowed winner", async () => { + const context = await contextFor("case-007"); + expect(run("winner-expectation", context).status).toBe("pass"); + }); + + it("catches a winner outside the allowed set", async () => { + const context = await contextFor("case-007"); + context.response.decision_result.recommended_candidate_id = "farrah-lindgren"; + expect(run("winner-expectation", context).details.join(" ")).toContain("not among the allowed winners"); + }); + + it("catches a forbidden winner", async () => { + const context = await contextFor("case-007"); + context.response.decision_result.recommended_candidate_id = "farrah-lindgren"; + expect(run("winner-expectation", context).details.join(" ")).toContain("forbidden winner"); + }); + + it("skips a case that deliberately makes no winner claim", async () => { + const context = await contextFor("case-008"); + expect(run("winner-expectation", context).status).toBe("skip"); + }); +}); + +describe("unsupported-claims", () => { + it("passes the fixture's honest narrative", async () => { + const context = await contextFor("case-007"); + expect(run("unsupported-claims", context).status).toBe("pass"); + }); + + it("catches a fairness claim", async () => { + const context = await contextFor("case-007"); + context.response.executive_summary.reason = "This is an unbiased assessment."; + expect(run("unsupported-claims", context).details.join(" ")).toContain("fairness"); + }); + + it("catches a calibration claim", async () => { + const context = await contextFor("case-007"); + context.response.decision_result.key_reason = "Backed by calibrated confidence across criteria."; + expect(run("unsupported-claims", context).details.join(" ")).toContain("calibrated"); + }); + + it("catches an empirical-validation claim", async () => { + const context = await contextFor("case-007"); + context.response.executive_summary.reason = "This ranking is statistically significant."; + expect(run("unsupported-claims", context).status).toBe("fail"); + }); + + it("catches a stability claim from a single repetition", async () => { + const context = await contextFor("case-007"); + context.response.executive_summary.adaptability = "Results are stable across runs."; + expect(run("unsupported-claims", context).details.join(" ")).toContain("single repetition"); + }); + + it("catches a cross-scenario claim while the field is not_measured", async () => { + const context = await contextFor("case-007"); + context.response.executive_summary.adaptability = + "Cross-scenario consistency is strong for this candidate."; + expect(run("unsupported-claims", context).details.join(" ")).toContain("not_measured"); + }); + + it("does not flag the pipeline's own honest not-measured wording", async () => { + const context = await contextFor("case-007"); + const note = context.response.adaptability_profiles[0].resilience_note; + expect(note).toContain("has not been measured"); + expect(run("unsupported-claims", context).status).toBe("pass"); + }); + + it("catches a narrative that recommends a different candidate", async () => { + const context = await contextFor("case-007", { profile: "contradictory-explanation" }); + const result = run("unsupported-claims", context); + expect(result.status).toBe("fail"); + expect(result.details.join(" ")).toContain("other than the ranked winner"); + }); + + it("declines the name-based contradiction check when the winner's name is shared", async () => { + const context = await contextFor("case-015"); + const result = run("unsupported-claims", context); + expect(result.status).toBe("pass"); + expect(result.summary).toContain("display name is shared"); + }); +}); + +describe("uncertainty-acknowledgement", () => { + it("passes when thin-evidence candidates are flagged for human review", async () => { + const context = await contextFor("case-008"); + expect(run("uncertainty-acknowledgement", context).status).toBe("pass"); + }); + + it("catches a thin-evidence candidate that was not flagged", async () => { + const context = await contextFor("case-008"); + for (const review of context.response.confidence_evidence_reviews) { + review.recommend_human_review = false; + } + expect(run("uncertainty-acknowledgement", context).details.join(" ")).toContain("not flagged"); + }); + + it("skips a case that makes no uncertainty claim", async () => { + const context = await contextFor("case-007"); + expect(run("uncertainty-acknowledgement", context).status).toBe("skip"); + }); +}); + +describe("scenario-coverage", () => { + it("passes a multi-scenario case in which every scenario ran", async () => { + const benchmarkCase = caseById.get("case-004"); + const result = await runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex }), + }); + const grader = CASE_GRADERS.find((entry) => entry.id === "scenario-coverage"); + expect( + grader.run({ benchmarkCase, executions: result.executions, repetitions: 1 }).status, + ).toBe("pass"); + }); + + it("catches a silently ignored scenario", async () => { + const benchmarkCase = caseById.get("case-004"); + const result = await runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex }), + }); + const grader = CASE_GRADERS.find((entry) => entry.id === "scenario-coverage"); + const outcome = grader.run({ + benchmarkCase, + executions: result.executions.slice(0, 1), + repetitions: 1, + }); + expect(outcome.status).toBe("fail"); + expect(outcome.details.join(" ")).toContain("produced 0 execution(s)"); + }); + + it("catches a response whose scenario does not match the submitted one", async () => { + const benchmarkCase = caseById.get("case-004"); + const result = await runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex }), + }); + const executions = structuredClone(result.executions); + executions[0].response.decision_result.scenario = "A scenario nobody submitted."; + const grader = CASE_GRADERS.find((entry) => entry.id === "scenario-coverage"); + expect(grader.run({ benchmarkCase, executions, repetitions: 1 }).details.join(" ")).toContain( + "decision_result.scenario does not match", + ); + }); +}); + +describe("known-defect handling", () => { + const defect = { + defect_id: "SR-TEST-001", + title: "A structured test defect with a scoped contract observation.", + case_id: "case-001", + execution_scope: { execution_id: "case-001#s1#r1", scenario_id: "scenario-2", scenario_index: 1, variant_id: null, repetition: 1 }, + expected_observations: [{ + grader_id: "contract-validity", + signature: { kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow" }, + }], + summary: "A documented, pre-existing defect used only by this test suite.", + reference: "docs/evaluation/BENCHMARK_V1.md", + }; + + it("downgrades a matching failure to expected_failure", () => { + const results = applyKnownDefects( + [{ grader_id: "contract-validity", severity: "required", status: "fail", summary: "broken", details: ["negative"], finding_codes: ["negative-risk-adjusted-score"], observations: [{ kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow" }], findings: [{ kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow", message: "negative" }] }], + [defect], + { execution: { execution_id: "case-001#s1#r1", scenario_index: 1, repetition: 1 }, benchmarkCase: { case_id: "case-001", variant_kind: null } }, + ); + expect(results[0].status).toBe("expected_failure"); + expect(results[0].summary).toContain("SR-TEST-001"); + }); + + it("leaves unrelated graders untouched", () => { + const results = applyKnownDefects( + [{ grader_id: "ranking-consistency", severity: "required", status: "fail", summary: "x", details: [] }], + [defect], + { execution: { execution_id: "case-001#s1#r1", scenario_index: 1, repetition: 1 }, benchmarkCase: { case_id: "case-001", variant_kind: null } }, + ); + expect(results[0].status).toBe("fail"); + }); + + it("stops an expected failure from gating the exit status", () => { + const results = applyKnownDefects( + [{ grader_id: "contract-validity", severity: "required", status: "fail", summary: "x", details: ["negative"], finding_codes: ["negative-risk-adjusted-score"], observations: [{ kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow" }], findings: [{ kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow", message: "negative" }] }], + [defect], + { execution: { execution_id: "case-001#s1#r1", scenario_index: 1, repetition: 1 }, benchmarkCase: { case_id: "case-001", variant_kind: null } }, + ); + expect(countFailures(results).required).toBe(0); + }); + + it("raises a required failure when a known defect stops reproducing", () => { + const executions = [ + { execution_id: "case-001#s1#r1", scenario_index: 1, grader_results: [{ grader_id: "contract-validity", severity: "required", status: "pass", summary: "ok", details: [] }] }, + ]; + const resurrections = checkKnownDefectsStillReproduce(executions, [], [defect]); + expect(resurrections).toHaveLength(1); + expect(resurrections[0].severity).toBe("required"); + expect(resurrections[0].summary).toContain("no longer reproduces"); + }); + + it("does not raise an alarm when the defect reproduces in any execution of the case", () => { + const executions = [ + { execution_id: "case-001#s0#r1", scenario_index: 0, grader_results: [{ grader_id: "contract-validity", severity: "required", status: "pass", summary: "ok", details: [] }] }, + { execution_id: "case-001#s1#r1", scenario_index: 1, grader_results: [{ grader_id: "contract-validity", known_defect_id: "SR-TEST-001", known_defect_observation_ids: ["SR-TEST-001:case-001#s1#r1:contract-validity:0"], severity: "required", status: "expected_failure", summary: "known", details: [] }] }, + ]; + expect(checkKnownDefectsStillReproduce(executions, [], [defect])).toHaveLength(0); + }); + + it("does not suppress a different failure from the same grader", () => { + const results = applyKnownDefects( + [{ grader_id: "contract-validity", severity: "required", status: "fail", summary: "different schema break", details: [], finding_codes: ["different-finding"], observations: [{ kind: "schema_issue", path_pattern: "other", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow" }] }], + [defect], + { execution: { execution_id: "case-001#s1#r1", scenario_index: 1, repetition: 1 }, benchmarkCase: { case_id: "case-001", variant_kind: null } }, + ); + expect(results[0].status).toBe("fail"); + }); + + it("does not suppress the defect outside its declared scenario", () => { + const results = applyKnownDefects( + [{ grader_id: "contract-validity", severity: "required", status: "fail", summary: "negative score", details: [], finding_codes: ["negative-risk-adjusted-score"], observations: [{ kind: "schema_issue", path_pattern: "candidate_evaluations.*.risk_adjusted_score", code: "too_small", minimum: 0, subject_candidate_id: "priya-tallow" }] }], + [defect], + { execution: { execution_id: "case-001#s0#r1", scenario_index: 0, repetition: 1 }, benchmarkCase: { case_id: "case-001", variant_kind: null } }, + ); + expect(results[0].status).toBe("fail"); + }); +}); + +describe("grader coverage of the documented checklist", () => { + it("implements a grader for every category Phase 3A committed to", () => { + const ids = new Set(EXECUTION_GRADERS.concat(CASE_GRADERS).map((grader) => grader.id)); + for (const required of [ + "contract-validity", + "candidate-coverage", + "scenario-coverage", + "ranking-consistency", + "score-integrity", + "pairing-integrity", + "pipeline-accounting", + "unsupported-claims", + ]) { + expect(ids, required).toContain(required); + } + }); +}); diff --git a/evals/graders/humanReview.js b/evals/graders/humanReview.js new file mode 100644 index 0000000..146933a --- /dev/null +++ b/evals/graders/humanReview.js @@ -0,0 +1,115 @@ +/** + * @file Human-review parsing and aggregation (Phase 3A evaluation harness). + * + * A completed review is read back with the same strictness as any other + * artifact, and aggregated in a way that deliberately refuses to hide the + * detail: per-dimension statistics are always produced, and the single + * convenience mean is `null` unless enough dimensions were actually scored to + * mean anything. + * + * `not_applicable` and `cannot_determine` are never silently coerced to a + * number. Counting "the reviewer could not tell" as a mid-range score would + * manufacture data that was explicitly declined. + */ +import { + humanReviewTemplateSchema, + humanReviewAggregateSchema, +} from "../schemas/evaluationReport.js"; + +/** Below this many scored dimensions, a single aggregate number is noise. */ +export const MINIMUM_SCORED_DIMENSIONS_FOR_AGGREGATE = 5; + +export const AGGREGATE_CAVEAT = + "This aggregate is a convenience only. It averages independent qualitative judgments that are not calibrated and not inter-rater validated, and it must never be reported without the per-dimension scores it was derived from."; + +/** + * @param {unknown} value a filled-in human-review template + * @returns {object} validated review + */ +export function parseHumanReview(value) { + const result = humanReviewTemplateSchema.safeParse(value); + if (!result.success) { + throw new Error( + `Human review file is not valid:\n${result.error.issues + .map((issue) => ` - ${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("\n")}`, + ); + } + return result.data; +} + +/** + * Aggregates a completed review, retaining dimension-level detail. + * @param {object} review validated human review + * @returns {object} matching `humanReviewAggregateSchema` + */ +export function aggregateHumanReview(review) { + const dimensionScores = {}; + let scoredEntries = 0; + + for (const entry of review.entries) { + let entryHasScore = false; + for (const dimension of entry.dimensions) { + if (!dimensionScores[dimension.dimension_id]) { + dimensionScores[dimension.dimension_id] = { + scored_count: 0, + not_applicable_count: 0, + cannot_determine_count: 0, + values: [], + }; + } + const bucket = dimensionScores[dimension.dimension_id]; + if (dimension.score === "not_applicable") { + bucket.not_applicable_count += 1; + } else if (dimension.score === "cannot_determine") { + bucket.cannot_determine_count += 1; + } else if (typeof dimension.score === "number") { + bucket.scored_count += 1; + bucket.values.push(dimension.score); + entryHasScore = true; + } + } + if (entryHasScore) scoredEntries += 1; + } + + const finalised = {}; + const allValues = []; + for (const [dimensionId, bucket] of Object.entries(dimensionScores)) { + allValues.push(...bucket.values); + finalised[dimensionId] = { + scored_count: bucket.scored_count, + not_applicable_count: bucket.not_applicable_count, + cannot_determine_count: bucket.cannot_determine_count, + mean: + bucket.values.length > 0 + ? Number((bucket.values.reduce((a, b) => a + b, 0) / bucket.values.length).toFixed(4)) + : null, + min: bucket.values.length > 0 ? Math.min(...bucket.values) : null, + max: bucket.values.length > 0 ? Math.max(...bucket.values) : null, + }; + } + + const aggregate = { + scored_entries: scoredEntries, + dimension_scores: finalised, + aggregate_mean: + allValues.length >= MINIMUM_SCORED_DIMENSIONS_FOR_AGGREGATE + ? Number((allValues.reduce((a, b) => a + b, 0) / allValues.length).toFixed(4)) + : null, + aggregate_caveat: AGGREGATE_CAVEAT, + }; + + return humanReviewAggregateSchema.parse(aggregate); +} + +/** + * True when a review file carries at least one real score. Used by the + * comparison command to decide whether rubric comparison is possible at all, + * rather than comparing two empty templates and calling the result unchanged. + * @param {object} review + */ +export function hasAnyScores(review) { + return review.entries.some((entry) => + entry.dimensions.some((dimension) => typeof dimension.score === "number"), + ); +} diff --git a/evals/graders/humanReview.test.js b/evals/graders/humanReview.test.js new file mode 100644 index 0000000..4b6783b --- /dev/null +++ b/evals/graders/humanReview.test.js @@ -0,0 +1,224 @@ +/** + * Human-review template and aggregation tests. + * + * The behaviour these lock down is mostly about what the harness refuses to + * do: coerce a declined judgment into a number, or collapse eight dimensions + * into one figure that hides which one was weak. + */ +import { describe, it, expect, beforeAll } from "vitest"; + +import { loadBenchmark } from "../datasets/loadBenchmark.js"; +import { createEvalFakeProvider } from "../fixtures/fakeProviderProfiles.js"; +import { runCase } from "../runners/runCase.js"; +import { buildHumanReviewTemplate, SCALE_LEGEND, REVIEW_INSTRUCTIONS } from "./rubricTemplate.js"; +import { + parseHumanReview, + aggregateHumanReview, + hasAnyScores, + MINIMUM_SCORED_DIMENSIONS_FOR_AGGREGATE, + AGGREGATE_CAVEAT, +} from "./humanReview.js"; + +let benchmark; +let caseById; + +beforeAll(async () => { + benchmark = await loadBenchmark(); + caseById = new Map(benchmark.cases.map((entry) => [entry.case_id, entry])); +}); + +async function templateFor(caseIds, { profile } = {}) { + const caseResults = []; + for (const caseId of caseIds) { + const benchmarkCase = caseById.get(caseId); + caseResults.push( + await runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile }), + }), + ); + } + return buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: { + run_id: "run-test", + benchmark_id: benchmark.manifest.benchmark_id, + benchmark_version: benchmark.manifest.benchmark_version, + rubric_version: benchmark.rubric.rubric_version, + }, + caseResults, + casesById: caseById, + }); +} + +describe("human-review template", () => { + it("creates one entry per completed execution", async () => { + const template = await templateFor(["case-004"]); + expect(template.entries).toHaveLength(2); + expect(template.entries.map((entry) => entry.scenario_index)).toEqual([0, 1]); + }); + + it("omits failed executions, which have no explanation to review", async () => { + const template = await templateFor(["case-007"], { profile: "unknown-candidate" }); + expect(template.entries).toHaveLength(0); + }); + + it("carries each dimension's anchors into the template", async () => { + const template = await templateFor(["case-007"]); + for (const dimension of template.entries[0].dimensions) { + expect(Object.keys(dimension.anchors).sort()).toEqual(["0", "1", "2", "3", "4"]); + expect(dimension.what_is_judged.length).toBeGreaterThan(20); + } + }); + + it("asks only for the dimensions a case declares", async () => { + const plain = await templateFor(["case-007"]); + const pairing = await templateFor(["case-015"]); + const dimensionIds = (template) => template.entries[0].dimensions.map((d) => d.dimension_id); + expect(dimensionIds(plain)).not.toContain("pairing_usefulness"); + expect(dimensionIds(pairing)).toContain("pairing_usefulness"); + }); + + it("leaves every score unset for the reviewer to fill in", async () => { + const template = await templateFor(["case-007"]); + for (const dimension of template.entries[0].dimensions) { + expect(dimension.score).toBeNull(); + expect(dimension.reviewer_notes).toBe(""); + } + expect(template.entries[0].reviewer).toBe(""); + }); + + it("includes the anchored scale legend and both non-scores", async () => { + const template = await templateFor(["case-007"]); + expect(template.scale_legend).toEqual(SCALE_LEGEND); + expect(template.scale_legend.not_applicable).toBeTruthy(); + expect(template.scale_legend.cannot_determine).toBeTruthy(); + }); + + it("states that the scores are opinion, not measurement", async () => { + const template = await templateFor(["case-007"]); + expect(template.instructions).toBe(REVIEW_INSTRUCTIONS); + expect(template.instructions).toContain("not calibrated"); + expect(template.instructions).toContain("structured opinion"); + }); + + it("validates as a human-review template", async () => { + const template = await templateFor(["case-007"]); + expect(() => parseHumanReview(template)).not.toThrow(); + }); +}); + +function review(dimensionsPerEntry) { + return parseHumanReview({ + schema_version: "1.0.0", + run_id: "r", + benchmark_id: "decision-benchmark-v1", + benchmark_version: "1.0.0", + rubric_version: "1.0.0", + instructions: "x".repeat(50), + scale_legend: { 0: "unacceptable" }, + entries: dimensionsPerEntry.map((dimensions, index) => ({ + execution_id: `e${index}`, + case_id: "case-001", + scenario_index: 0, + repetition: 1, + reviewer: "reviewer", + reviewed_at: "2026-08-02", + dimensions: Object.entries(dimensions).map(([id, score]) => ({ + dimension_id: id, + label: id, + what_is_judged: "judged", + anchors: { 0: "bad" }, + score, + reviewer_notes: "", + })), + overall_notes: "", + })), + }); +} + +describe("human-review aggregation", () => { + it("retains per-dimension statistics", () => { + const aggregate = aggregateHumanReview(review([{ clarity: 2 }, { clarity: 4 }])); + expect(aggregate.dimension_scores.clarity).toMatchObject({ + scored_count: 2, + mean: 3, + min: 2, + max: 4, + }); + }); + + it("never coerces not_applicable or cannot_determine into a number", () => { + const aggregate = aggregateHumanReview( + review([{ clarity: "not_applicable" }, { clarity: "cannot_determine" }, { clarity: 4 }]), + ); + expect(aggregate.dimension_scores.clarity).toMatchObject({ + scored_count: 1, + not_applicable_count: 1, + cannot_determine_count: 1, + mean: 4, + }); + }); + + it("reports a null mean for a dimension nobody scored", () => { + const aggregate = aggregateHumanReview(review([{ clarity: "not_applicable" }])); + expect(aggregate.dimension_scores.clarity.mean).toBeNull(); + expect(aggregate.dimension_scores.clarity.min).toBeNull(); + }); + + it("withholds the convenience aggregate below the minimum sample", () => { + const aggregate = aggregateHumanReview(review([{ clarity: 3 }])); + expect(aggregate.aggregate_mean).toBeNull(); + }); + + it("produces the convenience aggregate once enough dimensions are scored", () => { + const scores = Object.fromEntries( + Array.from({ length: MINIMUM_SCORED_DIMENSIONS_FOR_AGGREGATE }, (_, index) => [`d${index}`, 4]), + ); + const aggregate = aggregateHumanReview(review([scores])); + expect(aggregate.aggregate_mean).toBe(4); + }); + + it("always attaches the caveat to the aggregate", () => { + const aggregate = aggregateHumanReview(review([{ clarity: 3 }])); + expect(aggregate.aggregate_caveat).toBe(AGGREGATE_CAVEAT); + expect(aggregate.aggregate_caveat).toContain("never be reported without the per-dimension scores"); + }); + + it("counts only entries that carry at least one real score", () => { + const aggregate = aggregateHumanReview( + review([{ clarity: 3 }, { clarity: "cannot_determine" }, { clarity: null }]), + ); + expect(aggregate.scored_entries).toBe(1); + }); +}); + +describe("review completeness detection", () => { + it("recognises a blank template as unscored", () => { + expect(hasAnyScores(review([{ clarity: null }]))).toBe(false); + }); + + it("does not treat declined judgments as scores", () => { + expect(hasAnyScores(review([{ clarity: "cannot_determine" }]))).toBe(false); + }); + + it("recognises a single real score", () => { + expect(hasAnyScores(review([{ clarity: 0 }]))).toBe(true); + }); +}); + +describe("review parsing", () => { + it("rejects a score outside the anchored scale", () => { + expect(() => review([{ clarity: 7 }])).toThrow(/not valid/); + }); + + it("rejects an unrecognised non-score", () => { + expect(() => review([{ clarity: "probably_fine" }])).toThrow(/not valid/); + }); + + it("rejects a review with no dimensions on an entry", () => { + expect(() => review([{}])).toThrow(/not valid/); + }); +}); diff --git a/evals/graders/rubricTemplate.js b/evals/graders/rubricTemplate.js new file mode 100644 index 0000000..c718d19 --- /dev/null +++ b/evals/graders/rubricTemplate.js @@ -0,0 +1,80 @@ +/** + * @file Rubric template construction (Phase 3A evaluation harness). + * + * Turns a benchmark's rubric plus a run's executions into the blank + * human-review template a reviewer fills in. Every dimension carries its own + * anchors into the template, so a reviewer never has to hold the rubric open + * in another window and never has to guess what a 2 means. + */ +import { EVALUATION_REPORT_SCHEMA_VERSION } from "../schemas/evaluationReport.js"; + +export const SCALE_LEGEND = Object.freeze({ + 0: "unacceptable", + 1: "major problems", + 2: "mixed", + 3: "good", + 4: "excellent", + not_applicable: "this dimension does not apply to this case", + cannot_determine: "the output does not contain enough information to judge this dimension", +}); + +export const REVIEW_INSTRUCTIONS = + "Score each dimension independently on the 0-4 anchored scale below. Use not_applicable when the dimension genuinely does not apply to the case, and cannot_determine when the output does not give you enough to judge — do not split the difference with a 2. Leave reviewer_notes wherever a score would otherwise be unexplainable to someone else. These scores are structured opinion, not measurement: they are not calibrated, not inter-rater validated, and are not evidence that the system is fair or production-ready."; + +/** + * @param {object} options + * @param {object} options.rubric validated rubric + * @param {object} options.manifest run manifest + * @param {object[]} options.caseResults + * @param {Map} options.casesById benchmark cases, for their + * declared `rubric_dimensions` + * @returns {object} matching `humanReviewTemplateSchema` + */ +export function buildHumanReviewTemplate({ rubric, manifest, caseResults, casesById }) { + const dimensionsById = new Map(rubric.dimensions.map((dimension) => [dimension.id, dimension])); + + const entries = []; + for (const caseResult of caseResults) { + const benchmarkCase = casesById.get(caseResult.case_id); + const requested = benchmarkCase?.rubric_dimensions ?? rubric.dimensions.map((d) => d.id); + + for (const execution of caseResult.executions) { + // A failed execution has no explanation to review. Including a blank + // entry for it would invite a reviewer to score something that does not + // exist. + if (execution.status !== "completed") continue; + + entries.push({ + execution_id: execution.execution_id, + case_id: execution.case_id, + scenario_index: execution.scenario_index, + repetition: execution.repetition, + reviewer: "", + reviewed_at: "", + dimensions: requested + .map((dimensionId) => dimensionsById.get(dimensionId)) + .filter(Boolean) + .map((dimension) => ({ + dimension_id: dimension.id, + label: dimension.label, + what_is_judged: dimension.what_is_judged, + anchors: { ...dimension.anchors }, + score: null, + reviewer_notes: "", + })), + overall_notes: "", + }); + } + } + + return { + schema_version: EVALUATION_REPORT_SCHEMA_VERSION, + run_id: manifest.run_id, + benchmark_id: manifest.benchmark_id, + benchmark_version: manifest.benchmark_version, + rubric_version: manifest.rubric_version, + instructions: REVIEW_INSTRUCTIONS, + scale_legend: { ...SCALE_LEGEND }, + entries, + }; +} diff --git a/evals/reporters/jsonReporter.js b/evals/reporters/jsonReporter.js new file mode 100644 index 0000000..78f4b47 --- /dev/null +++ b/evals/reporters/jsonReporter.js @@ -0,0 +1,112 @@ +/** + * @file JSON run artifacts (Phase 3A evaluation harness). + * + * Writes a run directory under `.eval-runs/` (git-ignored). Every artifact is + * validated against its schema and scanned for policy violations *before* it + * touches the filesystem — writing first and checking later would leave a + * leaked value on disk even if the command then failed. + */ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { + runManifestSchema, + runSummarySchema, + caseResultSchema, + assertArtifactIsPolicyClean, +} from "../schemas/evaluationRun.js"; +import { humanReviewTemplateSchema } from "../schemas/evaluationReport.js"; + +/** Git-ignored. Run output is never committed (docs/evaluation/RUNBOOK.md). */ +export const RUN_ROOT_DIRNAME = ".eval-runs"; + +async function writeChecked(filePath, artifact, label) { + const serialized = typeof artifact === "string" ? artifact : `${JSON.stringify(artifact, null, 2)}\n`; + assertArtifactIsPolicyClean(serialized, label); + await writeFile(filePath, serialized, "utf8"); +} + +/** + * A case result carries full pipeline responses, which are large. They are + * kept — the whole point of an artifact is that a later comparison does not + * have to re-run anything — but each case is written as one JSONL line so a + * reader can stream them. + */ +function toJsonl(caseResults) { + return `${caseResults.map((caseResult) => JSON.stringify(caseResult)).join("\n")}\n`; +} + +/** + * @param {object} options + * @param {object} options.run result of runBenchmark() + * @param {object} options.humanReviewTemplate blank review template + * @param {string} options.markdown rendered summary.md + * @param {string} [options.rootDir] defaults to `/.eval-runs` + * @returns {Promise<{ runDir: string, files: string[] }>} + */ +export async function writeRunArtifacts({ run, humanReviewTemplate, markdown, rootDir }) { + const manifest = runManifestSchema.parse(run.manifest); + const summary = runSummarySchema.parse(run.summary); + const caseResults = run.caseResults.map((caseResult) => caseResultSchema.parse(caseResult)); + const template = humanReviewTemplateSchema.parse(humanReviewTemplate); + + const root = rootDir ?? path.join(process.cwd(), RUN_ROOT_DIRNAME); + const runDir = path.join(root, manifest.run_id); + await mkdir(runDir, { recursive: true }); + + // Permutation findings live beside the summary rather than inside it: they + // are observational, not a pass/fail signal, and mixing them into the + // summary would invite reading them as one. + const files = [ + ["run-manifest.json", manifest], + ["summary.json", summary], + ["permutations.json", { run_id: manifest.run_id, findings: run.permutations }], + ["human-review-template.json", template], + ]; + + for (const [name, artifact] of files) { + await writeChecked(path.join(runDir, name), artifact, name); + } + await writeChecked(path.join(runDir, "case-results.jsonl"), toJsonl(caseResults), "case-results.jsonl"); + await writeChecked(path.join(runDir, "summary.md"), markdown, "summary.md"); + + return { + runDir, + files: [ + "run-manifest.json", + "case-results.jsonl", + "summary.json", + "summary.md", + "permutations.json", + "human-review-template.json", + ], + }; +} + +/** + * Reads a run directory back for comparison. + * @param {string} runDir + */ +export async function readRunArtifacts(runDir) { + const { readFile } = await import("node:fs/promises"); + const readJson = async (name) => JSON.parse(await readFile(path.join(runDir, name), "utf8")); + + const manifest = runManifestSchema.parse(await readJson("run-manifest.json")); + const summary = runSummarySchema.parse(await readJson("summary.json")); + const jsonl = await readFile(path.join(runDir, "case-results.jsonl"), "utf8"); + const caseResults = jsonl + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => caseResultSchema.parse(JSON.parse(line))); + + let humanReview = null; + try { + humanReview = humanReviewTemplateSchema.parse(await readJson("human-review.json")); + } catch { + // A completed review is optional: a run that nobody has reviewed yet is + // the normal case, and the comparison says so rather than inventing one. + humanReview = null; + } + + return { manifest, summary, caseResults, humanReview }; +} diff --git a/evals/reporters/markdownReporter.js b/evals/reporters/markdownReporter.js new file mode 100644 index 0000000..886ea64 --- /dev/null +++ b/evals/reporters/markdownReporter.js @@ -0,0 +1,282 @@ +/** + * @file Markdown run summary (Phase 3A evaluation harness). + * + * Renders a run into prose a human can read without opening JSON. Every claim + * it makes is either a count or an explicit "not assessed" — it never + * describes a result as good, fair, validated, or production-ready. + * + * No ANSI escapes are emitted anywhere: artifacts and CLI output must stay + * greppable and diffable (docs/evaluation/RUNBOOK.md). + */ + +const STATUS_LABEL = { + pass: "pass", + fail: "FAIL", + skip: "skip", + error: "ERROR", + expected_failure: "known defect", +}; + +const RUN_STATUS_LABEL = { + clean_pass: "CLEAN PASS", + pass_with_known_defects: "PASS WITH KNOWN DEFECTS", + unexpected_failure: "UNEXPECTED FAILURE", + baseline_change_required: "BASELINE CHANGE REQUIRED", +}; + +function formatCost(value) { + return value === null ? "unavailable" : `$${value.toFixed(6)}`; +} + +function graderTable(graderTotals) { + const rows = Object.entries(graderTotals) + .sort(([a], [b]) => a.localeCompare(b)) + .map( + ([id, totals]) => + `| \`${id}\` | ${totals.severity} | ${totals.pass} | ${totals.fail} | ${totals.skip} | ${totals.error} | ${totals.expected_failure} |`, + ); + return [ + "| Grader | Severity | Pass | Fail | Skip | Error | Known defect |", + "|---|---|---|---|---|---|---|", + ...rows, + ].join("\n"); +} + +/** + * Known defects are listed separately and prominently. They are excluded from + * the pass/fail gate, so the report has to make sure nobody can read a green + * run as "nothing is wrong". + */ +function knownDefectSection(caseResults) { + const seen = new Map(); + for (const caseResult of caseResults) { + const results = [ + ...caseResult.executions.flatMap((execution) => execution.grader_results), + ...caseResult.grader_results, + ]; + for (const result of results) { + if (result.status !== "expected_failure") continue; + const key = `${result.known_defect_id ?? "unknown"}\u0000${result.grader_id}`; + if (!seen.has(key)) seen.set(key, { defect_id: result.known_defect_id ?? "unknown", grader_id: result.grader_id, cases: new Set(), detail: result.details.at(-1) }); + seen.get(key).cases.add(caseResult.case_id); + } + } + if (seen.size === 0) return ["_No known defects were reproduced in this run._", ""]; + + return [ + "These are documented, pre-existing product defects. They do not gate the exit status, and they are **not** resolved:", + "", + ...[...seen.values()].map( + (entry) => + `- \`${entry.defect_id}\` / \`${entry.grader_id}\` in ${[...entry.cases].sort().join(", ")} — ${entry.detail ?? "see the case's known_defects entry"}`, + ), + "", + ]; +} + +function failureDetail(caseResults) { + const lines = []; + for (const caseResult of caseResults) { + const failing = [ + ...caseResult.executions.flatMap((execution) => + execution.grader_results + .filter((result) => result.status === "fail" || result.status === "error") + .map((result) => ({ where: execution.execution_id, result })), + ), + ...caseResult.grader_results + .filter((result) => result.status === "fail" || result.status === "error") + .map((result) => ({ where: caseResult.case_id, result })), + ]; + if (failing.length === 0) continue; + + lines.push(`### ${caseResult.case_id} — ${caseResult.title}`, ""); + for (const { where, result } of failing) { + lines.push( + `- **${STATUS_LABEL[result.status]}** \`${result.grader_id}\` (${result.severity}) at \`${where}\`: ${result.summary}`, + ); + for (const detail of result.details.slice(0, 8)) lines.push(` - ${detail}`); + if (result.details.length > 8) { + lines.push(` - …and ${result.details.length - 8} more`); + } + } + lines.push(""); + } + return lines; +} + +function permutationSection(permutations) { + if (permutations.length === 0) return ["_No permutation variants were part of this run._", ""]; + const rows = permutations.map((finding) => { + if (!finding.compared) { + return `| \`${finding.case_id}\` | ${finding.variant_kind} | not compared | ${finding.reason} |`; + } + const changed = [ + finding.winner_changed ? "winner" : null, + finding.ranking_changed ? "ranking" : null, + finding.best_pair_changed ? "pair" : null, + finding.structured_evidence_changed ? "evidence" : null, + finding.explanation_changed ? "explanation" : null, + ].filter(Boolean); + return `| \`${finding.case_id}\` | ${finding.variant_kind} | ${changed.length === 0 ? "no change" : changed.join(", ")} | vs \`${finding.variant_of}\` |`; + }); + return [ + "| Variant | Kind | Changed | Notes |", + "|---|---|---|---|", + ...rows, + "", + "Under the offline fixture provider, wording and irrelevant-text variants cannot differ at all: the fixture scores by candidate ID and never reads description text. A \"no change\" result for those kinds validates the comparison machinery, not the model.", + "", + ]; +} + +/** + * @param {object} run result of runBenchmark() + * @returns {string} markdown + */ +export function renderRunMarkdown(run) { + const { manifest, summary, caseResults, permutations } = run; + + const lines = [ + `# Evaluation run \`${manifest.run_id}\``, + "", + `**Run state: ${RUN_STATUS_LABEL[summary.run_state]}**`, + "", + "| Field | Value |", + "|---|---|", + `| Mode | ${manifest.mode} |`, + `| Benchmark | \`${manifest.benchmark_id}\` v${manifest.benchmark_version} (schema ${manifest.benchmark_schema_version}) |`, + `| Rubric | v${manifest.rubric_version} |`, + `| Commit | ${manifest.git_commit ?? "unknown"}${manifest.git_branch ? ` (${manifest.git_branch})` : ""} |`, + `| Provider / model | ${manifest.provider} / ${manifest.model} |`, + `| Cases | ${summary.case_count} completed: ${summary.clean_pass_count} clean cases; ${summary.affected_execution_ids.length} cases containing expected known-defect observations; ${summary.unexpected_failures} unexpected failures |`, + `| Clean cases | ${summary.clean_pass_count} |`, + `| Known-defect observations | ${summary.expected_failures} |`, + `| Affected executions | ${summary.affected_execution_ids.length} |`, + `| Unexpected failures | ${summary.unexpected_failures} |`, + `| Unexpected defect resolutions | ${summary.unexpected_defect_resolutions} |`, + `| Executions | ${summary.execution_count} at ${manifest.repetitions} repetition(s) |`, + `| Pairing cases | ${manifest.pairing_cases} |`, + `| Logical provider stages | ${manifest.logical_provider_stages} |`, + `| Provider attempts | ${manifest.provider_attempts} |`, + `| Tokens (in / out / total) | ${manifest.input_tokens} / ${manifest.output_tokens} / ${manifest.total_tokens} |`, + `| Estimated cost | ${formatCost(manifest.estimated_cost_usd)} |`, + `| Duration | ${manifest.duration_ms} ms |`, + "", + "## Graders", + "", + graderTable(summary.grader_totals), + "", + "## Known defects reproduced", + "", + ...knownDefectSection(caseResults), + "## Run-to-run stability", + "", + summary.stability.assessed + ? `Winner agreement ${summary.stability.winner_agreement}, ranking agreement ${summary.stability.ranking_agreement}. ${summary.stability.reason}` + : summary.stability.reason, + "", + "## Permutation variants", + "", + ...permutationSection(permutations), + "## Cases", + "", + "| Case | Title | Tags | Required failures | Advisory | Result |", + "|---|---|---|---|---|---|", + ...caseResults.map( + (caseResult) => + `| \`${caseResult.case_id}\` | ${caseResult.title} | ${caseResult.tags.join(", ")} | ${caseResult.required_failures} | ${caseResult.advisory_failures} | ${caseResult.passed ? "pass" : "FAIL"} |`, + ), + "", + ]; + + const failures = failureDetail(caseResults); + if (failures.length > 0) { + lines.push("## Failures", "", ...failures); + } + + lines.push("## Scope", "", summary.disclaimer, ""); + return lines.join("\n"); +} + +/** + * Compact, non-ANSI console summary. Kept separate from the artifact renderer + * so CLI output can stay short without the artifact losing detail. + * @param {object} run + */ +export function renderConsoleSummary(run) { + const { manifest, summary } = run; + return [ + `run: ${manifest.run_id}`, + `mode: ${manifest.mode} benchmark: ${manifest.benchmark_id} v${manifest.benchmark_version} commit: ${manifest.git_commit ?? "unknown"}`, + `cases: ${summary.passed_cases}/${summary.case_count} passed executions: ${summary.execution_count} repetitions: ${manifest.repetitions}`, + `required failures: ${summary.required_failures} advisory failures: ${summary.advisory_failures} known defects: ${summary.expected_failures}`, + `stages: ${manifest.logical_provider_stages} attempts: ${manifest.provider_attempts} tokens: ${manifest.total_tokens} cost: ${formatCost(manifest.estimated_cost_usd)} duration: ${manifest.duration_ms}ms`, + `stability: ${summary.stability.assessed ? `winner ${summary.stability.winner_agreement}, ranking ${summary.stability.ranking_agreement}` : "not assessed (single repetition)"}`, + `fixture machinery: ${summary.run_state === "unexpected_failure" || summary.run_state === "baseline_change_required" ? "FAILED" : "PASSED"}`, + `production baseline: ${RUN_STATUS_LABEL[summary.run_state]}`, + `known defect observations: ${summary.expected_failures} across ${summary.affected_execution_ids.length} scoped execution(s)`, + `unexpected failures: ${summary.unexpected_failures}`, + `unexpected defect resolutions: ${summary.unexpected_defect_resolutions}`, + ].join("\n"); +} + +/** + * @param {object} report validated comparison report + * @returns {string} markdown + */ +export function renderComparisonMarkdown(report) { + const delta = (entry) => + entry.delta === null ? "n/a" : `${entry.baseline} → ${entry.candidate} (${entry.delta >= 0 ? "+" : ""}${entry.delta})`; + + const lines = [ + `# Comparison: \`${report.baseline_run_id}\` → \`${report.candidate_run_id}\``, + "", + `**Verdict: ${report.verdict}**`, + "", + ...report.verdict_reasons.map((reason) => `- ${reason}`), + "", + "| Measure | Baseline → Candidate |", + "|---|---|", + `| Required failures | ${delta(report.invariants.required_failures)} |`, + `| Advisory failures | ${delta(report.invariants.advisory_failures)} |`, + `| Expected known-defect observations | ${delta(report.invariants.expected_failures)} |`, + `| Schema failures | ${delta(report.invariants.schema_failures)} |`, + `| Passed cases | ${delta(report.invariants.passed_cases)} |`, + `| Total tokens | ${delta(report.tokens)} |`, + `| Estimated cost | ${delta(report.cost)} |`, + `| Duration (ms) | ${delta(report.duration_ms)} |`, + "", + `Winner changes: ${report.winner_changes.length === 0 ? "none" : report.winner_changes.join(", ")}`, + `Ranking changes: ${report.ranking_changes.length === 0 ? "none" : report.ranking_changes.join(", ")}`, + `Pair changes: ${report.pair_changes.length === 0 ? "none" : report.pair_changes.join(", ")}`, + "", + "## Rubric", + "", + report.rubric.compared + ? Object.entries(report.rubric.dimensions) + .map(([id, entry]) => `- \`${id}\`: ${delta(entry)}`) + .join("\n") + : report.rubric.reason, + "", + "## Stability", + "", + "## Known-defect observation comparison", + "", + `- Unchanged observations: ${report.defect_observations.unchanged.length}`, + `- Disappeared observations: ${report.defect_observations.disappeared.length}`, + `- New observations: ${report.defect_observations.appeared.length}`, + `- Changed signatures: ${report.defect_observations.changed_signature.length}`, + `- Moved observations: ${report.defect_observations.moved.length}`, + `- Count changes: ${report.invariants.expected_failures.delta === null ? "n/a" : report.invariants.expected_failures.delta}`, + "", + report.stability.compared + ? `Baseline winner agreement ${report.stability.baseline_winner_agreement}, candidate ${report.stability.candidate_winner_agreement}.` + : report.stability.reason, + "", + "## Limitations", + "", + ...report.limitations.map((limitation) => `- ${limitation}`), + "", + ]; + return lines.join("\n"); +} diff --git a/evals/repositoryProtection.test.js b/evals/repositoryProtection.test.js new file mode 100644 index 0000000..72c8f2d --- /dev/null +++ b/evals/repositoryProtection.test.js @@ -0,0 +1,318 @@ +/** + * Repository-protection tests. + * + * These guard the boundaries Phase 3A committed to, in the repository itself + * rather than in a module's own logic: + * + * - production code never imports the evaluation harness; + * - run artifacts are git-ignored and never committed; + * - nothing committed under evals/ contains a secret or an absolute path; + * - the CLI commands behave as the runbook promises, including exit status. + */ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import { readFileSync, readdirSync, statSync, existsSync } from "node:fs"; +import path from "node:path"; + +const ROOT = process.cwd(); + +function walk(dir, predicate = () => true) { + const out = []; + for (const entry of readdirSync(dir)) { + if (entry === "node_modules" || entry === ".git" || entry === "dist") continue; + const full = path.join(dir, entry); + if (statSync(full).isDirectory()) out.push(...walk(full, predicate)); + else if (predicate(full)) out.push(full); + } + return out; +} + +const isSource = (file) => /\.(js|mjs|ts|tsx|json)$/.test(file); + +/** Runs a CLI and captures status plus output, without throwing on nonzero. */ +function runCli(args, env = {}) { + try { + const stdout = execFileSync("node", args, { + cwd: ROOT, + encoding: "utf8", + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + }); + return { status: 0, stdout, stderr: "" }; + } catch (error) { + return { + status: error.status ?? 1, + stdout: error.stdout?.toString() ?? "", + stderr: error.stderr?.toString() ?? "", + }; + } +} + +describe("production code does not depend on the evaluation harness", () => { + const productionDirs = ["server", "src", "shared", "scripts"]; + + it("contains no import of evals/ anywhere in production source", () => { + const offenders = []; + for (const dir of productionDirs) { + for (const file of walk(path.join(ROOT, dir), isSource)) { + const contents = readFileSync(file, "utf8"); + if (/(?:from|import\(|require\()\s*["'][^"']*\bevals\//.test(contents)) { + offenders.push(path.relative(ROOT, file)); + } + } + } + expect(offenders).toEqual([]); + }); + + it("contains no import of evals/ in the composition root", () => { + expect(readFileSync(path.join(ROOT, "server.mjs"), "utf8")).not.toMatch(/\bevals\//); + }); + + it("keeps the dependency direction one-way: the harness may import production", () => { + const harnessImportsProduction = walk(path.join(ROOT, "evals"), isSource).some((file) => + /(?:from)\s*["']\.\.\/\.\.\/(?:server|shared)\//.test(readFileSync(file, "utf8")), + ); + expect(harnessImportsProduction).toBe(true); + }); + + it("does not reference the evaluation harness from any HTTP route or frontend component", () => { + for (const file of [ + ...walk(path.join(ROOT, "server", "http"), isSource), + ...walk(path.join(ROOT, "src"), isSource), + ]) { + expect(readFileSync(file, "utf8"), path.relative(ROOT, file)).not.toMatch(/\bevals\//); + } + }); +}); + +describe("run artifacts stay out of git", () => { + const gitignore = readFileSync(path.join(ROOT, ".gitignore"), "utf8"); + + it("ignores the .eval-runs directory", () => { + expect(gitignore).toMatch(/^\.eval-runs\/?$/m); + }); + + it("is confirmed ignored by git itself", () => { + const output = execFileSync("git", ["check-ignore", "-v", ".eval-runs/example/summary.json"], { + cwd: ROOT, + encoding: "utf8", + }); + expect(output).toContain(".eval-runs"); + }); + + it("has no run artifact tracked in the repository", () => { + const tracked = execFileSync("git", ["ls-files"], { cwd: ROOT, encoding: "utf8" }); + expect(tracked).not.toMatch(/^\.eval-runs\//m); + expect(tracked).not.toMatch(/run-manifest\.json/); + expect(tracked).not.toMatch(/case-results\.jsonl/); + }); + + it("still ignores .env files, unchanged by this phase", () => { + expect(gitignore).toMatch(/^\.env\.\*$/m); + expect(gitignore).toMatch(/^!\.env\.example$/m); + }); +}); + +describe("committed evaluation files contain nothing sensitive", () => { + // Test files are deliberately excluded. They contain example key- and + // path-shaped strings *as fixtures*, because the artifact scanner has to be + // proven to detect them. Scanning the detector's own test data would make + // this check impossible to satisfy without weakening the detector's tests. + const isTest = (file) => file.endsWith(".test.js"); + const files = walk( + path.join(ROOT, "evals"), + (file) => (isSource(file) || file.endsWith(".md")) && !isTest(file), + ); + + it("finds a non-trivial number of files to check", () => { + expect(files.length).toBeGreaterThan(20); + }); + + it("contains no secret-shaped string", () => { + for (const file of files) { + const contents = readFileSync(file, "utf8"); + expect(contents, path.relative(ROOT, file)).not.toMatch(/\bsk-[A-Za-z0-9_-]{16,}/); + expect(contents, path.relative(ROOT, file)).not.toMatch(/\bBearer\s+[A-Za-z0-9._-]{20,}/); + } + }); + + it("contains no absolute machine path", () => { + for (const file of files) { + const contents = readFileSync(file, "utf8"); + expect(contents, path.relative(ROOT, file)).not.toMatch(/\/(?:Users|home|root)\/[A-Za-z0-9_.-]+\//); + expect(contents, path.relative(ROOT, file)).not.toMatch(/[A-Za-z]:\\\\Users/); + } + }); + + it("reads the API key only in the live gate, and only to check it exists", () => { + for (const file of walk(path.join(ROOT, "evals"), (entry) => isSource(entry) && !isTest(entry))) { + const contents = readFileSync(file, "utf8"); + if (!contents.includes("OPENAI_API_KEY")) continue; + const relative = path.relative(ROOT, file); + expect(relative.includes("liveRunner") || relative.includes("cli/live"), relative).toBe(true); + // Presence check only. The value must never be interpolated, logged, or + // written anywhere. + expect(contents, relative).not.toMatch(/console\.\w+\([^)]*OPENAI_API_KEY/); + expect(contents, relative).not.toMatch(/\$\{[^}]*OPENAI_API_KEY[^}]*\}/); + } + }); + + it("never records a provider request or response body in the observer", () => { + const observer = readFileSync(path.join(ROOT, "evals/runners/observingProvider.js"), "utf8"); + // The observer keeps derived identifiers only; retaining prompt or + // response text would put synthetic-or-not content straight into artifacts. + expect(observer).not.toMatch(/trace\.(?:prompts|responses|bodies|headers)/); + expect(observer).toContain("never what came back"); + }); +}); + +describe("CLI ergonomics", () => { + const commands = { + validate: "evals/cli/validate.mjs", + fixtures: "evals/cli/fixtures.mjs", + live: "evals/cli/live.mjs", + compare: "evals/cli/compare.mjs", + }; + + for (const [name, script] of Object.entries(commands)) { + it(`eval:${name} supports --help and exits 0`, () => { + const result = runCli([script, "--help"]); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Usage:"); + expect(result.stdout).toContain("--help"); + }); + + it(`eval:${name} help output contains no ANSI escape codes`, () => { + // eslint-disable-next-line no-control-regex + expect(/\u001b\[/.test(runCli([script, "--help"]).stdout)).toBe(false); + }); + } + + it("eval:validate succeeds on the committed benchmark", () => { + const result = runCli([commands.validate]); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("decision-benchmark-v1"); + }); + + it("eval:validate fails with an actionable message for an unknown benchmark", () => { + const result = runCli([commands.validate, "--benchmark", "does-not-exist-v1"]); + expect(result.status).toBe(1); + expect(result.stderr.length).toBeGreaterThan(0); + expect(result.stderr).not.toMatch(/\/(?:Users|home|root)\//); + }); + + it("rejects unknown options instead of silently changing command behaviour", () => { + for (const script of Object.values(commands)) { + const result = runCli([script, "--definitely-unknown"]); + expect(result.status, script).toBe(1); + expect(result.stderr, script).toContain("Unknown option"); + } + }); + + it("eval:fixtures exits 0 on the committed baseline", () => { + const result = runCli([commands.fixtures, "--no-write"]); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("fixture machinery: PASSED"); + expect(result.stdout).toContain("production baseline: PASS WITH KNOWN DEFECTS"); + }); + + it("eval:fixtures exits nonzero when a required invariant fails", () => { + const result = runCli([ + commands.fixtures, + "--case", + "case-015", + "--profile", + "missing-pair", + "--no-write", + ]); + expect(result.status).toBe(1); + expect(result.stdout).toContain("fixture machinery: FAILED"); + expect(result.stdout).toContain("production baseline: UNEXPECTED FAILURE"); + expect(result.stderr).toContain("required grader failure"); + }); + + it("eval:fixtures rejects an unknown case with a pointer to eval:validate", () => { + const result = runCli([commands.fixtures, "--case", "case-999", "--no-write"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("eval:validate"); + }); + + it("eval:live refuses without --live, without reaching the provider", () => { + const result = runCli([commands.live, "--case", "case-001", "--max-budget-usd", "1"], { + OPENAI_API_KEY: "not-a-real-key", + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("--live"); + }); + + it("eval:live refuses without an API key", () => { + const result = runCli([commands.live, "--live", "--case", "case-001", "--max-budget-usd", "1"], { + OPENAI_API_KEY: "", + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("OPENAI_API_KEY"); + }); + + it("eval:live refuses without a budget", () => { + const result = runCli([commands.live, "--live", "--case", "case-001"], { + OPENAI_API_KEY: "not-a-real-key", + EVAL_MAX_BUDGET_USD: "", + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("budget"); + }); + + it("eval:live refuses in CI by default", () => { + const result = runCli( + [commands.live, "--live", "--case", "case-001", "--max-budget-usd", "1"], + { OPENAI_API_KEY: "not-a-real-key", CI: "true" }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain("CI"); + }); + + it("eval:live refuses when no case is selected", () => { + const result = runCli([commands.live, "--live", "--max-budget-usd", "1"], { + OPENAI_API_KEY: "not-a-real-key", + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("--all-cases"); + }); + + it("eval:compare refuses without both run directories", () => { + const result = runCli([commands.compare, "--baseline", ".eval-runs/whatever"]); + expect(result.status).toBe(1); + expect(result.stderr).toContain("--candidate"); + }); +}); + +describe("package wiring", () => { + const pkg = JSON.parse(readFileSync(path.join(ROOT, "package.json"), "utf8")); + + it("exposes all four evaluation commands", () => { + for (const script of ["eval:validate", "eval:fixtures", "eval:live", "eval:compare"]) { + expect(pkg.scripts, script).toHaveProperty(script); + } + }); + + it("includes the evaluation suite in npm test", () => { + expect(pkg.scripts.test).toContain("test:evals"); + expect(pkg.scripts["test:evals"]).toContain("vitest.evals.config.ts"); + }); + + it("adds no dependency for the evaluation harness", () => { + // Phase 3A introduced no package. The harness uses Node built-ins plus + // zod, which the application already depends on. + expect(pkg.dependencies).toHaveProperty("zod"); + for (const forbidden of ["commander", "yargs", "minimist", "chalk", "jest"]) { + expect(pkg.dependencies ?? {}, forbidden).not.toHaveProperty(forbidden); + expect(pkg.devDependencies ?? {}, forbidden).not.toHaveProperty(forbidden); + } + }); + + it("keeps the evaluation config present and separate", () => { + expect(existsSync(path.join(ROOT, "vitest.evals.config.ts"))).toBe(true); + const serverConfig = readFileSync(path.join(ROOT, "vitest.server.config.ts"), "utf8"); + expect(serverConfig).not.toContain("evals/"); + }); +}); diff --git a/evals/runners/caseVariants.js b/evals/runners/caseVariants.js new file mode 100644 index 0000000..ecdd9ba --- /dev/null +++ b/evals/runners/caseVariants.js @@ -0,0 +1,153 @@ +/** + * @file Controlled case-variant utilities (Phase 3A evaluation harness). + * + * These produce a *derived* case from an existing one, changing exactly one + * controlled thing and nothing else. Candidate IDs are always preserved, and + * `variant_of` always links back to the original, so the two results remain + * comparable. + * + * Honest scope note: under the offline fake provider, wording and + * irrelevant-text variants are guaranteed to produce identical results, + * because the fixture scores by candidate ID and never reads description text. + * They validate the machinery — the linkage, the comparison, the reporting — + * not the model. Only a live run can say whether wording actually moves a real + * model's scores. This is stated in docs/evaluation/BENCHMARK_V1.md rather + * than left for a reader to infer from a green result. + */ +import { benchmarkCaseSchema } from "../schemas/benchmarkCase.js"; + +function derive(original, overrides, variantKind, titleSuffix) { + const variant = { + ...structuredClone(original), + ...overrides, + variant_of: original.variant_of ?? original.case_id, + variant_kind: variantKind, + // Scoped observations belong only to the released source execution. A + // generated test variant must never inherit a suppression for another ID. + known_defects: [], + title: `${original.title} (${titleSuffix})`, + tags: [...new Set([...original.tags, "permutation"])], + }; + return variant; +} + +/** + * Reverses the submitted candidate order. Deterministic expectations are + * rewritten to match the new order, because `expected_candidate_ids` mirrors + * the submitted list — the *set* is unchanged, which is the point. + * @param {object} original validated benchmark case + * @param {string} caseId ID to give the variant + */ +export function createCandidateOrderVariant(original, caseId) { + const candidates = [...original.input.candidates].reverse(); + return derive( + original, + { + case_id: caseId, + input: { ...structuredClone(original.input), candidates }, + deterministic_expectations: { + ...structuredClone(original.deterministic_expectations), + expected_candidate_ids: candidates.map((candidate) => candidate.id), + }, + }, + "candidate-order", + "candidate order reversed", + ); +} + +/** + * Reverses scenario order, moving each scenario's weight deltas and score + * overrides with it so each scenario keeps its own configuration. + * @param {object} original + * @param {string} caseId + */ +export function createScenarioOrderVariant(original, caseId) { + const scenarios = [...original.input.scenarios].reverse(); + const lastIndex = original.input.scenarios.length - 1; + const remap = (record) => { + if (!record) return undefined; + return Object.fromEntries( + Object.entries(record).map(([index, value]) => [String(lastIndex - Number(index)), value]), + ); + }; + + const plan = structuredClone(original.fake_provider_plan); + const remappedOverrides = remap(plan.scenario_overrides); + const remappedDeltas = remap(plan.scenario_weight_deltas); + if (remappedOverrides) plan.scenario_overrides = remappedOverrides; + if (remappedDeltas) plan.scenario_weight_deltas = remappedDeltas; + + return derive( + original, + { + case_id: caseId, + input: { ...structuredClone(original.input), scenarios }, + deterministic_expectations: { + ...structuredClone(original.deterministic_expectations), + required_scenario_coverage: scenarios, + }, + fake_provider_plan: plan, + }, + "scenario-order", + "scenario order reversed", + ); +} + +/** + * Applies a caller-supplied rewrite to each candidate description. The caller + * owns the rewrite because "semantically equivalent" is a judgment a function + * cannot make: a mechanical synonym swap would silently change meaning. + * @param {object} original + * @param {string} caseId + * @param {(description: string, candidate: object) => string} rewrite + */ +export function createWordingVariant(original, caseId, rewrite) { + const input = structuredClone(original.input); + input.candidates = input.candidates.map((candidate) => ({ + ...candidate, + description: rewrite(candidate.description, candidate), + })); + return derive(original, { case_id: caseId, input }, "equivalent-wording", "reworded"); +} + +/** + * Appends one decision-irrelevant sentence to each candidate description. + * The default sentences are deliberately mundane and carry no demographic, + * protected, or evaluative signal — this case tests robustness to noise, and + * the benchmark makes no fairness claims of any kind. + * @param {object} original + * @param {string} caseId + * @param {string[]} [sentences] one per candidate, cycled if shorter + */ +export function createIrrelevantTextVariant( + original, + caseId, + sentences = [ + "The office is a twenty-minute walk from the nearest station.", + "The interview was scheduled for a Tuesday afternoon.", + "The application was submitted through the standard portal.", + ], +) { + const input = structuredClone(original.input); + input.candidates = input.candidates.map((candidate, index) => ({ + ...candidate, + description: `${candidate.description} ${sentences[index % sentences.length]}`, + })); + return derive(original, { case_id: caseId, input }, "irrelevant-text", "irrelevant sentence added"); +} + +/** + * Validates a generated variant against the same schema committed cases use. + * A variant that would not be accepted as a committed case is a bug in the + * generator, not an acceptable runtime shortcut. + * @param {object} variant + */ +export function validateVariant(variant) { + const result = benchmarkCaseSchema.safeParse(variant); + if (result.success) return result.data; + throw new Error( + `Generated variant "${variant.case_id}" is not a valid benchmark case:\n${result.error.issues + .map((issue) => ` - ${issue.path.join(".") || "(root)"}: ${issue.message}`) + .join("\n")}`, + ); +} diff --git a/evals/runners/caseVariants.test.js b/evals/runners/caseVariants.test.js new file mode 100644 index 0000000..7bda2c2 --- /dev/null +++ b/evals/runners/caseVariants.test.js @@ -0,0 +1,177 @@ +/** + * Case-variant utility tests: controlled changes, preserved linkage, and + * variants that are still valid benchmark cases. + */ +import { describe, it, expect, beforeAll } from "vitest"; + +import { loadBenchmark } from "../datasets/loadBenchmark.js"; +import { + createCandidateOrderVariant, + createScenarioOrderVariant, + createWordingVariant, + createIrrelevantTextVariant, + validateVariant, +} from "./caseVariants.js"; + +let caseById; + +beforeAll(async () => { + const benchmark = await loadBenchmark(); + caseById = new Map(benchmark.cases.map((entry) => [entry.case_id, entry])); +}); + +const ids = (benchmarkCase) => benchmarkCase.input.candidates.map((candidate) => candidate.id); + +describe("candidate-order variant", () => { + it("reverses the submitted order while preserving the candidate set", () => { + const original = caseById.get("case-001"); + const variant = createCandidateOrderVariant(original, "case-901"); + expect(ids(variant)).toEqual([...ids(original)].reverse()); + expect([...ids(variant)].sort()).toEqual([...ids(original)].sort()); + }); + + it("keeps expectations in step with the new order", () => { + const variant = createCandidateOrderVariant(caseById.get("case-001"), "case-901"); + expect(variant.deterministic_expectations.expected_candidate_ids).toEqual(ids(variant)); + }); + + it("records the linkage and tags itself a permutation", () => { + const variant = createCandidateOrderVariant(caseById.get("case-001"), "case-901"); + expect(variant.variant_of).toBe("case-001"); + expect(variant.variant_kind).toBe("candidate-order"); + expect(variant.tags).toContain("permutation"); + }); + + it("produces a case that still validates", () => { + expect(() => validateVariant(createCandidateOrderVariant(caseById.get("case-001"), "case-901"))).not.toThrow(); + }); + + it("changes nothing but the order", () => { + const original = caseById.get("case-001"); + const variant = createCandidateOrderVariant(original, "case-901"); + expect(variant.input.scenarios).toEqual(original.input.scenarios); + expect(variant.input.role).toEqual(original.input.role); + expect(variant.fake_provider_plan.candidate_scores).toEqual( + original.fake_provider_plan.candidate_scores, + ); + }); +}); + +describe("scenario-order variant", () => { + it("reverses scenario order", () => { + const original = caseById.get("case-004"); + const variant = createScenarioOrderVariant(original, "case-902"); + expect(variant.input.scenarios).toEqual([...original.input.scenarios].reverse()); + expect(variant.deterministic_expectations.required_scenario_coverage).toEqual( + variant.input.scenarios, + ); + }); + + it("moves each scenario's weight deltas with it", () => { + const original = caseById.get("case-004"); + const variant = createScenarioOrderVariant(original, "case-902"); + expect(variant.fake_provider_plan.scenario_weight_deltas["1"]).toEqual( + original.fake_provider_plan.scenario_weight_deltas["0"], + ); + expect(variant.fake_provider_plan.scenario_weight_deltas["0"]).toEqual( + original.fake_provider_plan.scenario_weight_deltas["1"], + ); + }); + + it("moves each scenario's score overrides with it", () => { + const original = caseById.get("case-006"); + const variant = createScenarioOrderVariant(original, "case-902"); + expect(variant.fake_provider_plan.scenario_overrides["0"]).toEqual( + original.fake_provider_plan.scenario_overrides["1"], + ); + }); + + it("produces a case that still validates", () => { + expect(() => validateVariant(createScenarioOrderVariant(caseById.get("case-004"), "case-902"))).not.toThrow(); + }); +}); + +describe("wording variant", () => { + it("applies the caller's rewrite to every description", () => { + const original = caseById.get("case-007"); + const variant = createWordingVariant(original, "case-903", (text) => `Rewritten: ${text}`); + for (const candidate of variant.input.candidates) { + expect(candidate.description.startsWith("Rewritten: ")).toBe(true); + } + }); + + it("preserves candidate IDs so the two cases stay comparable", () => { + const original = caseById.get("case-007"); + const variant = createWordingVariant(original, "case-903", (text) => `Reworded. ${text}`); + expect(ids(variant)).toEqual(ids(original)); + }); + + it("leaves the role, scenarios, and score plan untouched", () => { + const original = caseById.get("case-007"); + const variant = createWordingVariant(original, "case-903", (text) => `Reworded. ${text}`); + expect(variant.input.role).toEqual(original.input.role); + expect(variant.input.scenarios).toEqual(original.input.scenarios); + expect(variant.fake_provider_plan).toEqual(original.fake_provider_plan); + }); + + it("produces a case that still validates", () => { + const variant = createWordingVariant(caseById.get("case-007"), "case-903", (text) => `Reworded. ${text}`); + expect(() => validateVariant(variant)).not.toThrow(); + }); +}); + +describe("irrelevant-text variant", () => { + it("appends exactly one sentence per candidate", () => { + const original = caseById.get("case-007"); + const variant = createIrrelevantTextVariant(original, "case-904"); + variant.input.candidates.forEach((candidate, index) => { + expect(candidate.description.startsWith(original.input.candidates[index].description)).toBe(true); + expect(candidate.description.length).toBeGreaterThan( + original.input.candidates[index].description.length, + ); + }); + }); + + it("accepts caller-supplied sentences", () => { + const variant = createIrrelevantTextVariant(caseById.get("case-007"), "case-904", [ + "An entirely irrelevant sentence.", + ]); + for (const candidate of variant.input.candidates) { + expect(candidate.description.endsWith("An entirely irrelevant sentence.")).toBe(true); + } + }); + + it("preserves candidate IDs and the score plan", () => { + const original = caseById.get("case-007"); + const variant = createIrrelevantTextVariant(original, "case-904"); + expect(ids(variant)).toEqual(ids(original)); + expect(variant.fake_provider_plan).toEqual(original.fake_provider_plan); + }); + + it("produces a case that still validates", () => { + expect(() => validateVariant(createIrrelevantTextVariant(caseById.get("case-007"), "case-904"))).not.toThrow(); + }); +}); + +describe("variant validation", () => { + it("rejects a generated variant that is not a valid case", () => { + const broken = createCandidateOrderVariant(caseById.get("case-001"), "case-905"); + broken.tags = ["not-a-tag"]; + expect(() => validateVariant(broken)).toThrow(/not a valid benchmark case/); + }); + + it("links a variant of a variant back to the original case", () => { + const first = createCandidateOrderVariant(caseById.get("case-001"), "case-906"); + const second = createIrrelevantTextVariant(first, "case-907"); + expect(second.variant_of).toBe("case-001"); + }); + + it("does not mutate the case it derives from", () => { + const original = caseById.get("case-001"); + const before = JSON.stringify(original); + createCandidateOrderVariant(original, "case-908"); + createIrrelevantTextVariant(original, "case-909"); + createWordingVariant(original, "case-910", (text) => `x ${text}`); + expect(JSON.stringify(original)).toBe(before); + }); +}); diff --git a/evals/runners/compareRuns.js b/evals/runners/compareRuns.js new file mode 100644 index 0000000..a202812 --- /dev/null +++ b/evals/runners/compareRuns.js @@ -0,0 +1,391 @@ +/** + * @file Run comparison (Phase 3A evaluation harness). + * + * Compares two recorded runs and returns one of four verdicts: + * `improved | regressed | unchanged | inconclusive`. + * + * Three rules keep this honest: + * + * 1. **Invariants decide the verdict.** Required-grader failures are the only + * thing that can make a comparison say "improved" or "regressed". They are + * the only measure in the report that is objectively better or worse. + * + * 2. **Numeric deltas never imply significance.** Cost, tokens, and duration + * are reported with `significance: "not_assessed"`. Two runs cannot + * support a significance claim, so none is offered. + * + * 3. **Output changes without invariant changes are `inconclusive`, not + * `unchanged`.** If a candidate run picks a different winner while failing + * exactly as many invariants, the honest answer is that the benchmark + * cannot tell you which is better — not that nothing happened. + */ +import { + comparisonReportSchema, + numericDelta, + EVALUATION_REPORT_SCHEMA_VERSION, +} from "../schemas/evaluationReport.js"; +import { aggregateHumanReview, hasAnyScores } from "../graders/humanReview.js"; + +export const COMPARISON_LIMITATIONS = Object.freeze([ + "A comparison of two runs cannot establish statistical significance. Numeric differences in cost, tokens, and duration are reported as raw deltas and are explicitly not assessed for significance.", + "Only required-grader invariants determine the improved/regressed verdict. Everything else is reported for a human to interpret.", + "Rubric dimensions are compared only when both runs carry a completed human review with at least one real score. Two blank templates are never reported as agreement.", + "Run-to-run stability is only comparable when both runs used more than one repetition.", + "A changed winner is not automatically a regression: several benchmark cases have more than one legitimately defensible winner.", +]); + +class IncompatibleRunsError extends Error { + constructor(message) { + super(message); + this.name = "IncompatibleRunsError"; + } +} + +/** + * Refuses to compare runs of different benchmarks or benchmark versions. + * Comparing across a benchmark version change would silently attribute a + * benchmark edit to a pipeline change. + */ +function assertComparable(baseline, candidate) { + if (baseline.manifest.benchmark_id !== candidate.manifest.benchmark_id) { + throw new IncompatibleRunsError( + `Refusing to compare different benchmarks: "${baseline.manifest.benchmark_id}" vs "${candidate.manifest.benchmark_id}".`, + ); + } + if (baseline.manifest.benchmark_version !== candidate.manifest.benchmark_version) { + throw new IncompatibleRunsError( + `Refusing to compare benchmark versions ${baseline.manifest.benchmark_version} and ${candidate.manifest.benchmark_version}. ` + + "A benchmark version change alters what the cases mean, so the difference could not be attributed to the pipeline.", + ); + } + if (baseline.manifest.schema_version !== candidate.manifest.schema_version) { + throw new IncompatibleRunsError( + `Refusing to compare run schema versions ${baseline.manifest.schema_version} and ${candidate.manifest.schema_version}.`, + ); + } +} + +function schemaFailureCount(caseResults) { + return caseResults.reduce( + (total, caseResult) => + total + + caseResult.executions.reduce( + (executionTotal, execution) => + executionTotal + + execution.grader_results.filter( + (result) => result.grader_id === "contract-validity" && result.status !== "pass", + ).length, + 0, + ), + 0, + ); +} + +function expectedFailureCount(caseResults) { + return caseResults.reduce( + (total, caseResult) => + total + + [...caseResult.executions.flatMap((execution) => execution.grader_results ?? []), ...(caseResult.grader_results ?? [])].filter( + (result) => result.status === "expected_failure", + ).length, + 0, + ); +} + +function observationIdentity(observation, includeSignature = true) { + const fields = { + defect_id: observation.defect_id, + case_id: observation.case_id, + execution_id: observation.execution_id, + scenario_id: observation.scenario_id, + variant_id: observation.variant_id, + repetition: observation.repetition, + grader_id: observation.grader_id, + ...(includeSignature ? { signature: observation.signature } : {}), + }; + return JSON.stringify(fields); +} + +function compareDefectObservations(baseline, candidate) { + const before = baseline.summary.known_defect_observations ?? []; + const after = candidate.summary.known_defect_observations ?? []; + const beforeKeys = new Set(before.map((entry) => observationIdentity(entry))); + const afterKeys = new Set(after.map((entry) => observationIdentity(entry))); + const disappeared = [...beforeKeys].filter((key) => !afterKeys.has(key)); + const appeared = [...afterKeys].filter((key) => !beforeKeys.has(key)); + const group = (entries, identity) => entries.reduce((map, entry) => { + const key = identity(entry); + map.set(key, [...(map.get(key) ?? []), entry]); + return map; + }, new Map()); + const beforeScope = group(before, (entry) => observationIdentity(entry, false)); + const afterScope = group(after, (entry) => observationIdentity(entry, false)); + const changedSignature = [...beforeScope.keys()].filter((key) => + afterScope.has(key) && JSON.stringify(beforeScope.get(key).map((entry) => entry.signature).sort()) !== JSON.stringify(afterScope.get(key).map((entry) => entry.signature).sort()), + ); + // Repetition is deliberately retained: a signature in r2 cannot cancel or + // be called a move of an otherwise identical signature in r1. + const bySignature = (entry) => JSON.stringify({ defect_id: entry.defect_id, case_id: entry.case_id, repetition: entry.repetition, grader_id: entry.grader_id, signature: entry.signature }); + const placement = (entries) => group(entries, bySignature); + const beforePlacement = placement(before); + const afterPlacement = placement(after); + const moved = [...beforePlacement.keys()].filter((key) => afterPlacement.has(key) && + JSON.stringify(beforePlacement.get(key).map((entry) => observationIdentity(entry, false)).sort()) !== JSON.stringify(afterPlacement.get(key).map((entry) => observationIdentity(entry, false)).sort()), + ); + return { unchanged: [...beforeKeys].filter((key) => afterKeys.has(key)), disappeared, appeared, changed_signature: changedSignature, moved }; +} + +/** First completed execution per case+scenario; enough to detect a change. */ +function outcomesByScenario(caseResult) { + const map = new Map(); + for (const execution of caseResult.executions) { + if (execution.status !== "completed") continue; + if (!map.has(execution.scenario)) map.set(execution.scenario, execution); + } + return map; +} + +function compareCase(baselineCase, candidateCase) { + const reasons = []; + let winnerChanged = false; + let rankingChanged = false; + let pairChanged = false; + let evidenceChanged = false; + let explanationChanged = false; + + const baselineByScenario = outcomesByScenario(baselineCase); + const candidateByScenario = outcomesByScenario(candidateCase); + + for (const [scenario, candidateExecution] of candidateByScenario) { + const baselineExecution = baselineByScenario.get(scenario); + if (!baselineExecution) { + reasons.push("a scenario present in the candidate run has no counterpart in the baseline"); + continue; + } + if (candidateExecution.outcome.winner_id !== baselineExecution.outcome.winner_id) winnerChanged = true; + if ( + JSON.stringify(candidateExecution.outcome.ranking) !== + JSON.stringify(baselineExecution.outcome.ranking) + ) { + rankingChanged = true; + } + if (candidateExecution.outcome.best_pair_key !== baselineExecution.outcome.best_pair_key) { + pairChanged = true; + } + if ( + JSON.stringify(candidateExecution.response?.candidate_evaluations ?? null) !== + JSON.stringify(baselineExecution.response?.candidate_evaluations ?? null) + ) { + evidenceChanged = true; + } + if ( + JSON.stringify(candidateExecution.response?.executive_summary ?? null) !== + JSON.stringify(baselineExecution.response?.executive_summary ?? null) || + candidateExecution.response?.decision_result.key_reason !== + baselineExecution.response?.decision_result.key_reason + ) { + explanationChanged = true; + } + } + + const requiredDelta = numericDelta(baselineCase.required_failures, candidateCase.required_failures); + let verdict; + if (requiredDelta.delta !== null && requiredDelta.delta < 0) { + verdict = "improved"; + reasons.push(`required failures fell from ${baselineCase.required_failures} to ${candidateCase.required_failures}`); + } else if (requiredDelta.delta !== null && requiredDelta.delta > 0) { + verdict = "regressed"; + reasons.push(`required failures rose from ${baselineCase.required_failures} to ${candidateCase.required_failures}`); + } else if (winnerChanged || rankingChanged || pairChanged || evidenceChanged) { + verdict = "inconclusive"; + reasons.push("the decision output changed while required invariants stayed the same; the benchmark cannot say which output is better"); + } else if (explanationChanged) { + verdict = "inconclusive"; + reasons.push("only the explanation text changed; explanation quality is a human-review judgment, not a deterministic one"); + } else { + verdict = "unchanged"; + reasons.push("no invariant, decision, or explanation difference was detected"); + } + + return { + case_id: candidateCase.case_id, + verdict, + reasons, + winner_changed: winnerChanged, + ranking_changed: rankingChanged, + best_pair_changed: pairChanged, + structured_evidence_changed: evidenceChanged, + explanation_changed: explanationChanged, + required_failures: requiredDelta, + advisory_failures: numericDelta(baselineCase.advisory_failures, candidateCase.advisory_failures), + expected_failures: numericDelta( + expectedFailureCount([baselineCase]), + expectedFailureCount([candidateCase]), + ), + schema_failures: numericDelta( + schemaFailureCount([baselineCase]), + schemaFailureCount([candidateCase]), + ), + }; +} + +function compareRubric(baseline, candidate) { + const usable = (run) => run.humanReview && hasAnyScores(run.humanReview); + if (!usable(baseline) || !usable(candidate)) { + const missing = [ + usable(baseline) ? null : "baseline", + usable(candidate) ? null : "candidate", + ].filter(Boolean); + return { + compared: false, + reason: `Rubric dimensions were not compared: no completed human review with real scores for the ${missing.join(" and ")} run.`, + dimensions: {}, + }; + } + + const baselineAggregate = aggregateHumanReview(baseline.humanReview); + const candidateAggregate = aggregateHumanReview(candidate.humanReview); + const dimensionIds = new Set([ + ...Object.keys(baselineAggregate.dimension_scores), + ...Object.keys(candidateAggregate.dimension_scores), + ]); + + return { + compared: true, + reason: "Both runs carry a completed human review with at least one real score. Dimension means are reported individually and are never collapsed into a single quality number.", + dimensions: Object.fromEntries( + [...dimensionIds].sort().map((id) => [ + id, + numericDelta( + baselineAggregate.dimension_scores[id]?.mean ?? undefined, + candidateAggregate.dimension_scores[id]?.mean ?? undefined, + ), + ]), + ), + }; +} + +function compareStability(baseline, candidate) { + const baselineStability = baseline.summary.stability; + const candidateStability = candidate.summary.stability; + if (!baselineStability.assessed || !candidateStability.assessed) { + return { + compared: false, + reason: "Stability was not compared: at least one run used a single repetition, which cannot demonstrate run-to-run behaviour.", + baseline_winner_agreement: baselineStability.winner_agreement, + candidate_winner_agreement: candidateStability.winner_agreement, + }; + } + return { + compared: true, + reason: "Both runs assessed stability across more than one repetition. Agreement rates are reported without a significance claim.", + baseline_winner_agreement: baselineStability.winner_agreement, + candidate_winner_agreement: candidateStability.winner_agreement, + }; +} + +/** + * @param {object} baseline result of readRunArtifacts() + * @param {object} candidate result of readRunArtifacts() + * @returns {object} validated comparison report + */ +export function compareRuns(baseline, candidate) { + assertComparable(baseline, candidate); + + const baselineById = new Map(baseline.caseResults.map((entry) => [entry.case_id, entry])); + const shared = candidate.caseResults.filter((entry) => baselineById.has(entry.case_id)); + const onlyInCandidate = candidate.caseResults.length - shared.length; + const onlyInBaseline = baseline.caseResults.length - shared.length; + + const cases = shared.map((candidateCase) => + compareCase(baselineById.get(candidateCase.case_id), candidateCase), + ); + + const invariants = { + required_failures: numericDelta( + baseline.summary.required_failures, + candidate.summary.required_failures, + ), + advisory_failures: numericDelta( + baseline.summary.advisory_failures, + candidate.summary.advisory_failures, + ), + expected_failures: numericDelta( + baseline.summary.expected_failures, + candidate.summary.expected_failures, + ), + schema_failures: numericDelta( + schemaFailureCount(baseline.caseResults), + schemaFailureCount(candidate.caseResults), + ), + passed_cases: numericDelta(baseline.summary.passed_cases, candidate.summary.passed_cases), + }; + const defectObservations = compareDefectObservations(baseline, candidate); + + const verdictReasons = []; + let verdict; + + if (shared.length === 0) { + verdict = "inconclusive"; + verdictReasons.push("the two runs share no cases, so nothing could be compared"); + } else if (invariants.required_failures.delta > 0 || defectObservations.appeared.length > 0) { + verdict = "regressed"; + verdictReasons.push(defectObservations.appeared.length > 0 + ? `${defectObservations.appeared.length} additional exact known-defect observation(s) appeared` + : `required-grader failures rose from ${baseline.summary.required_failures} to ${candidate.summary.required_failures}`); + } else if (defectObservations.disappeared.length > 0 || defectObservations.changed_signature.length > 0 || defectObservations.moved.length > 0) { + verdict = "baseline_change_required"; + verdictReasons.push("known-defect observations changed; review and deliberately update the production baseline"); + } else if (invariants.required_failures.delta < 0) { + verdict = "improved"; + verdictReasons.push( + `required-grader failures fell from ${baseline.summary.required_failures} to ${candidate.summary.required_failures}`, + ); + } else if (cases.some((entry) => entry.verdict === "inconclusive")) { + verdict = "inconclusive"; + verdictReasons.push( + `${cases.filter((entry) => entry.verdict === "inconclusive").length} case(s) changed their output without changing any required invariant`, + ); + } else { + verdict = "unchanged"; + verdictReasons.push("required invariants, decisions, and explanations are identical across the two runs"); + } + + if (onlyInBaseline > 0 || onlyInCandidate > 0) { + verdictReasons.push( + `case selections differ: ${onlyInBaseline} case(s) only in the baseline, ${onlyInCandidate} only in the candidate; only the ${shared.length} shared case(s) were compared`, + ); + } + if (baseline.manifest.mode !== candidate.manifest.mode) { + verdictReasons.push( + `the runs used different modes (${baseline.manifest.mode} vs ${candidate.manifest.mode}); a fixture run and a live run are not directly comparable`, + ); + } + + const report = { + schema_version: EVALUATION_REPORT_SCHEMA_VERSION, + generated_at: new Date().toISOString(), + baseline_run_id: baseline.manifest.run_id, + candidate_run_id: candidate.manifest.run_id, + benchmark_id: candidate.manifest.benchmark_id, + benchmark_version: candidate.manifest.benchmark_version, + verdict, + verdict_reasons: verdictReasons, + invariants, + defect_observations: defectObservations, + cost: numericDelta(baseline.manifest.estimated_cost_usd, candidate.manifest.estimated_cost_usd), + tokens: numericDelta(baseline.manifest.total_tokens, candidate.manifest.total_tokens), + duration_ms: numericDelta(baseline.manifest.duration_ms, candidate.manifest.duration_ms), + rubric: compareRubric(baseline, candidate), + stability: compareStability(baseline, candidate), + winner_changes: cases.filter((entry) => entry.winner_changed).map((entry) => entry.case_id), + ranking_changes: cases.filter((entry) => entry.ranking_changed).map((entry) => entry.case_id), + pair_changes: cases.filter((entry) => entry.best_pair_changed).map((entry) => entry.case_id), + cases, + limitations: [...COMPARISON_LIMITATIONS], + }; + + return comparisonReportSchema.parse(report); +} + +export { IncompatibleRunsError }; diff --git a/evals/runners/compareRuns.test.js b/evals/runners/compareRuns.test.js new file mode 100644 index 0000000..40ac4e3 --- /dev/null +++ b/evals/runners/compareRuns.test.js @@ -0,0 +1,357 @@ +/** + * Comparison tests: verdicts, incompatibility refusals, and the limits the + * comparison is required to state rather than paper over. + */ +import { describe, it, expect } from "vitest"; + +import { compareRuns, IncompatibleRunsError, COMPARISON_LIMITATIONS } from "./compareRuns.js"; +import { renderComparisonMarkdown } from "../reporters/markdownReporter.js"; + +function makeRun({ + runId = "run-a", + requiredFailures = 0, + advisoryFailures = 0, + passedCases = 1, + winner = "cand-a", + ranking = ["cand-a", "cand-b"], + bestPair = null, + keyReason = "Highest score.", + benchmarkVersion = "1.0.0", + benchmarkId = "decision-benchmark-v1", + mode = "fixtures", + repetitions = 1, + stabilityAssessed = false, + tokens = 100, + cost = 0.01, + duration = 1000, + humanReview = null, + caseIds = ["case-001"], + contractValidityStatus = "pass", + expectedFailures = 0, + knownDefectObservations = [], +} = {}) { + return { + manifest: { + run_id: runId, + schema_version: "1.0.0", + benchmark_id: benchmarkId, + benchmark_version: benchmarkVersion, + mode, + repetitions, + total_tokens: tokens, + estimated_cost_usd: cost, + duration_ms: duration, + }, + summary: { + required_failures: requiredFailures, + advisory_failures: advisoryFailures, + expected_failures: expectedFailures, + known_defect_observations: knownDefectObservations, + passed_cases: passedCases, + stability: stabilityAssessed + ? { assessed: true, reason: "assessed", winner_agreement: 1, ranking_agreement: 1 } + : { assessed: false, reason: "single repetition", winner_agreement: null, ranking_agreement: null }, + }, + caseResults: caseIds.map((caseId) => ({ + case_id: caseId, + required_failures: requiredFailures, + advisory_failures: advisoryFailures, + grader_results: [], + executions: [ + { + status: "completed", + scenario: "A fictional scenario.", + scenario_index: 0, + outcome: { winner_id: winner, ranking, best_pair_key: bestPair }, + grader_results: [ + { grader_id: "contract-validity", severity: "required", status: contractValidityStatus, summary: "", details: [] }, + ], + response: { + candidate_evaluations: ranking.map((id) => ({ candidate_id: id })), + decision_result: { key_reason: keyReason }, + executive_summary: { recommendation: `${winner} recommended.` }, + }, + }, + ], + })), + humanReview, + }; +} + +function reviewWith(scores) { + return { + schema_version: "1.0.0", + run_id: "r", + benchmark_id: "decision-benchmark-v1", + benchmark_version: "1.0.0", + rubric_version: "1.0.0", + instructions: "x".repeat(50), + scale_legend: { 0: "unacceptable" }, + entries: [ + { + execution_id: "e", + case_id: "case-001", + scenario_index: 0, + repetition: 1, + reviewer: "reviewer", + reviewed_at: "2026-08-02", + dimensions: Object.entries(scores).map(([id, score]) => ({ + dimension_id: id, + label: id, + what_is_judged: "judged", + anchors: { 0: "bad" }, + score, + reviewer_notes: "", + })), + overall_notes: "", + }, + ], + }; +} + +describe("comparison verdicts", () => { + it("reports unchanged for two identical runs", () => { + const report = compareRuns(makeRun(), makeRun({ runId: "run-b" })); + expect(report.verdict).toBe("unchanged"); + expect(report.verdict_reasons.join(" ")).toContain("identical"); + }); + + it("reports improved when required failures fall", () => { + const report = compareRuns( + makeRun({ requiredFailures: 3, passedCases: 0 }), + makeRun({ runId: "run-b", requiredFailures: 0, passedCases: 1 }), + ); + expect(report.verdict).toBe("improved"); + expect(report.invariants.required_failures.delta).toBe(-3); + }); + + it("reports regressed when required failures rise", () => { + const report = compareRuns( + makeRun({ requiredFailures: 0 }), + makeRun({ runId: "run-b", requiredFailures: 2 }), + ); + expect(report.verdict).toBe("regressed"); + expect(report.invariants.required_failures.delta).toBe(2); + }); + + it("reports expected known-defect observations without changing an otherwise unchanged verdict", () => { + const report = compareRuns( + makeRun({ expectedFailures: 2 }), + makeRun({ runId: "run-b", expectedFailures: 1 }), + ); + expect(report.verdict).toBe("unchanged"); + expect(report.invariants.expected_failures.delta).toBe(-1); + }); + + it("reports inconclusive when the winner changes without an invariant change", () => { + const report = compareRuns( + makeRun(), + makeRun({ runId: "run-b", winner: "cand-b", ranking: ["cand-b", "cand-a"] }), + ); + expect(report.verdict).toBe("inconclusive"); + expect(report.winner_changes).toEqual(["case-001"]); + expect(report.ranking_changes).toEqual(["case-001"]); + }); + + it("reports inconclusive when only the explanation text changed", () => { + const report = compareRuns(makeRun(), makeRun({ runId: "run-b", keyReason: "Different wording." })); + expect(report.verdict).toBe("inconclusive"); + expect(report.cases[0].explanation_changed).toBe(true); + expect(report.cases[0].winner_changed).toBe(false); + }); + + it("detects a changed best pair", () => { + const report = compareRuns( + makeRun({ bestPair: "a::b" }), + makeRun({ runId: "run-b", bestPair: "a::c" }), + ); + expect(report.pair_changes).toEqual(["case-001"]); + }); + + it("detects a schema failure change", () => { + const report = compareRuns( + makeRun(), + makeRun({ runId: "run-b", contractValidityStatus: "fail", requiredFailures: 1 }), + ); + expect(report.invariants.schema_failures.delta).toBe(1); + expect(report.verdict).toBe("regressed"); + }); + + it("reports inconclusive when the runs share no cases", () => { + const report = compareRuns( + makeRun({ caseIds: ["case-001"] }), + makeRun({ runId: "run-b", caseIds: ["case-002"] }), + ); + expect(report.verdict).toBe("inconclusive"); + expect(report.verdict_reasons.join(" ")).toContain("share no cases"); + }); + + it("notes when only some cases overlap", () => { + const report = compareRuns( + makeRun({ caseIds: ["case-001", "case-002"] }), + makeRun({ runId: "run-b", caseIds: ["case-001"] }), + ); + expect(report.verdict_reasons.join(" ")).toContain("case selections differ"); + }); + + it("notes when a fixture run is compared against a live run", () => { + const report = compareRuns(makeRun(), makeRun({ runId: "run-b", mode: "live" })); + expect(report.verdict_reasons.join(" ")).toContain("different modes"); + }); +}); + +describe("exact known-defect observation comparison", () => { + const observation = { + defect_id: "SR-P3A-001", + case_id: "case-001", + execution_id: "case-001#s0#r1", + scenario_id: "scenario-1", + variant_id: null, + repetition: 1, + grader_id: "score-integrity", + signature: { kind: "score_bound_violation", metric: "risk_adjusted_score", operator: "lt", bound: 0, subject_candidate_id: "priya-tallow" }, + }; + + it("requires a baseline review when an exact observation disappears", () => { + const report = compareRuns( + makeRun({ knownDefectObservations: [observation], expectedFailures: 1 }), + makeRun({ runId: "run-b" }), + ); + expect(report.verdict).toBe("baseline_change_required"); + expect(report.defect_observations.disappeared).toHaveLength(1); + }); + + it("treats an additional scoped observation as a regression", () => { + const moved = { ...observation, execution_id: "case-001#s1#r1", scenario_id: "scenario-2" }; + const report = compareRuns( + makeRun({ knownDefectObservations: [observation], expectedFailures: 1 }), + makeRun({ runId: "run-b", knownDefectObservations: [observation, moved], expectedFailures: 2 }), + ); + expect(report.verdict).toBe("regressed"); + expect(report.defect_observations.appeared).toHaveLength(1); + }); +}); + +describe("comparison refusals", () => { + it("refuses different benchmarks", () => { + expect(() => + compareRuns(makeRun(), makeRun({ runId: "run-b", benchmarkId: "other-benchmark-v1" })), + ).toThrow(IncompatibleRunsError); + }); + + it("refuses different benchmark versions", () => { + expect(() => + compareRuns(makeRun(), makeRun({ runId: "run-b", benchmarkVersion: "2.0.0" })), + ).toThrow(/A benchmark version change alters what the cases mean/); + }); + + it("refuses different run schema versions", () => { + const candidate = makeRun({ runId: "run-b" }); + candidate.manifest.schema_version = "2.0.0"; + expect(() => compareRuns(makeRun(), candidate)).toThrow(/run schema versions/); + }); +}); + +describe("cost, tokens, and duration", () => { + it("reports raw deltas and never claims significance", () => { + const report = compareRuns( + makeRun({ cost: 0.02, tokens: 200, duration: 2000 }), + makeRun({ runId: "run-b", cost: 0.01, tokens: 100, duration: 1000 }), + ); + expect(report.cost.delta).toBeCloseTo(-0.01, 10); + expect(report.tokens.delta).toBe(-100); + expect(report.duration_ms.delta).toBe(-1000); + for (const entry of [report.cost, report.tokens, report.duration_ms]) { + expect(entry.significance).toBe("not_assessed"); + } + }); + + it("does not let a cost change alter the verdict", () => { + const report = compareRuns(makeRun({ cost: 10 }), makeRun({ runId: "run-b", cost: 0.0001 })); + expect(report.verdict).toBe("unchanged"); + }); + + it("reports a null delta when a cost is unavailable", () => { + const report = compareRuns(makeRun({ cost: null }), makeRun({ runId: "run-b", cost: 0.01 })); + expect(report.cost.delta).toBeNull(); + }); +}); + +describe("rubric comparison", () => { + it("declines to compare when neither run carries a review", () => { + const report = compareRuns(makeRun(), makeRun({ runId: "run-b" })); + expect(report.rubric.compared).toBe(false); + expect(report.rubric.reason).toContain("baseline and candidate"); + }); + + it("declines to compare when only one run carries a review", () => { + const report = compareRuns( + makeRun({ humanReview: reviewWith({ clarity: 3 }) }), + makeRun({ runId: "run-b" }), + ); + expect(report.rubric.compared).toBe(false); + }); + + it("declines to compare two blank templates", () => { + const blank = reviewWith({ clarity: null }); + const report = compareRuns( + makeRun({ humanReview: blank }), + makeRun({ runId: "run-b", humanReview: blank }), + ); + expect(report.rubric.compared).toBe(false); + }); + + it("compares dimension means when both runs carry real scores", () => { + const report = compareRuns( + makeRun({ humanReview: reviewWith({ clarity: 2, evidence_grounding: 3 }) }), + makeRun({ runId: "run-b", humanReview: reviewWith({ clarity: 4, evidence_grounding: 3 }) }), + ); + expect(report.rubric.compared).toBe(true); + expect(report.rubric.dimensions.clarity.delta).toBe(2); + expect(report.rubric.dimensions.evidence_grounding.delta).toBe(0); + }); + + it("does not let a rubric change alter the verdict", () => { + const report = compareRuns( + makeRun({ humanReview: reviewWith({ clarity: 0 }) }), + makeRun({ runId: "run-b", humanReview: reviewWith({ clarity: 4 }) }), + ); + expect(report.verdict).toBe("unchanged"); + }); +}); + +describe("stability comparison", () => { + it("declines when either run used a single repetition", () => { + const report = compareRuns( + makeRun({ stabilityAssessed: true, repetitions: 3 }), + makeRun({ runId: "run-b" }), + ); + expect(report.stability.compared).toBe(false); + expect(report.stability.reason).toContain("single repetition"); + }); + + it("compares when both runs assessed stability", () => { + const report = compareRuns( + makeRun({ stabilityAssessed: true, repetitions: 3 }), + makeRun({ runId: "run-b", stabilityAssessed: true, repetitions: 3 }), + ); + expect(report.stability.compared).toBe(true); + expect(report.stability.baseline_winner_agreement).toBe(1); + }); +}); + +describe("comparison report shape", () => { + it("always states its limitations", () => { + const report = compareRuns(makeRun(), makeRun({ runId: "run-b" })); + expect(report.limitations).toEqual([...COMPARISON_LIMITATIONS]); + expect(report.limitations.join(" ")).toContain("cannot establish statistical significance"); + }); + + it("renders to markdown without ANSI escapes", () => { + const markdown = renderComparisonMarkdown(compareRuns(makeRun(), makeRun({ runId: "run-b" }))); + expect(markdown).toContain("**Verdict: unchanged**"); + expect(markdown).toContain("## Limitations"); + // eslint-disable-next-line no-control-regex + expect(/\u001b\[/.test(markdown)).toBe(false); + }); +}); diff --git a/evals/runners/liveRunner.js b/evals/runners/liveRunner.js new file mode 100644 index 0000000..430d462 --- /dev/null +++ b/evals/runners/liveRunner.js @@ -0,0 +1,349 @@ +/** + * @file Live-mode gating and budget enforcement (Phase 3A evaluation harness). + * + * Live mode spends real money against a real OpenAI account. Every guard below + * exists because its absence has a plausible, expensive failure mode: + * + * - `--live` is required, so no ordinary command can drift into billing. + * - `OPENAI_API_KEY` must be present, checked before anything is built. + * - CI is refused by default, so a pipeline cannot quietly spend a budget. + * - A budget limit is mandatory and must be a positive, finite number. + * - The plan (model, cases, repetitions, maximum estimated calls and cost) + * is printed before any request is made. + * - The pre-flight worst-case estimate must fit inside the budget, or the + * run is refused before the first call rather than stopped part-way. + * - Spend is re-checked between executions and the run stops *before* + * starting one that could exceed the limit. + * - Default repetitions is 1, and default case selection is nothing at all — + * running the whole benchmark takes a separate explicit flag. + * - An unpriced model is refused: a budget that cannot be computed cannot + * be enforced, and guessing a price would defeat the purpose. + * + * Nothing in this module is exercised against the real API during Phase 3A + * implementation. Its tests drive the refusal paths and the budget arithmetic + * with injected values, and no automated test in this repository calls OpenAI. + */ +import { estimateCostUsd, getPricingForModel } from "../../server/ai/pricing/openaiPricing.js"; +import { PROVIDER_COST_POLICY } from "../../server/pipeline/runPipeline.js"; + +/** + * Worst-case output-token budgets per logical stage, derived from the frozen + * production cost policy export in server/pipeline/runPipeline.js. The + * estimator therefore tracks the same reviewed limits without maintaining a + * second copy. + */ +export const STAGE_OUTPUT_TOKEN_BUDGETS = PROVIDER_COST_POLICY.outputTokenBudgets; + +/** + * Deliberately pessimistic assumptions. Under-estimating spend is the only + * error mode that actually costs money, so both are set high. + */ +export const BUDGET_ASSUMPTIONS = Object.freeze({ + /** The production adapter can make an initial request plus one retry. */ + providerAttemptsPerRequest: PROVIDER_COST_POLICY.maxProviderAttemptsPerRequest, + /** Batch scoring and pairing can each issue a corrective second request. */ + batchIntegrityPasses: PROVIDER_COST_POLICY.maxBatchIntegrityExecutions, + /** A truncation retry may raise its output cap by 1.5x. */ + truncationRetryOutputMultiplier: 1.5, + /** Assumed prompt size per attempt; real prompts are far smaller. */ + inputTokensPerAttempt: 6000, +}); + +/** Safety caps, not default spend or throughput targets. */ +export const MAX_EVALUATION_BUDGET_USD = 100; +export const MAX_EVALUATION_REPETITIONS = 20; + +export class LiveModeRefusedError extends Error { + constructor(message) { + super(message); + this.name = "LiveModeRefusedError"; + } +} + +/** + * Worst-case cost for one execution (one scenario, one repetition) of a case. + * @param {object} benchmarkCase + * @param {string} model + * @returns {{ estimatedCostUsd: number|null, maxAttempts: number, outputTokens: number, inputTokens: number }} + */ +export function estimateExecutionCost(benchmarkCase, model, { policy = PROVIDER_COST_POLICY } = {}) { + const candidateCount = benchmarkCase.input.candidates.length; + const pairingEnabled = benchmarkCase.deterministic_expectations.pairing_enabled; + const pairCount = benchmarkCase.deterministic_expectations.expected_pair_count ?? 0; + const requestOutputWorstCase = (cap) => + cap * (1 + BUDGET_ASSUMPTIONS.truncationRetryOutputMultiplier); + const budgets = policy.outputTokenBudgets; + const providerAttempts = policy.maxProviderAttemptsPerRequest; + const batchPasses = policy.maxBatchIntegrityExecutions; + const contextOutput = requestOutputWorstCase(budgets.contextAnalysis); + const scoringOutput = requestOutputWorstCase( + candidateCount * budgets.candidateScoringPerCandidate + budgets.candidateScoringOverhead, + ) * batchPasses; + const pairingOutput = pairingEnabled + ? requestOutputWorstCase( + pairCount * budgets.pairingPerPair + budgets.pairingOverhead, + ) * batchPasses + : 0; + const decisionOutput = requestOutputWorstCase(budgets.decisionExplanation); + const maxAttempts = + providerAttempts * (1 + batchPasses + (pairingEnabled ? batchPasses : 0) + 1); + const outputTokens = contextOutput + scoringOutput + pairingOutput + decisionOutput; + const inputTokens = maxAttempts * BUDGET_ASSUMPTIONS.inputTokensPerAttempt; + + return { + estimatedCostUsd: estimateCostUsd({ model, inputTokens, cachedInputTokens: 0, outputTokens }), + maxAttempts, + outputTokens, + inputTokens, + }; +} + +/** + * Whole-run worst case: every selected case, every scenario, every repetition. + * @param {object[]} selectedCases + * @param {string} model + * @param {number} repetitions + */ +export function estimateRunBudget(selectedCases, model, repetitions, { policy = PROVIDER_COST_POLICY } = {}) { + if (getPricingForModel(model) === null) { + return { + priced: false, + model, + maxEstimatedCostUsd: null, + maxProviderAttempts: null, + executionCount: null, + perCase: [], + }; + } + + let cost = 0; + let attempts = 0; + let executionCount = 0; + const perCase = []; + + for (const benchmarkCase of selectedCases) { + const executions = benchmarkCase.input.scenarios.length * repetitions; + const estimate = estimateExecutionCost(benchmarkCase, model, { policy }); + cost += estimate.estimatedCostUsd * executions; + attempts += estimate.maxAttempts * executions; + executionCount += executions; + perCase.push({ + case_id: benchmarkCase.case_id, + executions, + max_attempts: estimate.maxAttempts * executions, + max_cost_usd: Number((estimate.estimatedCostUsd * executions).toFixed(6)), + }); + } + + return { + priced: true, + model, + maxEstimatedCostUsd: Number(cost.toFixed(6)), + maxProviderAttempts: attempts, + executionCount, + perCase, + }; +} + +/** + * Parses and validates the budget limit from an argument or the environment. + * @param {{ maxBudgetUsd?: string|number, env?: Record }} options + */ +export function resolveBudgetLimit({ maxBudgetUsd, env = process.env } = {}) { + const raw = maxBudgetUsd ?? env.EVAL_MAX_BUDGET_USD; + if (raw === undefined || raw === null || `${raw}`.trim() === "") { + throw new LiveModeRefusedError( + "Live mode requires an explicit budget limit. Pass --max-budget-usd or set EVAL_MAX_BUDGET_USD.", + ); + } + const normalized = `${raw}`.trim(); + // Deliberately decimal-only: exponent forms and trailing junk make an + // operator's budget review harder, and this CLI does not document them. + if (!/^\d+(?:\.\d+)?$/.test(normalized)) { + throw new LiveModeRefusedError( + `Invalid budget limit "${raw}". Use a positive, finite number of US dollars written as a decimal.`, + ); + } + const parsed = Number(normalized); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new LiveModeRefusedError( + `Invalid budget limit "${raw}". It must be a positive, finite number of US dollars.`, + ); + } + if (parsed > MAX_EVALUATION_BUDGET_USD) { + throw new LiveModeRefusedError( + `Invalid budget limit "${raw}". The evaluation safety cap is $${MAX_EVALUATION_BUDGET_USD} per command; select fewer cases instead.`, + ); + } + return parsed; +} + +/** + * Every live-mode precondition, checked before a provider is constructed or a + * single request is made. Throws `LiveModeRefusedError` with an actionable + * message; returns the resolved plan inputs on success. + * + * @param {object} options + * @param {boolean} options.live the `--live` flag + * @param {boolean} [options.allowCi] the `--allow-ci` escape hatch + * @param {string[]} [options.caseIds] explicitly selected cases + * @param {boolean} [options.allCases] the `--all-cases` flag + * @param {number} [options.repetitions] + * @param {string|number} [options.maxBudgetUsd] + * @param {Record} [options.env] + * @param {object[]} options.benchmarkCases every case in the benchmark + */ +export function assertLiveModeAllowed({ + live, + allowCi = false, + caseIds = [], + allCases = false, + repetitions = 1, + maxBudgetUsd, + env = process.env, + benchmarkCases, +}) { + if (!live) { + throw new LiveModeRefusedError( + "Live mode requires the explicit --live flag. Without it, nothing is sent to OpenAI. Use `npm run eval:fixtures` for an offline run.", + ); + } + if (!env.OPENAI_API_KEY) { + throw new LiveModeRefusedError( + "Live mode requires OPENAI_API_KEY to be set. It is read from the environment and never recorded in any artifact.", + ); + } + // Truthiness of CI is checked as a string: CI="false" is still a CI + // environment declaring itself, and treating it as "not CI" would be a + // surprising place to be clever. + const inCi = env.CI !== undefined && env.CI !== "" && env.CI !== "0" && env.CI !== "false"; + if (inCi && !allowCi) { + throw new LiveModeRefusedError( + "Live mode refuses to run in CI by default, because a scheduled job should not spend an API budget. Pass --allow-ci if this is genuinely intended.", + ); + } + + const budgetUsd = resolveBudgetLimit({ maxBudgetUsd, env }); + + if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > MAX_EVALUATION_REPETITIONS) { + throw new LiveModeRefusedError( + `Invalid --repetitions "${repetitions}". It must be a positive integer no greater than ${MAX_EVALUATION_REPETITIONS}; the default is 1.`, + ); + } + + if (!allCases && caseIds.length === 0) { + throw new LiveModeRefusedError( + "No cases selected. Live mode never defaults to the whole benchmark: pass --case (repeatable), or --all-cases to run every case deliberately.", + ); + } + if (allCases && caseIds.length > 0) { + throw new LiveModeRefusedError( + "Pass either --case or --all-cases, not both.", + ); + } + + const knownIds = new Set(benchmarkCases.map((entry) => entry.case_id)); + const unknown = caseIds.filter((caseId) => !knownIds.has(caseId)); + if (unknown.length > 0) { + throw new LiveModeRefusedError( + `Unknown case id(s): ${unknown.join(", ")}. Run \`npm run eval:validate\` to list the benchmark's cases.`, + ); + } + + const selectedIds = allCases ? benchmarkCases.map((entry) => entry.case_id) : caseIds; + return { budgetUsd, selectedIds, repetitions }; +} + +/** + * Checks the pre-flight worst case against the budget. An unpriced model is + * refused outright rather than run without enforcement. + * @param {object} estimate result of estimateRunBudget() + * @param {number} budgetUsd + */ +export function assertBudgetCoversPlan(estimate, budgetUsd) { + if (!estimate.priced) { + throw new LiveModeRefusedError( + `No recorded pricing for model "${estimate.model}", so a budget cannot be enforced. ` + + "Add the model to server/ai/pricing/openaiPricing.js from OpenAI's own pricing page before running live.", + ); + } + if (!Number.isFinite(estimate.maxEstimatedCostUsd) || !Number.isSafeInteger(estimate.maxProviderAttempts)) { + throw new LiveModeRefusedError( + "Refusing to start: the requested plan overflows conservative budget arithmetic.", + ); + } + if (estimate.maxEstimatedCostUsd > budgetUsd) { + throw new LiveModeRefusedError( + `Refusing to start: the worst-case estimate for this plan is $${estimate.maxEstimatedCostUsd.toFixed(6)}, ` + + `above the $${budgetUsd.toFixed(6)} budget limit. Reduce --repetitions, select fewer cases, or raise the limit deliberately.`, + ); + } + return true; +} + +/** + * Between-execution budget guard. + * + * Two things make this conservative. It compares against the *worst-case* + * estimate for the next execution rather than the average so far, and it stops + * before starting that execution rather than after discovering the overrun. + * It also accounts for the pipeline's documented limitation that an attempt + * failing before any response body reports no usage, so real spend can exceed + * the reported total: the guard therefore never treats the reported figure as + * an exact ledger. + */ +export function createBudgetGuard({ budgetUsd, model }) { + let spentUsd = 0; + let stoppedReason = null; + + return { + get spentUsd() { + return spentUsd; + }, + get stoppedReason() { + return stoppedReason; + }, + /** @param {number|null} costUsd cost reported by a completed execution */ + record(costUsd) { + if (typeof costUsd === "number" && Number.isFinite(costUsd)) spentUsd += costUsd; + }, + /** + * @param {object} benchmarkCase the case about to be executed + * @returns {boolean} true when the next execution may proceed + */ + canProceed(benchmarkCase) { + if (stoppedReason) return false; + const next = estimateExecutionCost(benchmarkCase, model); + if (next.estimatedCostUsd === null) { + stoppedReason = `Stopped: no recorded pricing for model "${model}", so remaining spend cannot be bounded.`; + return false; + } + if (spentUsd + next.estimatedCostUsd > budgetUsd) { + stoppedReason = + `Stopped before executing ${benchmarkCase.case_id}: $${spentUsd.toFixed(6)} already reported as spent, and the worst case for the next execution ` + + `($${next.estimatedCostUsd.toFixed(6)}) would exceed the $${budgetUsd.toFixed(6)} limit. Reported spend excludes attempts that failed before returning a response body, so true spend may be higher.`; + return false; + } + return true; + }, + }; +} + +/** + * The plan text shown before any request is made. Returned rather than printed + * so it can be asserted in a test without capturing stdout. + */ +export function renderLivePlan({ model, selectedIds, repetitions, estimate, budgetUsd }) { + return [ + "Live evaluation plan — review before continuing:", + ` model: ${model}`, + ` cases: ${selectedIds.length} (${selectedIds.join(", ")})`, + ` repetitions: ${repetitions}`, + ` executions: ${estimate.executionCount ?? "unknown"}`, + ` max provider calls: ${estimate.maxProviderAttempts ?? "unknown"} (worst case, assuming every stage uses its full retry allowance)`, + ` max estimated cost: ${estimate.maxEstimatedCostUsd === null ? "unavailable — model is not in the pricing table" : `$${estimate.maxEstimatedCostUsd.toFixed(6)}`}`, + ` budget limit: $${budgetUsd.toFixed(6)}`, + "", + "The estimate is a worst case, not a quote. OpenAI's billing dashboard is the source of truth.", + ].join("\n"); +} diff --git a/evals/runners/liveRunner.test.js b/evals/runners/liveRunner.test.js new file mode 100644 index 0000000..66052e4 --- /dev/null +++ b/evals/runners/liveRunner.test.js @@ -0,0 +1,315 @@ +/** + * Live-mode safeguard and budget tests. + * + * Every test here drives injected values. None of them constructs an OpenAI + * client, and none of them makes a network request — the point is to prove the + * refusals fire *before* anything reaches the API. + */ +import { describe, it, expect, beforeAll } from "vitest"; + +import { loadBenchmark } from "../datasets/loadBenchmark.js"; +import { PROVIDER_COST_POLICY } from "../../server/pipeline/runPipeline.js"; +import { + assertLiveModeAllowed, + assertBudgetCoversPlan, + estimateExecutionCost, + estimateRunBudget, + resolveBudgetLimit, + createBudgetGuard, + renderLivePlan, + LiveModeRefusedError, + BUDGET_ASSUMPTIONS, + STAGE_OUTPUT_TOKEN_BUDGETS, + MAX_EVALUATION_BUDGET_USD, + MAX_EVALUATION_REPETITIONS, +} from "./liveRunner.js"; + +const PRICED_MODEL = "gpt-5-mini"; +let benchmarkCases; + +beforeAll(async () => { + benchmarkCases = (await loadBenchmark()).cases; +}); + +/** A configuration that passes every guard, so each test can break exactly one. */ +const validOptions = (overrides = {}) => ({ + live: true, + allowCi: false, + caseIds: ["case-001"], + allCases: false, + repetitions: 1, + maxBudgetUsd: "1.00", + env: { OPENAI_API_KEY: "test-key-not-used" }, + benchmarkCases, + ...overrides, +}); + +describe("live mode refusals", () => { + it("refuses without the --live flag", () => { + expect(() => assertLiveModeAllowed(validOptions({ live: false }))).toThrow( + /requires the explicit --live flag/, + ); + }); + + it("refuses without an API key", () => { + expect(() => assertLiveModeAllowed(validOptions({ env: {} }))).toThrow(/OPENAI_API_KEY/); + }); + + it("refuses in CI by default", () => { + expect(() => + assertLiveModeAllowed(validOptions({ env: { OPENAI_API_KEY: "k", CI: "true" } })), + ).toThrow(/refuses to run in CI by default/); + }); + + it("permits CI only with the explicit escape hatch", () => { + expect(() => + assertLiveModeAllowed( + validOptions({ env: { OPENAI_API_KEY: "k", CI: "true" }, allowCi: true }), + ), + ).not.toThrow(); + }); + + it("treats CI=false as a CI environment declaring itself", () => { + expect(() => + assertLiveModeAllowed(validOptions({ env: { OPENAI_API_KEY: "k", CI: "false" } })), + ).not.toThrow(); + }); + + it("refuses without a budget limit", () => { + expect(() => assertLiveModeAllowed(validOptions({ maxBudgetUsd: undefined }))).toThrow( + /requires an explicit budget limit/, + ); + }); + + it("refuses a zero, negative, or non-numeric budget", () => { + for (const value of ["0", "-1", "abc", "Infinity"]) { + expect(() => assertLiveModeAllowed(validOptions({ maxBudgetUsd: value })), value).toThrow( + /positive, finite number/, + ); + } + }); + + it("refuses budgets above the per-command safety cap", () => { + expect(() => resolveBudgetLimit({ maxBudgetUsd: "100.01" })).toThrow(/safety cap/); + expect(MAX_EVALUATION_BUDGET_USD).toBe(100); + }); + + it("accepts a budget from the environment when no flag is given", () => { + const resolved = assertLiveModeAllowed( + validOptions({ + maxBudgetUsd: undefined, + env: { OPENAI_API_KEY: "k", EVAL_MAX_BUDGET_USD: "0.5" }, + }), + ); + expect(resolved.budgetUsd).toBe(0.5); + }); + + it("prefers an explicit flag over the environment", () => { + expect(resolveBudgetLimit({ maxBudgetUsd: "2", env: { EVAL_MAX_BUDGET_USD: "9" } })).toBe(2); + }); + + it("refuses when no case is selected", () => { + expect(() => assertLiveModeAllowed(validOptions({ caseIds: [] }))).toThrow( + /never defaults to the whole benchmark/, + ); + }); + + it("requires an explicit flag to run the whole benchmark", () => { + const resolved = assertLiveModeAllowed(validOptions({ caseIds: [], allCases: true })); + expect(resolved.selectedIds).toHaveLength(benchmarkCases.length); + }); + + it("refuses both --case and --all-cases together", () => { + expect(() => assertLiveModeAllowed(validOptions({ allCases: true }))).toThrow(/not both/); + }); + + it("refuses an unknown case id", () => { + expect(() => assertLiveModeAllowed(validOptions({ caseIds: ["case-999"] }))).toThrow( + /Unknown case id/, + ); + }); + + it("refuses a non-integer or non-positive repetition count", () => { + for (const value of [0, -1, 1.5, 21, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => assertLiveModeAllowed(validOptions({ repetitions: value })), value).toThrow( + /--repetitions/, + ); + } + }); + + it("caps repetitions at the documented safety limit", () => { + expect(MAX_EVALUATION_REPETITIONS).toBe(20); + }); + + it("defaults to a single repetition and a single selected case", () => { + const resolved = assertLiveModeAllowed(validOptions()); + expect(resolved.repetitions).toBe(1); + expect(resolved.selectedIds).toEqual(["case-001"]); + }); + + it("throws a typed error so callers can distinguish a refusal from a crash", () => { + try { + assertLiveModeAllowed(validOptions({ live: false })); + expect.unreachable(); + } catch (error) { + expect(error).toBeInstanceOf(LiveModeRefusedError); + } + }); +}); + +describe("budget estimation", () => { + it("derives its production-cost policy rather than copying constants", () => { + expect(STAGE_OUTPUT_TOKEN_BUDGETS).toBe(PROVIDER_COST_POLICY.outputTokenBudgets); + expect(BUDGET_ASSUMPTIONS.providerAttemptsPerRequest).toBe(PROVIDER_COST_POLICY.maxProviderAttemptsPerRequest); + expect(BUDGET_ASSUMPTIONS.batchIntegrityPasses).toBe(PROVIDER_COST_POLICY.maxBatchIntegrityExecutions); + }); + it("estimates more for a pairing case than a non-pairing case", () => { + const pairing = benchmarkCases.find((entry) => entry.deterministic_expectations.pairing_enabled); + const plain = benchmarkCases.find( + (entry) => !entry.deterministic_expectations.pairing_enabled && entry.input.candidates.length === 3, + ); + expect(estimateExecutionCost(pairing, PRICED_MODEL).estimatedCostUsd).toBeGreaterThan( + estimateExecutionCost(plain, PRICED_MODEL).estimatedCostUsd, + ); + }); + + it("includes adapter retries, batch-integrity retries, and truncation headroom", () => { + const plain = benchmarkCases.find((entry) => !entry.deterministic_expectations.pairing_enabled); + // Context and decision each permit two provider attempts; scoring can make + // two integrity passes, each of which can make two provider attempts. + expect(estimateExecutionCost(plain, PRICED_MODEL).maxAttempts).toBe( + BUDGET_ASSUMPTIONS.providerAttemptsPerRequest * + (1 + BUDGET_ASSUMPTIONS.batchIntegrityPasses + 1), + ); + }); + + it("includes a separately retryable pairing batch when pairing is enabled", () => { + const pairing = benchmarkCases.find((entry) => entry.deterministic_expectations.pairing_enabled); + expect(estimateExecutionCost(pairing, PRICED_MODEL).maxAttempts).toBe( + BUDGET_ASSUMPTIONS.providerAttemptsPerRequest * + (1 + BUDGET_ASSUMPTIONS.batchIntegrityPasses * 2 + 1), + ); + }); + + it("uses an injected policy fixture rather than ignoring policy changes", () => { + const plain = benchmarkCases.find((entry) => !entry.deterministic_expectations.pairing_enabled); + const changed = { ...PROVIDER_COST_POLICY, outputTokenBudgets: { ...PROVIDER_COST_POLICY.outputTokenBudgets, contextAnalysis: 9999 } }; + expect(estimateExecutionCost(plain, PRICED_MODEL, { policy: changed }).outputTokens).toBeGreaterThan( + estimateExecutionCost(plain, PRICED_MODEL).outputTokens, + ); + }); + + it("scales with repetitions and scenario count", () => { + const multi = benchmarkCases.find((entry) => entry.input.scenarios.length === 2); + const single = estimateRunBudget([multi], PRICED_MODEL, 1); + const doubled = estimateRunBudget([multi], PRICED_MODEL, 2); + expect(single.executionCount).toBe(2); + expect(doubled.executionCount).toBe(4); + expect(doubled.maxEstimatedCostUsd).toBeCloseTo(single.maxEstimatedCostUsd * 2, 6); + }); + + it("reports an unpriced model rather than guessing a price", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], "not-a-real-model", 1); + expect(estimate.priced).toBe(false); + expect(estimate.maxEstimatedCostUsd).toBeNull(); + }); + + it("refuses to start when the worst case exceeds the budget", () => { + const estimate = estimateRunBudget(benchmarkCases, PRICED_MODEL, 5); + expect(() => assertBudgetCoversPlan(estimate, 0.0001)).toThrow(/above the .* budget limit/); + }); + + it("permits a plan that fits inside the budget", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], PRICED_MODEL, 1); + expect(assertBudgetCoversPlan(estimate, 100)).toBe(true); + }); + + it("refuses an unpriced model outright, because a budget cannot be enforced", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], "not-a-real-model", 1); + expect(() => assertBudgetCoversPlan(estimate, 100)).toThrow(/No recorded pricing/); + }); +}); + +describe("budget guard", () => { + it("permits execution while the worst case fits", () => { + const guard = createBudgetGuard({ budgetUsd: 100, model: PRICED_MODEL }); + expect(guard.canProceed(benchmarkCases[0])).toBe(true); + expect(guard.stoppedReason).toBeNull(); + }); + + it("stops before starting an execution that could exceed the limit", () => { + const guard = createBudgetGuard({ budgetUsd: 0.02, model: PRICED_MODEL }); + guard.record(0.019); + expect(guard.canProceed(benchmarkCases[0])).toBe(false); + expect(guard.stoppedReason).toContain("would exceed the"); + }); + + it("stays stopped once it has stopped", () => { + const guard = createBudgetGuard({ budgetUsd: 0.000001, model: PRICED_MODEL }); + expect(guard.canProceed(benchmarkCases[0])).toBe(false); + const reason = guard.stoppedReason; + expect(guard.canProceed(benchmarkCases[0])).toBe(false); + expect(guard.stoppedReason).toBe(reason); + }); + + it("stops when the model has no recorded pricing", () => { + const guard = createBudgetGuard({ budgetUsd: 100, model: "not-a-real-model" }); + expect(guard.canProceed(benchmarkCases[0])).toBe(false); + expect(guard.stoppedReason).toContain("no recorded pricing"); + }); + + it("accumulates only real reported spend", () => { + const guard = createBudgetGuard({ budgetUsd: 100, model: PRICED_MODEL }); + guard.record(0.01); + guard.record(null); + guard.record(Number.NaN); + guard.record(0.02); + expect(guard.spentUsd).toBeCloseTo(0.03, 10); + }); + + it("warns that reported spend can under-report true spend", () => { + const guard = createBudgetGuard({ budgetUsd: 0.001, model: PRICED_MODEL }); + guard.record(0.001); + guard.canProceed(benchmarkCases[0]); + expect(guard.stoppedReason).toContain("true spend may be higher"); + }); +}); + +describe("live plan disclosure", () => { + it("shows the model, cases, repetitions, worst-case calls, cost, and budget", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], PRICED_MODEL, 1); + const plan = renderLivePlan({ + model: PRICED_MODEL, + selectedIds: ["case-001"], + repetitions: 1, + estimate, + budgetUsd: 0.25, + }); + expect(plan).toContain(PRICED_MODEL); + expect(plan).toContain("case-001"); + expect(plan).toContain("repetitions: 1"); + expect(plan).toContain("max provider calls:"); + expect(plan).toContain("max estimated cost:"); + expect(plan).toContain("budget limit: $0.250000"); + expect(plan).toContain("worst case, not a quote"); + }); + + it("says so plainly when the model is unpriced", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], "not-a-real-model", 1); + const plan = renderLivePlan({ + model: "not-a-real-model", + selectedIds: ["case-001"], + repetitions: 1, + estimate, + budgetUsd: 1, + }); + expect(plan).toContain("not in the pricing table"); + }); + + it("contains no ANSI escape codes", () => { + const estimate = estimateRunBudget([benchmarkCases[0]], PRICED_MODEL, 1); + const plan = renderLivePlan({ model: PRICED_MODEL, selectedIds: ["case-001"], repetitions: 1, estimate, budgetUsd: 1 }); + // eslint-disable-next-line no-control-regex + expect(/\u001b\[/.test(plan)).toBe(false); + }); +}); diff --git a/evals/runners/observingProvider.js b/evals/runners/observingProvider.js new file mode 100644 index 0000000..4c5e57b --- /dev/null +++ b/evals/runners/observingProvider.js @@ -0,0 +1,57 @@ +/** + * @file Provider request observer (Phase 3A evaluation harness). + * + * Wraps any `AIProvider` (fake or real) and records *what the pipeline asked + * for* — never what came back, never credentials, never raw payloads. Two + * graders depend on this: + * + * - candidate coverage, to confirm the scoring stage requested exactly the + * submitted candidate set; + * - pairing integrity, to confirm every expected unordered pair was + * evaluated. The response only exposes the top three pairs, so pair + * coverage genuinely cannot be verified from the response alone. + * + * Only derived identifiers are kept (candidate IDs, canonical pair keys, per- + * stage attempt counts). Prompt text, system text, response bodies, headers, + * and API keys are never retained, so a trace is safe to write into a run + * artifact. + */ +import { canonicalPairKey } from "../schemas/benchmarkCase.js"; + +const CANDIDATE_ID_PATTERN = /candidate_id: (\S+)\nName:/g; +const PAIR_PATTERN = /candidate_id_a: ([^,\s]+), candidate_id_b: ([^,)\s]+)/g; + +/** + * @param {{ name: string, generateStructured: Function }} inner + * @returns {{ provider: object, trace: { requestedCandidateIds: string[]|null, requestedPairKeys: string[]|null, attemptsByStage: Record } }} + */ +export function createObservingProvider(inner) { + const trace = { + requestedCandidateIds: null, + requestedPairKeys: null, + attemptsByStage: {}, + }; + + const provider = { + name: inner.name, + model: inner.model, + async generateStructured(request) { + trace.attemptsByStage[request.promptId] = (trace.attemptsByStage[request.promptId] ?? 0) + 1; + + if (request.promptId === "batch-candidate-scoring" && trace.requestedCandidateIds === null) { + trace.requestedCandidateIds = [ + ...request.prompt.matchAll(CANDIDATE_ID_PATTERN), + ].map((match) => match[1]); + } + if (request.promptId === "batch-pairing-analysis" && trace.requestedPairKeys === null) { + trace.requestedPairKeys = [...request.prompt.matchAll(PAIR_PATTERN)].map((match) => + canonicalPairKey(match[1], match[2]), + ); + } + + return inner.generateStructured(request); + }, + }; + + return { provider, trace }; +} diff --git a/evals/runners/runBenchmark.js b/evals/runners/runBenchmark.js new file mode 100644 index 0000000..e2014c1 --- /dev/null +++ b/evals/runners/runBenchmark.js @@ -0,0 +1,428 @@ +/** + * @file Benchmark runner (Phase 3A evaluation harness). + * + * Orchestrates a whole benchmark run: case selection, execution, grading, + * stability accounting, permutation analysis, and summary assembly. It is + * transport-agnostic — writing artifacts is the reporters' job, and choosing a + * provider is the caller's job, which is what keeps fixture mode and live mode + * on exactly the same code path. + */ +import { execFileSync } from "node:child_process"; + +import { EVALUATION_RUN_SCHEMA_VERSION } from "../schemas/evaluationRun.js"; +import { graderVersions, GRADER_SUITE_VERSION } from "../graders/deterministicGraders.js"; +import { runCase, structuredEvidenceFingerprint, explanationFingerprint } from "./runCase.js"; + +export { GRADER_SUITE_VERSION }; + +/** + * Carried into every artifact so a report can never be read as a stronger + * claim than it is. + */ +export const RUN_DISCLAIMER = + "This is a development benchmark result. It is not scientifically validated, not representative of real hiring decisions, not evidence of fairness or demographic neutrality, not a legal-compliance test, not a calibrated-confidence benchmark, and not a production service-level objective. All benchmark data is synthetic."; + +/** + * Reads the current commit and branch. Returns nulls outside a git checkout + * rather than failing the run — the harness is useful in a plain directory, + * it just cannot say which commit produced the numbers. + * @param {string} cwd + */ +export function readGitContext(cwd = process.cwd()) { + const read = (args) => { + try { + return execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || null; + } catch { + return null; + } + }; + return { commit: read(["rev-parse", "HEAD"]), branch: read(["rev-parse", "--abbrev-ref", "HEAD"]) }; +} + +/** + * Winner and ranking agreement across repetitions of the same case+scenario. + * + * With a single repetition this returns `assessed: false` and null agreements. + * Reporting "100% stable" from one sample would be exactly the kind of + * unsupported claim this phase exists to prevent. + * @param {object[]} caseResults + * @param {number} repetitions + */ +export function computeStability(caseResults, repetitions) { + if (repetitions < 2) { + return { + assessed: false, + reason: `Stability was not assessed: ${repetitions} repetition per case is not enough to observe run-to-run variation.`, + winner_agreement: null, + ranking_agreement: null, + }; + } + + const groups = new Map(); + for (const caseResult of caseResults) { + for (const execution of caseResult.executions) { + if (execution.status !== "completed") continue; + const key = `${execution.case_id}#${execution.scenario_index}`; + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(execution.outcome); + } + } + + const comparable = [...groups.values()].filter((outcomes) => outcomes.length > 1); + if (comparable.length === 0) { + return { + assessed: false, + reason: "Stability was not assessed: no case+scenario produced more than one completed execution.", + winner_agreement: null, + ranking_agreement: null, + }; + } + + const agreeing = (outcomes, pick) => + outcomes.every((outcome) => JSON.stringify(pick(outcome)) === JSON.stringify(pick(outcomes[0]))); + + const winnerAgreeing = comparable.filter((outcomes) => agreeing(outcomes, (o) => o.winner_id)).length; + const rankingAgreeing = comparable.filter((outcomes) => agreeing(outcomes, (o) => o.ranking)).length; + + return { + assessed: true, + reason: `Assessed across ${comparable.length} case+scenario group(s) at ${repetitions} repetitions each.`, + winner_agreement: Number((winnerAgreeing / comparable.length).toFixed(4)), + ranking_agreement: Number((rankingAgreeing / comparable.length).toFixed(4)), + }; +} + +/** + * Compares each variant case against the original it was derived from. + * + * Reported, never gated: a difference here is information, not automatically a + * defect. Under the offline fixture a wording variant cannot differ at all + * (the fixture ignores description text), so a green result here says nothing + * about a real model's sensitivity to wording. + * @param {object[]} caseResults + */ +export function analysePermutations(caseResults) { + const byId = new Map(caseResults.map((caseResult) => [caseResult.case_id, caseResult])); + const findings = []; + + for (const caseResult of caseResults) { + if (!caseResult.variant_of) continue; + const original = byId.get(caseResult.variant_of); + if (!original) { + findings.push({ + case_id: caseResult.case_id, + variant_of: caseResult.variant_of, + variant_kind: caseResult.variant_kind, + compared: false, + reason: "The original case was not part of this run's case selection.", + winner_changed: null, + ranking_changed: null, + best_pair_changed: null, + structured_evidence_changed: null, + explanation_changed: null, + }); + continue; + } + + // Scenarios are matched by their text first, so a scenario-order variant + // compares like with like regardless of position. + const originalCompleted = original.executions.filter( + (execution) => execution.status === "completed", + ); + const originalByScenario = new Map( + originalCompleted.map((execution) => [execution.scenario, execution]), + ); + // Index fallback for variants whose scenario text legitimately differs + // (a wording variant may reword the scenario too). Never applied to a + // scenario-order variant, where position is precisely what changed and + // matching by it would compare the wrong pairs. + const originalByIndex = new Map( + originalCompleted.map((execution) => [execution.scenario_index, execution]), + ); + const matchFor = (execution) => + originalByScenario.get(execution.scenario) ?? + (caseResult.variant_kind === "scenario-order" + ? undefined + : originalByIndex.get(execution.scenario_index)); + + let compared = 0; + let winnerChanged = false; + let rankingChanged = false; + let pairChanged = false; + let evidenceChanged = false; + let explanationChanged = false; + + for (const execution of caseResult.executions) { + if (execution.status !== "completed") continue; + const counterpart = matchFor(execution); + if (!counterpart) continue; + compared += 1; + + if (execution.outcome.winner_id !== counterpart.outcome.winner_id) winnerChanged = true; + if (JSON.stringify(execution.outcome.ranking) !== JSON.stringify(counterpart.outcome.ranking)) { + rankingChanged = true; + } + if (execution.outcome.best_pair_key !== counterpart.outcome.best_pair_key) pairChanged = true; + if ( + structuredEvidenceFingerprint(execution.response) !== + structuredEvidenceFingerprint(counterpart.response) + ) { + evidenceChanged = true; + } + if ( + explanationFingerprint(execution.response) !== explanationFingerprint(counterpart.response) + ) { + explanationChanged = true; + } + } + + findings.push( + compared === 0 + ? { + case_id: caseResult.case_id, + variant_of: caseResult.variant_of, + variant_kind: caseResult.variant_kind, + compared: false, + reason: "No scenario matched between the variant and its original.", + winner_changed: null, + ranking_changed: null, + best_pair_changed: null, + structured_evidence_changed: null, + explanation_changed: null, + } + : { + case_id: caseResult.case_id, + variant_of: caseResult.variant_of, + variant_kind: caseResult.variant_kind, + compared: true, + reason: `Compared ${compared} matching scenario execution(s).`, + winner_changed: winnerChanged, + ranking_changed: rankingChanged, + best_pair_changed: pairChanged, + structured_evidence_changed: evidenceChanged, + explanation_changed: explanationChanged, + }, + ); + } + + return findings; +} + +function aggregateGraderTotals(caseResults) { + const totals = {}; + const record = (result) => { + if (!totals[result.grader_id]) { + totals[result.grader_id] = { + pass: 0, + fail: 0, + skip: 0, + error: 0, + expected_failure: 0, + severity: result.severity, + }; + } + totals[result.grader_id][result.status] += 1; + }; + for (const caseResult of caseResults) { + for (const execution of caseResult.executions) execution.grader_results.forEach(record); + caseResult.grader_results.forEach(record); + } + return totals; +} + +/** + * Runs a benchmark end to end. + * + * @param {object} options + * @param {object} options.benchmark loaded via evals/datasets/loadBenchmark.js + * @param {string[]} options.caseIds cases to execute, in run order + * @param {"fixtures"|"live"} options.mode + * @param {(context: object) => object} options.createProvider provider factory + * @param {string} options.provider provider label for the manifest + * @param {string} options.model model label for the manifest + * @param {number} [options.repetitions] + * @param {string} [options.runId] + * @param {(execution: object) => void} [options.onExecution] budget hook + * @param {(context: object) => boolean} [options.beforeExecution] live budget gate + * @param {(caseResult: object) => void} [options.onCase] progress hook + */ +export async function runBenchmark({ + benchmark, + caseIds, + mode, + createProvider, + provider, + model, + repetitions = 1, + runId, + onExecution, + beforeExecution, + onCase, +}) { + const selected = caseIds.map((caseId) => { + const benchmarkCase = benchmark.cases.find((entry) => entry.case_id === caseId); + if (!benchmarkCase) { + throw new Error( + `Case "${caseId}" is not part of benchmark "${benchmark.manifest.benchmark_id}".`, + ); + } + return benchmarkCase; + }); + + const startedAt = Date.now(); + const timestamp = new Date().toISOString(); + const resolvedRunId = runId ?? `${benchmark.manifest.benchmark_id}-${mode}-${timestamp.replace(/[:.]/g, "-")}`; + const git = readGitContext(); + + const caseResults = []; + for (const benchmarkCase of selected) { + const caseResult = await runCase({ + benchmarkCase, + createProvider, + model, + repetitions, + beforeExecution, + onExecution, + }); + caseResults.push(caseResult); + onCase?.(caseResult); + } + const durationMs = Date.now() - startedAt; + + const executions = caseResults.flatMap((caseResult) => caseResult.executions); + const completed = executions.filter((execution) => execution.status === "completed"); + const sum = (pick) => completed.reduce((total, execution) => total + pick(execution.response.run_metadata), 0); + + const anyCostKnown = completed.some( + (execution) => execution.response.run_metadata.estimatedCostUsd !== null, + ); + const estimatedCostUsd = anyCostKnown + ? Number( + completed + .reduce((total, execution) => total + (execution.response.run_metadata.estimatedCostUsd ?? 0), 0) + .toFixed(6), + ) + : null; + + const requiredFailures = caseResults.reduce((total, caseResult) => total + caseResult.required_failures, 0); + const advisoryFailures = caseResults.reduce((total, caseResult) => total + caseResult.advisory_failures, 0); + const expectedFailures = caseResults.reduce( + (total, caseResult) => + total + + [...caseResult.executions.flatMap((execution) => execution.grader_results), ...caseResult.grader_results].filter( + (result) => result.status === "expected_failure", + ).length, + 0, + ); + const allGraderResults = caseResults.flatMap((caseResult) => [ + ...caseResult.executions.flatMap((execution) => execution.grader_results), + ...caseResult.grader_results, + ]); + const unexpectedDefectResolutions = allGraderResults.filter( + (result) => result.unexpected_defect_resolution, + ).length; + const unexpectedFailures = Math.max(0, requiredFailures - unexpectedDefectResolutions); + const expectedResults = caseResults.flatMap((caseResult) => + caseResult.executions.flatMap((execution) => + execution.grader_results + .filter((result) => result.status === "expected_failure") + .map((result) => ({ execution, result })), + ), + ); + const affectedDefectIds = [...new Set(expectedResults.map(({ result }) => result.known_defect_id).filter(Boolean))].sort(); + const affectedExecutionIds = [...new Set(expectedResults.map(({ execution }) => execution.execution_id))].sort(); + const knownDefectObservations = expectedResults.flatMap(({ execution, result }) => + result.observations.map((signature) => ({ + defect_id: result.known_defect_id, + case_id: execution.case_id, + execution_id: execution.execution_id, + scenario_id: `scenario-${execution.scenario_index + 1}`, + variant_id: caseResults.find((entry) => entry.case_id === execution.case_id)?.variant_kind ?? null, + repetition: execution.repetition, + grader_id: result.grader_id, + signature, + })), + ); + // A genuine failure is never hidden by a simultaneous XPASS/baseline + // change. Review still receives the disappeared observation separately. + const runState = requiredFailures > 0 + ? "unexpected_failure" + : unexpectedDefectResolutions > 0 + ? "baseline_change_required" + : expectedFailures > 0 + ? "pass_with_known_defects" + : "clean_pass"; + + const manifest = { + run_id: resolvedRunId, + schema_version: EVALUATION_RUN_SCHEMA_VERSION, + timestamp, + mode, + benchmark_id: benchmark.manifest.benchmark_id, + benchmark_version: benchmark.manifest.benchmark_version, + benchmark_schema_version: benchmark.manifest.schema_version, + rubric_version: benchmark.rubric.rubric_version, + git_commit: git.commit, + git_branch: git.branch, + provider, + model, + case_selection: caseIds, + repetitions, + pairing_cases: selected.filter((entry) => entry.deterministic_expectations.pairing_enabled).length, + logical_provider_stages: completed.length === 0 ? 0 : sum((meta) => meta.logicalProviderStageCount), + provider_attempts: completed.length === 0 ? 0 : sum((meta) => meta.providerAttemptCount), + input_tokens: completed.length === 0 ? 0 : sum((meta) => meta.inputTokens), + output_tokens: completed.length === 0 ? 0 : sum((meta) => meta.outputTokens), + total_tokens: completed.length === 0 ? 0 : sum((meta) => meta.totalTokens), + estimated_cost_usd: estimatedCostUsd, + duration_ms: durationMs, + grader_versions: { ...graderVersions(), "grader-suite": GRADER_SUITE_VERSION }, + artifact_policy: { + synthetic_data_only: true, + secrets_recorded: false, + absolute_paths_recorded: false, + }, + }; + + const summary = { + run_id: resolvedRunId, + schema_version: EVALUATION_RUN_SCHEMA_VERSION, + benchmark_id: benchmark.manifest.benchmark_id, + benchmark_version: benchmark.manifest.benchmark_version, + mode, + git_commit: git.commit, + repetitions, + case_count: caseResults.length, + execution_count: executions.length, + passed_cases: caseResults.filter((caseResult) => caseResult.passed).length, + failed_cases: caseResults.filter((caseResult) => !caseResult.passed).length, + required_failures: requiredFailures, + advisory_failures: advisoryFailures, + run_state: runState, + grader_totals: aggregateGraderTotals(caseResults), + expected_failures: expectedFailures, + clean_pass_count: caseResults.filter((caseResult) => caseResult.passed && !caseResult.executions.some((execution) => execution.grader_results.some((result) => result.status === "expected_failure"))).length, + affected_defect_ids: affectedDefectIds, + affected_execution_ids: affectedExecutionIds, + unexpected_failures: unexpectedFailures, + unexpected_defect_resolutions: unexpectedDefectResolutions, + known_defect_observations: knownDefectObservations, + stability: computeStability(caseResults, repetitions), + totals: { + logical_provider_stages: manifest.logical_provider_stages, + provider_attempts: manifest.provider_attempts, + total_tokens: manifest.total_tokens, + estimated_cost_usd: manifest.estimated_cost_usd, + duration_ms: durationMs, + }, + disclaimer: RUN_DISCLAIMER, + }; + + return { + manifest, + summary, + caseResults, + permutations: analysePermutations(caseResults), + passed: requiredFailures === 0, + }; +} diff --git a/evals/runners/runCase.js b/evals/runners/runCase.js new file mode 100644 index 0000000..322b616 --- /dev/null +++ b/evals/runners/runCase.js @@ -0,0 +1,266 @@ +/** + * @file Single-case runner (Phase 3A evaluation harness). + * + * Executes one benchmark case against the *real* production pipeline + * (`server/pipeline/runPipeline.js`) — real prompts, real schemas, real + * deterministic scoring, real batch-identity validation — and grades the + * result. + * + * A case with N scenarios and R repetitions produces N*R executions. They are + * never collapsed: keeping them separate is what makes per-scenario behaviour + * and run-to-run stability visible at all. + * + * This module imports production code. Production code never imports this + * module, and a repository-protection test enforces that direction. + */ +import { runPipeline } from "../../server/pipeline/runPipeline.js"; +import { DECISION_INPUT_LIMITS } from "../../shared/contracts/decisionInputLimits.js"; +import { DEFAULT_AI_MAX_CANDIDATES } from "../../server/config/env.js"; +import { createObservingProvider } from "./observingProvider.js"; +import { + EXECUTION_GRADERS, + CASE_GRADERS, + runGraders, + countFailures, + applyKnownDefects, + checkKnownDefectsStillReproduce, +} from "../graders/deterministicGraders.js"; +import { canonicalPairKey } from "../schemas/benchmarkCase.js"; + +/** + * The harness raises the per-run candidate cap to whatever a case actually + * submits, still bounded by the shared technical ceiling. `AI_MAX_CANDIDATES` + * is a deployment budget choice, not a property of the benchmark, and letting + * a local budget setting silently drop benchmark cases would make results + * incomparable between machines. + * @param {object} benchmarkCase + */ +export function resolveMaxCandidatesForCase(benchmarkCase) { + return Math.min( + DECISION_INPUT_LIMITS.candidates.max, + Math.max(DEFAULT_AI_MAX_CANDIDATES, benchmarkCase.input.candidates.length), + ); +} + +/** Builds the production request shape for one scenario of a case. */ +export function buildEvaluationRequest(benchmarkCase, scenarioIndex) { + return { + role: benchmarkCase.input.role, + scenario: benchmarkCase.input.scenarios[scenarioIndex], + decision_mode: benchmarkCase.input.decision_mode, + candidates: benchmarkCase.input.candidates, + options: benchmarkCase.input.options, + }; +} + +/** + * The subset of a response the comparison command diffs. Deliberately excludes + * `request_id`, timestamps, and stage durations: those differ between any two + * runs and would make every comparison look like a change. + */ +function extractOutcome(response, durationMs) { + const ranked = [...response.candidate_evaluations].sort((a, b) => a.rank - b.rank); + const pairing = response.pairing_result; + return { + winner_id: response.decision_result.recommended_candidate_id, + ranking: ranked.map((candidate) => candidate.candidate_id), + best_pair_key: + pairing?.status === "ok" + ? canonicalPairKey(pairing.best_pair.candidate_id_a, pairing.best_pair.candidate_id_b) + : null, + pairing_status: pairing ? pairing.status : "absent", + logical_provider_stage_count: response.run_metadata.logicalProviderStageCount, + provider_attempt_count: response.run_metadata.providerAttemptCount, + total_tokens: response.run_metadata.totalTokens, + estimated_cost_usd: response.run_metadata.estimatedCostUsd, + duration_ms: durationMs, + }; +} + +/** + * Structured evidence fingerprint used by the permutation comparison. It + * covers the parts of the result a permutation must not change — scores, + * confidences, risk profiles — while excluding narrative text, which is + * compared separately. + */ +export function structuredEvidenceFingerprint(response) { + return JSON.stringify( + [...response.candidate_evaluations] + .sort((a, b) => a.candidate_id.localeCompare(b.candidate_id)) + .map((candidate) => ({ + id: candidate.candidate_id, + weighted_fit_score: candidate.weighted_fit_score, + risk_adjusted_score: candidate.risk_adjusted_score, + expected_outcome_score: candidate.expected_outcome_score, + overall_confidence: candidate.overall_confidence, + criteria: Object.fromEntries( + Object.entries(candidate.criteria_scores) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => [key, [value.score, value.confidence]]), + ), + risk_profile: candidate.risk_profile, + })), + ); +} + +export function explanationFingerprint(response) { + return JSON.stringify({ + key_reason: response.decision_result.key_reason, + executive_interpretation: response.decision_result.executive_interpretation, + executive_summary: response.executive_summary, + trade_offs: response.trade_offs, + }); +} + +/** + * Executes and grades one benchmark case. + * + * @param {object} options + * @param {object} options.benchmarkCase validated benchmark case + * @param {(context: { benchmarkCase: object, scenarioIndex: number, repetition: number }) => object} options.createProvider + * Returns a provider implementing the production contract. In fixture mode + * this builds an offline fake; in live mode it returns the single real + * provider instance the caller resolved once. + * @param {string} options.model model label recorded in run metadata + * @param {number} [options.repetitions] + * @param {(context: { benchmarkCase: object, scenarioIndex: number, repetition: number }) => boolean} [options.beforeExecution] + * @param {(execution: object) => void} [options.onExecution] called after each + * execution, so a live runner can enforce its budget between executions. + * @returns {Promise} a case result matching `caseResultSchema` + */ +export async function runCase({ + benchmarkCase, + createProvider, + model, + repetitions = 1, + beforeExecution, + onExecution, +}) { + const executions = []; + const maxCandidates = resolveMaxCandidatesForCase(benchmarkCase); + + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + for (let scenarioIndex = 0; scenarioIndex < benchmarkCase.input.scenarios.length; scenarioIndex += 1) { + const scenario = benchmarkCase.input.scenarios[scenarioIndex]; + const executionId = `${benchmarkCase.case_id}#s${scenarioIndex}#r${repetition}`; + if (beforeExecution && !beforeExecution({ benchmarkCase, scenarioIndex, repetition })) { + const execution = { + execution_id: executionId, + case_id: benchmarkCase.case_id, + scenario_index: scenarioIndex, + scenario, + repetition, + status: "skipped", + skip_reason: "Execution was not started because the live budget guard would be exceeded.", + grader_results: [ + { + grader_id: "execution-completion", + grader_version: "1.0.0", + severity: "required", + status: "fail", + summary: "The execution was not started.", + finding_codes: [], + details: ["The live budget guard stopped this execution before any provider request."], + findings: [{ kind: "budget_guard", code: "not_started", message: "The live budget guard stopped this execution before any provider request." }], + }, + ], + }; + executions.push(execution); + onExecution?.(execution); + continue; + } + const stageSnapshots = []; + const { provider, trace } = createObservingProvider( + createProvider({ benchmarkCase, scenarioIndex, repetition }), + ); + + const startedAt = Date.now(); + let response; + let failureReason; + try { + response = await runPipeline( + provider, + model, + buildEvaluationRequest(benchmarkCase, scenarioIndex), + (stages) => stageSnapshots.push(stages.map((stage) => ({ ...stage }))), + { maxCandidates }, + ); + } catch (error) { + // Only the safe message is kept. A stack trace can contain absolute + // paths, and a provider error can carry payload fragments — neither + // belongs in an artifact. + failureReason = error.message; + } + const durationMs = Date.now() - startedAt; + + const execution = { + execution_id: executionId, + case_id: benchmarkCase.case_id, + scenario_index: scenarioIndex, + scenario, + repetition, + status: response ? "completed" : "failed", + ...(response ? { response } : {}), + ...(failureReason ? { failure_reason: failureReason } : {}), + ...(response ? { outcome: extractOutcome(response, durationMs) } : {}), + grader_results: [], + }; + + execution.grader_results = response + ? applyKnownDefects( + runGraders(EXECUTION_GRADERS, { + benchmarkCase, + execution, + response, + stageSnapshots, + trace, + repetitions, + }), + benchmarkCase.known_defects, + { execution, benchmarkCase }, + ) + : [ + { + grader_id: "execution-completion", + grader_version: "1.0.0", + severity: "required", + status: "fail", + summary: "The pipeline did not produce a completed response for this execution.", + finding_codes: [], + details: [failureReason ?? "unknown failure"], + findings: [{ kind: "execution_error", code: "pipeline_failed", message: failureReason ?? "unknown failure" }], + }, + ]; + + executions.push(execution); + onExecution?.(execution); + } + } + + const caseGraderResults = applyKnownDefects( + runGraders(CASE_GRADERS, { benchmarkCase, executions, repetitions }), + benchmarkCase.known_defects, + ); + caseGraderResults.push( + ...checkKnownDefectsStillReproduce(executions, caseGraderResults, benchmarkCase.known_defects), + ); + + const allResults = [ + ...executions.flatMap((execution) => execution.grader_results), + ...caseGraderResults, + ]; + const { required, advisory } = countFailures(allResults); + + return { + case_id: benchmarkCase.case_id, + title: benchmarkCase.title, + tags: [...benchmarkCase.tags], + variant_of: benchmarkCase.variant_of, + variant_kind: benchmarkCase.variant_kind, + executions, + grader_results: caseGraderResults, + required_failures: required, + advisory_failures: advisory, + passed: required === 0, + }; +} diff --git a/evals/runners/runner.test.js b/evals/runners/runner.test.js new file mode 100644 index 0000000..4ae1d15 --- /dev/null +++ b/evals/runners/runner.test.js @@ -0,0 +1,465 @@ +/** + * Runner tests: fixture mode, offline guarantees, artifacts, and stability. + */ +import { describe, it, expect, beforeAll, afterEach } from "vitest"; +import { mkdtemp, readFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { loadBenchmark } from "../datasets/loadBenchmark.js"; +import { createEvalFakeProvider, FAKE_PROVIDER_PROFILES } from "../fixtures/fakeProviderProfiles.js"; +import { runCase, resolveMaxCandidatesForCase, buildEvaluationRequest } from "./runCase.js"; +import { runBenchmark, computeStability, analysePermutations, readGitContext } from "./runBenchmark.js"; +import { writeRunArtifacts, readRunArtifacts } from "../reporters/jsonReporter.js"; +import { renderRunMarkdown, renderConsoleSummary } from "../reporters/markdownReporter.js"; +import { buildHumanReviewTemplate } from "../graders/rubricTemplate.js"; + +let benchmark; +let caseById; + +beforeAll(async () => { + benchmark = await loadBenchmark(); + caseById = new Map(benchmark.cases.map((entry) => [entry.case_id, entry])); +}); + +const fixtureProvider = + (benchmarkCase, profile) => + ({ scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile }); + +async function runOne(caseId, options = {}) { + const benchmarkCase = caseById.get(caseId); + return runCase({ + benchmarkCase, + model: "fixture:test", + createProvider: fixtureProvider(benchmarkCase, options.profile), + repetitions: options.repetitions ?? 1, + }); +} + +async function runAll(options = {}) { + return runBenchmark({ + benchmark, + caseIds: options.caseIds ?? benchmark.cases.map((entry) => entry.case_id), + mode: "fixtures", + provider: "fake-eval", + model: "fixture:per-case", + repetitions: options.repetitions ?? 1, + createProvider: ({ benchmarkCase, scenarioIndex }) => + createEvalFakeProvider({ benchmarkCase, scenarioIndex, profile: options.profile }), + }); +} + +describe("fixture mode makes no network request", () => { + const originals = {}; + + afterEach(() => { + for (const [key, value] of Object.entries(originals)) globalThis[key] = value; + }); + + it("completes the whole benchmark with every network primitive disabled", async () => { + for (const key of ["fetch", "XMLHttpRequest"]) { + originals[key] = globalThis[key]; + globalThis[key] = () => { + throw new Error(`Network access attempted via ${key} during a fixture run.`); + }; + } + const run = await runAll(); + expect(run.summary.execution_count).toBeGreaterThan(0); + expect(run.passed).toBe(true); + }); + + it("does not read an API key", async () => { + const previous = process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + try { + const result = await runOne("case-007"); + expect(result.passed).toBe(true); + } finally { + if (previous !== undefined) process.env.OPENAI_API_KEY = previous; + } + }); +}); + +describe("fixture baseline", () => { + it("passes every committed case", async () => { + const run = await runAll(); + expect(run.summary.required_failures).toBe(0); + expect(run.summary.failed_cases).toBe(0); + expect(run.passed).toBe(true); + }); + + it("produces one execution per scenario per repetition", async () => { + const run = await runAll({ caseIds: ["case-004"], repetitions: 3 }); + expect(run.caseResults[0].executions).toHaveLength(2 * 3); + }); + + it("does not construct a provider after a per-execution budget stop", async () => { + const benchmarkCase = caseById.get("case-004"); + let providerCount = 0; + const result = await runCase({ + benchmarkCase, + model: "fixture:test", + beforeExecution: ({ scenarioIndex }) => scenarioIndex === 0, + createProvider: (context) => { + providerCount += 1; + return createEvalFakeProvider({ + benchmarkCase, + scenarioIndex: context.scenarioIndex, + }); + }, + }); + + expect(providerCount).toBe(1); + expect(result.executions.map((execution) => execution.status)).toEqual([ + "completed", + "skipped", + ]); + expect(result.executions[1].skip_reason).toContain("budget guard"); + expect(result.passed).toBe(false); + }); + + it("distinguishes a clean pass from a pass that observes known defects", async () => { + const clean = await runAll({ caseIds: ["case-007"] }); + const withKnownDefect = await runAll({ caseIds: ["case-001"] }); + expect(clean.summary.run_state).toBe("clean_pass"); + expect(withKnownDefect.summary.run_state).toBe("pass_with_known_defects"); + }); + + it("is deterministic in its decision content across repeated runs", async () => { + const first = await runAll({ caseIds: ["case-004", "case-015"] }); + const second = await runAll({ caseIds: ["case-004", "case-015"] }); + const outcomes = (run) => + run.caseResults.flatMap((caseResult) => + caseResult.executions.map((execution) => execution.outcome), + ); + // duration_ms is wall-clock and deliberately excluded. + const strip = (list) => + list.map((outcome) => + Object.fromEntries(Object.entries(outcome).filter(([key]) => key !== "duration_ms")), + ); + expect(strip(outcomes(second))).toEqual(strip(outcomes(first))); + }); + + it("produces scenario-sensitive winners for a multi-scenario case", async () => { + const run = await runAll({ caseIds: ["case-004"] }); + const winners = run.caseResults[0].executions.map((execution) => execution.outcome.winner_id); + expect(new Set(winners).size).toBeGreaterThan(1); + }); + + it("selects the expected best pair even when it is not the two strongest individuals", async () => { + const run = await runAll({ caseIds: ["case-016"] }); + const execution = run.caseResults[0].executions[0]; + expect(execution.outcome.best_pair_key).toBe("finnegan-adler::hollis-nakamura"); + const ranking = execution.outcome.ranking; + expect([ranking[0], ranking[1]].sort().join("::")).not.toBe(execution.outcome.best_pair_key); + }); +}); + +describe("fake provider profiles", () => { + it("recovers from a single malformed batch without adding a logical stage", async () => { + const clean = await runOne("case-007"); + const retried = await runOne("case-007", { profile: "malformed-once-then-success" }); + + const metadata = (result) => result.executions[0].response.run_metadata; + expect(retried.passed).toBe(true); + expect(metadata(retried).logicalProviderStageCount).toBe( + metadata(clean).logicalProviderStageCount, + ); + expect(metadata(retried).providerAttemptCount).toBeGreaterThan( + metadata(clean).providerAttemptCount, + ); + expect(metadata(retried).attempts.scoring).toBe(2); + }); + + it("reports pairing honestly unavailable when a pair is always missing", async () => { + const result = await runOne("case-015", { profile: "missing-pair" }); + expect(result.passed).toBe(false); + expect(result.executions[0].response.pairing_result.status).toBe("unavailable"); + expect(result.executions[0].response.pairing_result.best_pair).toBeNull(); + }); + + it("fails the scoring stage when an unknown candidate is returned", async () => { + const result = await runOne("case-007", { profile: "unknown-candidate" }); + expect(result.passed).toBe(false); + expect(result.executions[0].status).toBe("failed"); + expect(result.executions[0].failure_reason).toContain("candidate"); + }); + + it("is caught by the graders when the narrative contradicts the ranking", async () => { + const result = await runOne("case-007", { profile: "contradictory-explanation" }); + expect(result.passed).toBe(false); + const failing = result.executions[0].grader_results.filter((entry) => entry.status === "fail"); + expect(failing.map((entry) => entry.grader_id)).toContain("unsupported-claims"); + }); + + it("rejects an unknown profile name", () => { + expect(() => + createEvalFakeProvider({ + benchmarkCase: caseById.get("case-007"), + scenarioIndex: 0, + profile: "does-not-exist", + }), + ).toThrow(/Unknown fake provider profile/); + }); + + it("documents every profile it offers", () => { + for (const [id, meta] of Object.entries(FAKE_PROVIDER_PROFILES)) { + expect(meta.description.length, id).toBeGreaterThan(20); + expect(typeof meta.valid, id).toBe("boolean"); + } + }); + + it("scores by candidate ID, not by submission order", async () => { + const original = await runOne("case-001"); + const permuted = await runOne("case-011"); + const scores = (result) => + Object.fromEntries( + result.executions[0].response.candidate_evaluations.map((candidate) => [ + candidate.candidate_id, + candidate.weighted_fit_score, + ]), + ); + expect(scores(permuted)).toEqual(scores(original)); + }); +}); + +describe("request construction", () => { + it("builds one production request per scenario", () => { + const benchmarkCase = caseById.get("case-004"); + const request = buildEvaluationRequest(benchmarkCase, 1); + expect(request.scenario).toBe(benchmarkCase.input.scenarios[1]); + expect(request.candidates).toHaveLength(benchmarkCase.input.candidates.length); + }); + + it("raises the candidate cap to what a case needs, within the shared ceiling", () => { + expect(resolveMaxCandidatesForCase(caseById.get("case-015"))).toBeGreaterThanOrEqual(4); + expect(resolveMaxCandidatesForCase(caseById.get("case-015"))).toBeLessThanOrEqual(10); + }); +}); + +describe("stability accounting", () => { + it("refuses to assess stability from a single repetition", () => { + const stability = computeStability([], 1); + expect(stability.assessed).toBe(false); + expect(stability.winner_agreement).toBeNull(); + expect(stability.reason).toContain("not enough"); + }); + + it("reports agreement when repetitions agree", async () => { + const run = await runAll({ caseIds: ["case-007"], repetitions: 2 }); + expect(run.summary.stability.assessed).toBe(true); + expect(run.summary.stability.winner_agreement).toBe(1); + expect(run.summary.stability.ranking_agreement).toBe(1); + }); + + it("reports disagreement when repetitions disagree", () => { + const caseResults = [ + { + executions: [ + { status: "completed", case_id: "case-001", scenario_index: 0, outcome: { winner_id: "a", ranking: ["a", "b"] } }, + { status: "completed", case_id: "case-001", scenario_index: 0, outcome: { winner_id: "b", ranking: ["b", "a"] } }, + ], + }, + ]; + const stability = computeStability(caseResults, 2); + expect(stability.assessed).toBe(true); + expect(stability.winner_agreement).toBe(0); + }); +}); + +describe("permutation analysis", () => { + it("finds no change across every committed variant under the fixture provider", async () => { + const run = await runAll(); + expect(run.permutations).toHaveLength(4); + for (const finding of run.permutations) { + expect(finding.compared, finding.case_id).toBe(true); + expect(finding.winner_changed, finding.case_id).toBe(false); + expect(finding.ranking_changed, finding.case_id).toBe(false); + expect(finding.structured_evidence_changed, finding.case_id).toBe(false); + } + }); + + it("reports 'not compared' when the original is outside the case selection", async () => { + const run = await runAll({ caseIds: ["case-011"] }); + expect(run.permutations[0].compared).toBe(false); + expect(run.permutations[0].reason).toContain("not part of this run"); + }); + + it("detects a winner change between a variant and its original", () => { + const findings = analysePermutations([ + { + case_id: "case-001", + variant_of: null, + variant_kind: null, + executions: [ + { status: "completed", scenario: "S", scenario_index: 0, outcome: { winner_id: "a", ranking: ["a"], best_pair_key: null }, response: { candidate_evaluations: [], decision_result: {}, executive_summary: {}, trade_offs: [] } }, + ], + }, + { + case_id: "case-011", + variant_of: "case-001", + variant_kind: "candidate-order", + executions: [ + { status: "completed", scenario: "S", scenario_index: 0, outcome: { winner_id: "b", ranking: ["b"], best_pair_key: null }, response: { candidate_evaluations: [], decision_result: {}, executive_summary: {}, trade_offs: [] } }, + ], + }, + ]); + const variant = findings.find((entry) => entry.case_id === "case-011"); + expect(variant.winner_changed).toBe(true); + expect(variant.ranking_changed).toBe(true); + }); +}); + +describe("run artifacts", () => { + it("writes every promised file, with no secrets or absolute paths", async () => { + const run = await runAll({ caseIds: ["case-007", "case-015"] }); + const root = await mkdtemp(path.join(tmpdir(), "eval-artifacts-")); + const { runDir, files } = await writeRunArtifacts({ + run, + markdown: renderRunMarkdown(run), + humanReviewTemplate: buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: run.manifest, + caseResults: run.caseResults, + casesById: caseById, + }), + rootDir: root, + }); + + expect(files).toEqual( + expect.arrayContaining([ + "run-manifest.json", + "case-results.jsonl", + "summary.json", + "summary.md", + "human-review-template.json", + ]), + ); + const written = await readdir(runDir); + expect(written.sort()).toEqual([...files].sort()); + + for (const name of written) { + const content = await readFile(path.join(runDir, name), "utf8"); + expect(content, name).not.toMatch(/\bsk-[A-Za-z0-9_-]{16,}/); + expect(content, name).not.toMatch(/\/(?:Users|home|root)\//); + expect(content, name).not.toMatch(/[A-Za-z]:\\\\/); + } + }); + + it("records benchmark version, commit, grader versions, and the artifact policy", async () => { + const run = await runAll({ caseIds: ["case-007"] }); + expect(run.manifest.benchmark_version).toBe(benchmark.manifest.benchmark_version); + expect(run.manifest.grader_versions["contract-validity"]).toBeTruthy(); + expect(run.manifest.artifact_policy).toEqual({ + synthetic_data_only: true, + secrets_recorded: false, + absolute_paths_recorded: false, + }); + const git = readGitContext(); + expect(run.manifest.git_commit).toBe(git.commit); + }); + + it("carries the scope disclaimer into the summary and markdown", async () => { + const run = await runAll({ caseIds: ["case-007"] }); + expect(run.summary.disclaimer).toContain("not scientifically validated"); + expect(renderRunMarkdown(run)).toContain("not scientifically validated"); + }); + + it("emits no ANSI escape codes", async () => { + const run = await runAll({ caseIds: ["case-007"] }); + // eslint-disable-next-line no-control-regex + const ansi = /\u001b\[/; + expect(ansi.test(renderRunMarkdown(run))).toBe(false); + expect(ansi.test(renderConsoleSummary(run))).toBe(false); + }); + + it("can record a response that violates the public contract", async () => { + // The artifact schema must never enforce the contract the grader exists to + // check, or the harness would crash while recording its own finding. + const run = await runAll({ caseIds: ["case-001"] }); + const root = await mkdtemp(path.join(tmpdir(), "eval-defect-")); + await expect( + writeRunArtifacts({ + run, + markdown: renderRunMarkdown(run), + humanReviewTemplate: buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: run.manifest, + caseResults: run.caseResults, + casesById: caseById, + }), + rootDir: root, + }), + ).resolves.toBeTruthy(); + }); + + it("round-trips a written run through the reader", async () => { + const run = await runAll({ caseIds: ["case-007"] }); + const root = await mkdtemp(path.join(tmpdir(), "eval-roundtrip-")); + const { runDir } = await writeRunArtifacts({ + run, + markdown: renderRunMarkdown(run), + humanReviewTemplate: buildHumanReviewTemplate({ + rubric: benchmark.rubric, + manifest: run.manifest, + caseResults: run.caseResults, + casesById: caseById, + }), + rootDir: root, + }); + const read = await readRunArtifacts(runDir); + expect(read.manifest.run_id).toBe(run.manifest.run_id); + expect(read.caseResults).toHaveLength(run.caseResults.length); + expect(read.humanReview).toBeNull(); + }); +}); + +describe("known defects in the committed baseline", () => { + it("keeps the baseline green while reporting the defect prominently", async () => { + const run = await runAll(); + expect(run.passed).toBe(true); + expect(run.summary.expected_failures).toBeGreaterThan(0); + expect(renderRunMarkdown(run)).toContain("Known defects reproduced"); + }); + + it("attributes every expected failure to a documented defect", async () => { + const run = await runAll(); + const expected = run.caseResults.flatMap((caseResult) => + caseResult.executions + .flatMap((execution) => execution.grader_results) + .filter((result) => result.status === "expected_failure"), + ); + expect(expected).toHaveLength(8); + expect(new Set(expected.map((result) => result.known_defect_id))).toEqual(new Set(["SR-P3A-001"])); + expect(run.summary.affected_execution_ids).toHaveLength(4); + for (const result of expected) { + expect(result.summary).toMatch(/^Known defect SR-/); + expect(result.details.join(" ")).toContain("see docs/"); + } + }); + + it("renders all top-level run states without a bare pass label", async () => { + const run = await runAll({ caseIds: ["case-007"] }); + const states = { + clean_pass: "CLEAN PASS", + pass_with_known_defects: "PASS WITH KNOWN DEFECTS", + unexpected_failure: "UNEXPECTED FAILURE", + baseline_change_required: "BASELINE CHANGE REQUIRED", + }; + for (const [runState, label] of Object.entries(states)) { + const rendered = renderConsoleSummary({ + ...run, + summary: { ...run.summary, run_state: runState }, + }); + expect(rendered).toContain(`production baseline: ${label}`); + expect(rendered).not.toContain("result: PASSED"); + } + }); +}); + +describe("unknown case selection", () => { + it("refuses a case that is not part of the benchmark", async () => { + await expect(runAll({ caseIds: ["case-999"] })).rejects.toThrow(/is not part of benchmark/); + }); +}); diff --git a/evals/schemas/benchmarkCase.js b/evals/schemas/benchmarkCase.js new file mode 100644 index 0000000..5dd1d1d --- /dev/null +++ b/evals/schemas/benchmarkCase.js @@ -0,0 +1,547 @@ +/** + * @file Benchmark case schema (Phase 3A evaluation harness). + * + * A benchmark case is a *fully synthetic* decision problem plus the + * expectations the harness is allowed to check automatically. It deliberately + * separates two very different things (docs/evaluation/BENCHMARK_V1.md): + * + * - `deterministic_expectations` — invariants that are objectively true or + * false about a pipeline response (candidate coverage, pair coverage, + * stage accounting, whether the reported winner matches the deterministic + * ranking). These are machine-checkable and a failure is a real defect. + * - `rubric_dimensions` — qualitative judgments about explanation quality. + * These are NOT objective labels. Phase 3A scores them only through the + * human-review format (evals/graders/humanReview.js); no LLM-as-judge + * grading exists in this phase. + * + * A case never hardcodes one "perfect" natural-language answer. Where more + * than one winner is legitimately defensible, `allowed_winner_ids` lists every + * acceptable outcome rather than pretending a single one is correct. + * + * This module wraps — and never duplicates — the production public contracts + * in shared/contracts/. The decision input a case describes is validated with + * the real `evaluationRequestSchema` before it is ever executed. + */ +import { z } from "zod"; +import { + roleTitleSchema, + roleDescriptionSchema, + scenarioInputSchema, + candidateNameSchema, + candidateDescriptionSchema, +} from "../../shared/contracts/decisionApi.js"; +import { DECISION_INPUT_LIMITS } from "../../shared/contracts/decisionInputLimits.js"; +import { CRITERIA_KEYS } from "../../server/ai/schemas/criteriaKeys.js"; + +/** Bumped only when the *shape* of a case file changes incompatibly. */ +export const BENCHMARK_CASE_SCHEMA_VERSION = "1.0.0"; + +/** + * The closed tag vocabulary. An unknown tag is rejected rather than silently + * accepted, so "cases tagged X" can never quietly mean "cases someone spelled + * X-ish". + */ +export const BENCHMARK_TAGS = Object.freeze([ + "basic-ranking", + "multi-scenario", + "close-call", + "missing-evidence", + "conflicting-evidence", + "permutation", + "duplicate-name", + "pairing", + "uncertainty", +]); + +/** How a variant case was derived from its original. */ +export const VARIANT_KINDS = Object.freeze([ + "candidate-order", + "scenario-order", + "equivalent-wording", + "irrelevant-text", +]); + +/** + * Evidence-quality profiles the offline fake provider uses to shape the + * evidence strings it returns. These drive the *deterministic* confidence and + * evidence review in server/pipeline/runPipeline.js — they are a controlled + * stand-in for evidence quality, not a claim that a real model would react the + * same way. + */ +export const EVIDENCE_QUALITY_PROFILES = Object.freeze([ + "specific", + "vague", + "conflicting", + "missing", +]); + +export const benchmarkCaseIdSchema = z + .string() + .regex(/^case-\d{3}$/, "Case IDs must look like case-001."); + +const candidateIdSchema = z + .string() + .regex( + /^[a-z0-9][a-z0-9-]{1,62}$/, + "Candidate IDs must be lowercase alphanumeric with hyphens (2-63 chars).", + ); + +const criteriaOverrideSchema = z + .object( + Object.fromEntries( + CRITERIA_KEYS.map((key) => [key, z.number().min(1).max(10).optional()]), + ), + ) + .strict(); + +/** + * Per-candidate instructions for the offline fake provider. Scores are keyed + * by candidate ID — never by array position — so a candidate-order permutation + * of a case produces byte-identical scoring input, which is exactly what makes + * the permutation check meaningful. + */ +const fakeCandidateScorePlanSchema = z + .object({ + default: z.number().min(1).max(10), + criteria: criteriaOverrideSchema.optional(), + confidence: z.number().min(0).max(1).optional(), + evidence_quality: z.enum(EVIDENCE_QUALITY_PROFILES).optional(), + }) + .strict(); + +const criteriaDeltaSchema = z + .object( + Object.fromEntries( + CRITERIA_KEYS.map((key) => [key, z.number().min(-20).max(20).optional()]), + ), + ) + .strict(); + +const fakePairMetricsSchema = z + .object({ + scenario_coverage: z.number().min(0).max(1), + complementarity: z.number().min(0).max(1), + overlap_risk: z.number().min(0).max(1), + conflict_risk: z.number().min(0).max(1), + execution_cohesion: z.number().min(0).max(1), + pair_adaptability: z.number().min(0).max(1), + }) + .strict(); + +export const fakeProviderPlanSchema = z + .object({ + profile: z.string().min(1), + candidate_scores: z.record(candidateIdSchema, fakeCandidateScorePlanSchema), + /** Keyed by zero-based scenario index, as a string (JSON object keys). */ + scenario_overrides: z + .record( + z.string().regex(/^\d+$/, "Scenario override keys are scenario indexes."), + z.record(candidateIdSchema, fakeCandidateScorePlanSchema.partial()), + ) + .optional(), + /** + * Per-scenario criterion weight deltas, keyed by zero-based scenario + * index. This is how a multi-scenario case makes one scenario genuinely + * favour different skills from another: the same candidate scores are + * re-weighted, exactly as the production scenario-analysis stage does via + * `applyDeltas` (server/domain/scoring.js). + */ + scenario_weight_deltas: z + .record( + z.string().regex(/^\d+$/, "Scenario weight-delta keys are scenario indexes."), + criteriaDeltaSchema, + ) + .optional(), + /** Keyed by canonical pair key: the two candidate IDs sorted, joined by "::". */ + pair_overrides: z.record(z.string().min(3), fakePairMetricsSchema).optional(), + }) + .strict(); + +const deterministicExpectationsSchema = z + .object({ + /** Every candidate ID the response must account for, exactly once. */ + expected_candidate_ids: z.array(candidateIdSchema).min(2), + pairing_enabled: z.boolean(), + /** + * Unordered pairs the pairing stage must evaluate. `null` when pairing is + * disabled. For N top-ranked candidates (N capped at 4) this is N*(N-1)/2. + */ + expected_pair_count: z.number().int().min(0).nullable(), + /** Best pair the deterministic pair score must select, as sorted IDs. */ + expected_best_pair_ids: z.tuple([candidateIdSchema, candidateIdSchema]).nullable(), + /** Logical model-backed stage count: 3 without pairing, 4 with it. */ + required_stage_count: z.number().int().min(1).max(4), + /** Every scenario string that must appear in this case's executions. */ + required_scenario_coverage: z.array(scenarioInputSchema).min(1), + /** + * Winners that are legitimately defensible for this case. `null` means the + * case makes no winner claim at all (used where the point is coverage or + * structure, not who wins). A single-element array is a strong claim and + * should only be used where one candidate really does dominate. + */ + allowed_winner_ids: z.array(candidateIdSchema).min(1).nullable(), + forbidden_winner_ids: z.array(candidateIdSchema), + /** Response paths that must honestly report "not_measured". */ + required_not_measured_fields: z.array(z.string().min(1)), + /** Ceiling on real provider attempts for one execution of this case. */ + maximum_provider_attempts: z.number().int().min(1).max(24), + /** + * Candidates whose weak/missing evidence must produce a human-review + * recommendation. Empty when the case makes no uncertainty claim. + */ + expect_human_review_for_candidate_ids: z.array(candidateIdSchema), + }) + .strict(); + +const caseInputSchema = z + .object({ + role: z + .object({ title: roleTitleSchema, description: roleDescriptionSchema }) + .strict(), + /** + * One or more scenarios. Each scenario is executed as its own pipeline + * run against the unchanged production contract (which takes exactly one + * scenario per request) — the harness never invents a multi-scenario + * request shape the server does not support. + */ + scenarios: z + .array(scenarioInputSchema) + .min(DECISION_INPUT_LIMITS.scenarios.min) + .max(DECISION_INPUT_LIMITS.scenarios.max), + decision_mode: z.enum(["best_fit", "lowest_risk", "best_outcome"]), + candidates: z + .array( + z + .object({ + id: candidateIdSchema, + /** + * Display names are deliberately allowed to collide. Duplicate + * display names with distinct IDs are a real, tested case + * (case-014) — the pipeline must stay unambiguous through IDs. + */ + name: candidateNameSchema, + description: candidateDescriptionSchema, + }) + .strict(), + ) + .min(DECISION_INPUT_LIMITS.candidates.min) + .max(DECISION_INPUT_LIMITS.candidates.max), + options: z.object({ enable_pair_simulation: z.boolean() }).strict(), + }) + .strict(); + +const defectObservationSchema = z.discriminatedUnion("kind", [ + z + .object({ + kind: z.literal("schema_issue"), + path_pattern: z.literal("candidate_evaluations.*.risk_adjusted_score"), + code: z.literal("too_small"), + minimum: z.literal(0), + subject_candidate_id: candidateIdSchema, + }) + .strict(), + z + .object({ + kind: z.literal("score_bound_violation"), + metric: z.literal("risk_adjusted_score"), + operator: z.literal("lt"), + bound: z.literal(0), + subject_candidate_id: candidateIdSchema, + }) + .strict(), +]); + +const knownDefectSchema = z + .object({ + defect_id: z.string().regex(/^SR-[A-Z0-9-]+$/, "Defect IDs look like SR-P3A-001."), + title: z.string().min(20).max(240), + /** Repeated deliberately so a record remains meaningful when extracted. */ + case_id: benchmarkCaseIdSchema, + execution_scope: z + .object({ + execution_id: z.string().min(1), + scenario_id: z.string().regex(/^scenario-[1-9]\d*$/), + scenario_index: z.number().int().min(0), + variant_id: z.enum(VARIANT_KINDS).nullable(), + repetition: z.number().int().min(1), + }) + .strict(), + expected_observations: z + .array( + z + .object({ + grader_id: z.string().min(1), + signature: defectObservationSchema, + }) + .strict(), + ) + .min(1), + summary: z.string().min(20).max(500), + reference: z.string().min(1), + }) + .strict(); + +export const benchmarkCaseSchema = z + .object({ + case_id: benchmarkCaseIdSchema, + schema_version: z.string().min(1), + title: z.string().min(1).max(160), + description: z.string().min(1).max(1000), + tags: z.array(z.enum(BENCHMARK_TAGS)).min(1), + /** + * Synthetic-data policy metadata. Both fields are literals, not booleans a + * future case could quietly flip: every committed case is invented, and no + * real person, applicant, employee, company, or record appears anywhere in + * this benchmark (docs/evaluation/BENCHMARK_V1.md). + */ + synthetic: z.literal(true), + data_policy: z.literal("synthetic-only"), + input: caseInputSchema, + deterministic_expectations: deterministicExpectationsSchema, + /** Rubric dimension IDs a human reviewer should score for this case. */ + rubric_dimensions: z.array(z.string().min(1)).min(1), + /** Set on a permutation/wording variant; null on an original case. */ + variant_of: benchmarkCaseIdSchema.nullable(), + variant_kind: z.enum(VARIANT_KINDS).nullable(), + fake_provider_plan: fakeProviderPlanSchema, + /** + * Graders this case is currently *expected* to fail because of a + * documented, pre-existing defect in the product — not because the case is + * wrong. + * + * This exists so a real finding can stay visible without leaving the + * baseline permanently red, which would train everyone to ignore it. Two + * rules keep it honest: + * - a known defect must name a documented reference, so it cannot be + * used as a quiet suppression; + * - when a listed grader starts *passing*, the harness reports a + * required failure. A fix can never land unnoticed, and the record can + * never silently rot. + */ + known_defects: z.array(knownDefectSchema).default([]), + notes: z.string().max(2000).optional(), + }) + .strict() + .superRefine((benchmarkCase, context) => { + const declaredIds = benchmarkCase.input.candidates.map((c) => c.id); + const uniqueIds = new Set(declaredIds); + if (uniqueIds.size !== declaredIds.length) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["input", "candidates"], + message: "Candidate IDs must be unique within a case.", + }); + } + + const expectations = benchmarkCase.deterministic_expectations; + const knownDefectIds = new Set(); + const knownObservationKeys = new Set(); + benchmarkCase.known_defects.forEach((defect, defectIndex) => { + if (defect.case_id !== benchmarkCase.case_id) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "case_id"], message: "Known-defect case_id must equal its enclosing case." }); + } + if (defect.execution_scope.scenario_index >= benchmarkCase.input.scenarios.length) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "execution_scope", "scenario_index"], message: "Known-defect scenario index is outside this case's scenario list." }); + } + const expectedScenarioId = `scenario-${defect.execution_scope.scenario_index + 1}`; + const expectedExecutionId = `${benchmarkCase.case_id}#s${defect.execution_scope.scenario_index}#r${defect.execution_scope.repetition}`; + if (defect.execution_scope.scenario_id !== expectedScenarioId || defect.execution_scope.execution_id !== expectedExecutionId) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "execution_scope"], message: `Known-defect execution scope must identify ${expectedExecutionId} / ${expectedScenarioId}.` }); + } + if (defect.execution_scope.variant_id !== benchmarkCase.variant_kind) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "execution_scope", "variant_id"], message: "Known-defect variant_id must match this case's variant kind (or null)." }); + } + if (knownDefectIds.has(defect.defect_id)) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "defect_id"], message: "Duplicate known-defect ID in one case." }); + } + knownDefectIds.add(defect.defect_id); + defect.expected_observations.forEach((observation, observationIndex) => { + if (!declaredIds.includes(observation.signature.subject_candidate_id)) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "expected_observations", observationIndex, "signature", "subject_candidate_id"], message: "Known-defect observation references a candidate that is not in this case." }); + } + const key = `${defect.execution_scope.execution_id}\u0000${observation.grader_id}\u0000${JSON.stringify(observation.signature)}`; + if (knownObservationKeys.has(key)) context.addIssue({ code: z.ZodIssueCode.custom, path: ["known_defects", defectIndex, "expected_observations", observationIndex], message: "Duplicate or ambiguous known-defect observation signature." }); + knownObservationKeys.add(key); + }); + }); + const expectedIds = [...expectations.expected_candidate_ids].sort(); + if (JSON.stringify(expectedIds) !== JSON.stringify([...declaredIds].sort())) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "expected_candidate_ids"], + message: "expected_candidate_ids must match the case's candidate IDs exactly.", + }); + } + + if (expectations.pairing_enabled !== benchmarkCase.input.options.enable_pair_simulation) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "pairing_enabled"], + message: "pairing_enabled must match input.options.enable_pair_simulation.", + }); + } + + // The pipeline pairs the top four ranked candidates, so the expected pair + // count is fully determined by the candidate count — it is never a free + // parameter a case can get wrong silently. + const pairedCount = Math.min(4, declaredIds.length); + const expectedPairs = expectations.pairing_enabled + ? (pairedCount * (pairedCount - 1)) / 2 + : null; + if (expectations.expected_pair_count !== expectedPairs) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "expected_pair_count"], + message: `expected_pair_count must be ${expectedPairs} for this case.`, + }); + } + + const expectedStages = expectations.pairing_enabled ? 4 : 3; + if (expectations.required_stage_count !== expectedStages) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "required_stage_count"], + message: `required_stage_count must be ${expectedStages} for this case.`, + }); + } + + if ( + JSON.stringify(expectations.required_scenario_coverage) !== + JSON.stringify(benchmarkCase.input.scenarios) + ) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "required_scenario_coverage"], + message: "required_scenario_coverage must list this case's scenarios in order.", + }); + } + + if (!expectations.pairing_enabled && expectations.expected_best_pair_ids !== null) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "expected_best_pair_ids"], + message: "expected_best_pair_ids requires pairing to be enabled.", + }); + } + + for (const id of expectations.allowed_winner_ids ?? []) { + if (!uniqueIds.has(id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "allowed_winner_ids"], + message: `allowed_winner_ids references unknown candidate "${id}".`, + }); + } + } + for (const id of expectations.forbidden_winner_ids) { + if (!uniqueIds.has(id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "forbidden_winner_ids"], + message: `forbidden_winner_ids references unknown candidate "${id}".`, + }); + } + if (expectations.allowed_winner_ids?.includes(id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "forbidden_winner_ids"], + message: `"${id}" cannot be both allowed and forbidden.`, + }); + } + } + + for (const id of declaredIds) { + if (!(id in benchmarkCase.fake_provider_plan.candidate_scores)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["fake_provider_plan", "candidate_scores"], + message: `fake_provider_plan is missing a score plan for "${id}".`, + }); + } + } + for (const id of Object.keys(benchmarkCase.fake_provider_plan.candidate_scores)) { + if (!uniqueIds.has(id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["fake_provider_plan", "candidate_scores"], + message: `fake_provider_plan scores unknown candidate "${id}".`, + }); + } + } + + for (const [field, overrides] of [ + ["scenario_overrides", benchmarkCase.fake_provider_plan.scenario_overrides], + ["scenario_weight_deltas", benchmarkCase.fake_provider_plan.scenario_weight_deltas], + ]) { + for (const index of Object.keys(overrides ?? {})) { + if (Number(index) >= benchmarkCase.input.scenarios.length) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["fake_provider_plan", field], + message: `Scenario index ${index} is out of range.`, + }); + } + } + } + + const hasVariantOf = benchmarkCase.variant_of !== null; + const hasVariantKind = benchmarkCase.variant_kind !== null; + if (hasVariantOf !== hasVariantKind) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["variant_kind"], + message: "variant_of and variant_kind must both be set, or both be null.", + }); + } + if (hasVariantOf && !benchmarkCase.tags.includes("permutation")) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["tags"], + message: "A variant case must carry the \"permutation\" tag.", + }); + } + if (hasVariantOf && benchmarkCase.variant_of === benchmarkCase.case_id) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["variant_of"], + message: "A case cannot be a variant of itself.", + }); + } + + for (const id of expectations.expect_human_review_for_candidate_ids) { + if (!uniqueIds.has(id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_expectations", "expect_human_review_for_candidate_ids"], + message: `expect_human_review_for_candidate_ids references unknown candidate "${id}".`, + }); + } + } + }); + +/** + * Canonical unordered pair key. Pair identity is always ID-based and + * order-independent, mirroring `mapPairResultsByIdentity` in + * server/pipeline/runPipeline.js so the harness cannot disagree with + * production about what "the same pair" means. + * @param {string} a + * @param {string} b + */ +export function canonicalPairKey(a, b) { + return [a, b].sort().join("::"); +} + +/** + * @param {unknown} value + * @returns {{ ok: true, data: object } | { ok: false, issues: string[] }} + */ +export function parseBenchmarkCase(value) { + const result = benchmarkCaseSchema.safeParse(value); + if (result.success) return { ok: true, data: result.data }; + return { + ok: false, + issues: result.error.issues.map( + (issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`, + ), + }; +} diff --git a/evals/schemas/benchmarkManifest.js b/evals/schemas/benchmarkManifest.js new file mode 100644 index 0000000..7dafe84 --- /dev/null +++ b/evals/schemas/benchmarkManifest.js @@ -0,0 +1,226 @@ +/** + * @file Benchmark manifest and rubric schemas (Phase 3A evaluation harness). + * + * The manifest is the immutable identity of a benchmark. Everything a report + * needs in order to be interpreted later — which benchmark, which version, + * which rubric, which case IDs — is recorded here and copied into every run + * artifact. + * + * Versioning policy (enforced by the schema below and by + * evals/schemas/benchmarkManifest.test.js, documented in + * docs/evaluation/BENCHMARK_V1.md): + * + * 1. `benchmark_id` never changes once published. + * 2. A case ID never changes or is reused once published. + * 3. Changing what an existing case *means* — its inputs, its expectations, + * what a passing result implies — requires a NEW `benchmark_version`, and + * by convention a new `benchmark_id` suffix (decision-benchmark-v2). + * 4. A change that cannot alter any result — a typo in prose, a clearer + * description — increments `metadata_revision` only. + * 5. `schema_version` describes the case/manifest file *shape*. A runner + * refuses a manifest whose `schema_version` it does not support rather + * than guessing. + */ +import { z } from "zod"; +import { benchmarkCaseIdSchema, BENCHMARK_TAGS } from "./benchmarkCase.js"; + +/** Manifest file shape version. Runners refuse anything they do not support. */ +export const BENCHMARK_MANIFEST_SCHEMA_VERSION = "1.0.0"; + +/** Every manifest schema version this harness build can execute. */ +export const SUPPORTED_BENCHMARK_SCHEMA_VERSIONS = Object.freeze(["1.0.0"]); + +/** + * A harness-side declaration of which ScenarioRank pipeline generation this + * benchmark's expectations were written against. + * + * Honest scope note: production does not emit a pipeline version string, and + * Phase 3A deliberately does not add one (no production behavior changes in + * this phase). This constant is therefore a marker maintained *by the + * harness*, not a value read from the running pipeline. It is backed by a real + * structural probe (`assertPipelineCompatibility` below) so a mismatch between + * the declared marker and the actual pipeline shape cannot go unnoticed. + */ +export const EVAL_PIPELINE_VERSION = "v2-phase-2d"; + +export const rubricDimensionSchema = z + .object({ + id: z.string().regex(/^[a-z][a-z0-9_]*$/, "Rubric dimension IDs are snake_case."), + label: z.string().min(1).max(120), + /** Exactly what a reviewer is judging — never a vague quality gesture. */ + what_is_judged: z.string().min(20).max(600), + scale: z + .object({ min: z.literal(0), max: z.literal(4) }) + .strict(), + /** One anchor per point on the scale, so scores mean the same thing twice. */ + anchors: z + .object({ + 0: z.string().min(5), + 1: z.string().min(5), + 2: z.string().min(5), + 3: z.string().min(5), + 4: z.string().min(5), + }) + .strict(), + failure_examples: z.array(z.string().min(5)).min(1), + human_review_required: z.boolean(), + /** + * Whether *any* part of this dimension can be checked deterministically. + * Where true, the named deterministic grader covers a conservative subset + * only — it never replaces the human judgment (docs/evaluation/HUMAN_REVIEW_GUIDE.md). + */ + deterministic_automation_possible: z.boolean(), + deterministic_grader_id: z.string().min(1).nullable(), + }) + .strict() + .superRefine((dimension, context) => { + if (dimension.deterministic_automation_possible && !dimension.deterministic_grader_id) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_grader_id"], + message: "A dimension claiming automation must name the grader that provides it.", + }); + } + if (!dimension.deterministic_automation_possible && dimension.deterministic_grader_id) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["deterministic_grader_id"], + message: "A dimension without automation must not name a grader.", + }); + } + }); + +export const rubricSchema = z + .object({ + rubric_version: z.string().min(1), + schema_version: z.string().min(1), + description: z.string().min(1), + /** Guardrail prose that must travel with the rubric, not just the docs. */ + interpretation_warning: z.string().min(20), + allowed_non_scores: z.array(z.enum(["not_applicable", "cannot_determine"])).length(2), + dimensions: z.array(rubricDimensionSchema).min(1), + }) + .strict() + .superRefine((rubric, context) => { + const seen = new Set(); + rubric.dimensions.forEach((dimension, index) => { + if (seen.has(dimension.id)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["dimensions", index, "id"], + message: `Duplicate rubric dimension id "${dimension.id}".`, + }); + } + seen.add(dimension.id); + }); + }); + +export const benchmarkManifestSchema = z + .object({ + benchmark_id: z + .string() + .regex(/^[a-z][a-z0-9-]*-v\d+$/, "Benchmark IDs end in a version suffix, e.g. -v1."), + benchmark_version: z + .string() + .regex(/^\d+\.\d+\.\d+$/, "benchmark_version is semver-shaped."), + schema_version: z.string().min(1), + /** + * Incremented for changes that cannot alter any result (typos, clearer + * prose). A meaning change requires a new benchmark_version instead. + */ + metadata_revision: z.number().int().min(0), + created_at: z.string().datetime(), + description: z.string().min(1), + case_count: z.number().int().min(1), + case_ids: z.array(benchmarkCaseIdSchema).min(1), + rubric_version: z.string().min(1), + supported_modes: z.array(z.enum(["fixtures", "live"])).min(1), + required_pipeline_version: z.string().min(1), + /** Closed tag vocabulary, mirrored from the case schema. */ + tag_catalog: z.array(z.enum(BENCHMARK_TAGS)).min(1), + data_policy: z.literal("synthetic-only"), + /** Prose the harness refuses to let a benchmark drop. */ + scope_disclaimer: z.string().min(40), + versioning_policy: z + .object({ + case_ids_immutable: z.literal(true), + meaning_change_requires_new_version: z.literal(true), + cosmetic_change_increments_metadata_revision: z.literal(true), + reports_record_benchmark_version_and_commit: z.literal(true), + }) + .strict(), + }) + .strict() + .superRefine((manifest, context) => { + if (manifest.case_count !== manifest.case_ids.length) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["case_count"], + message: `case_count (${manifest.case_count}) does not match case_ids length (${manifest.case_ids.length}).`, + }); + } + const seen = new Set(); + manifest.case_ids.forEach((caseId, index) => { + if (seen.has(caseId)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["case_ids", index], + message: `Duplicate case id "${caseId}".`, + }); + } + seen.add(caseId); + }); + }); + +/** + * Refuses a benchmark whose file shape this build does not understand, rather + * than attempting a best-effort read of an unknown format. + * @param {string} schemaVersion + */ +export function assertSupportedSchemaVersion(schemaVersion) { + if (!SUPPORTED_BENCHMARK_SCHEMA_VERSIONS.includes(schemaVersion)) { + throw new Error( + `Unsupported benchmark schema_version "${schemaVersion}". ` + + `This harness supports: ${SUPPORTED_BENCHMARK_SCHEMA_VERSIONS.join(", ")}. ` + + "Refusing to run rather than guess at an unknown case format.", + ); + } +} + +/** + * Structural probe backing `EVAL_PIPELINE_VERSION`. Rather than trusting a + * hand-maintained string alone, this asserts the observable facts the + * benchmark's expectations actually depend on: + * + * - seven scoring criteria (weights, coverage, and score-integrity graders); + * - at most four logical model-backed stages (pipeline-accounting grader). + * + * A future pipeline change that breaks either assumption fails loudly here + * instead of silently invalidating every recorded benchmark result. + * @param {{ criteriaKeys: string[], maxLogicalStages: number, declaredVersion?: string }} probe + */ +export function assertPipelineCompatibility({ + criteriaKeys, + maxLogicalStages, + declaredVersion = EVAL_PIPELINE_VERSION, +}) { + const problems = []; + if (criteriaKeys.length !== 7) { + problems.push(`expected 7 scoring criteria, found ${criteriaKeys.length}`); + } + if (maxLogicalStages !== 4) { + problems.push(`expected 4 maximum logical stages, found ${maxLogicalStages}`); + } + if (declaredVersion !== EVAL_PIPELINE_VERSION) { + problems.push( + `benchmark requires pipeline "${declaredVersion}" but this harness targets "${EVAL_PIPELINE_VERSION}"`, + ); + } + if (problems.length > 0) { + throw new Error( + `Benchmark is incompatible with the current pipeline: ${problems.join("; ")}. ` + + "Re-validate the benchmark's expectations before recording any further results.", + ); + } + return true; +} diff --git a/evals/schemas/benchmarkSchemas.test.js b/evals/schemas/benchmarkSchemas.test.js new file mode 100644 index 0000000..1fcca20 --- /dev/null +++ b/evals/schemas/benchmarkSchemas.test.js @@ -0,0 +1,389 @@ +/** + * Benchmark manifest, rubric, and case *schema* tests. + * + * These prove the schemas reject the specific malformed inputs that would + * otherwise let a quietly broken benchmark produce confident-looking numbers. + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +import { + benchmarkCaseSchema, + parseBenchmarkCase, + canonicalPairKey, + BENCHMARK_TAGS, +} from "./benchmarkCase.js"; +import { + benchmarkManifestSchema, + rubricSchema, + assertSupportedSchemaVersion, + assertPipelineCompatibility, + SUPPORTED_BENCHMARK_SCHEMA_VERSIONS, + EVAL_PIPELINE_VERSION, +} from "./benchmarkManifest.js"; +import { findArtifactPolicyViolations, assertArtifactIsPolicyClean } from "./evaluationRun.js"; +import { numericDelta } from "./evaluationReport.js"; + +const DATASET = path.resolve("evals/datasets/decision-benchmark-v1"); +const readJson = (relative) => JSON.parse(readFileSync(path.join(DATASET, relative), "utf8")); +const manifestFixture = () => readJson("manifest.json"); +const caseFixture = (id = "case-001") => readJson(`cases/${id}.json`); + +describe("benchmark manifest schema", () => { + it("accepts the committed manifest", () => { + expect(benchmarkManifestSchema.safeParse(manifestFixture()).success).toBe(true); + }); + + it("rejects duplicate case IDs", () => { + const manifest = manifestFixture(); + manifest.case_ids = [...manifest.case_ids.slice(0, -1), manifest.case_ids[0]]; + const result = benchmarkManifestSchema.safeParse(manifest); + expect(result.success).toBe(false); + expect(JSON.stringify(result.error.issues)).toContain("Duplicate case id"); + }); + + it("rejects a case_count that disagrees with case_ids", () => { + const manifest = manifestFixture(); + manifest.case_count = manifest.case_ids.length + 1; + const result = benchmarkManifestSchema.safeParse(manifest); + expect(result.success).toBe(false); + expect(JSON.stringify(result.error.issues)).toContain("does not match case_ids length"); + }); + + it("rejects an unsupported schema version rather than guessing", () => { + expect(() => assertSupportedSchemaVersion("9.9.9")).toThrow(/Unsupported benchmark schema_version/); + expect(() => assertSupportedSchemaVersion(SUPPORTED_BENCHMARK_SCHEMA_VERSIONS[0])).not.toThrow(); + }); + + it("requires the synthetic-data policy and a scope disclaimer", () => { + const withoutPolicy = manifestFixture(); + delete withoutPolicy.data_policy; + expect(benchmarkManifestSchema.safeParse(withoutPolicy).success).toBe(false); + + const shortDisclaimer = manifestFixture(); + shortDisclaimer.scope_disclaimer = "fine"; + expect(benchmarkManifestSchema.safeParse(shortDisclaimer).success).toBe(false); + }); + + it("requires every versioning-policy commitment to be affirmed", () => { + const manifest = manifestFixture(); + manifest.versioning_policy.case_ids_immutable = false; + expect(benchmarkManifestSchema.safeParse(manifest).success).toBe(false); + }); +}); + +describe("rubric schema", () => { + it("accepts the committed rubric", () => { + expect(rubricSchema.safeParse(readJson("rubric.json")).success).toBe(true); + }); + + it("rejects duplicate dimension IDs", () => { + const rubric = readJson("rubric.json"); + rubric.dimensions.push({ ...rubric.dimensions[0] }); + const result = rubricSchema.safeParse(rubric); + expect(result.success).toBe(false); + expect(JSON.stringify(result.error.issues)).toContain("Duplicate rubric dimension"); + }); + + it("rejects a dimension claiming automation without naming a grader", () => { + const rubric = readJson("rubric.json"); + rubric.dimensions[0].deterministic_automation_possible = true; + rubric.dimensions[0].deterministic_grader_id = null; + expect(rubricSchema.safeParse(rubric).success).toBe(false); + }); + + it("rejects a dimension naming a grader while claiming no automation", () => { + const rubric = readJson("rubric.json"); + const automated = rubric.dimensions.find((d) => d.deterministic_automation_possible); + automated.deterministic_automation_possible = false; + expect(rubricSchema.safeParse(rubric).success).toBe(false); + }); + + it("requires an anchor for every point on the scale", () => { + const rubric = readJson("rubric.json"); + delete rubric.dimensions[0].anchors[2]; + expect(rubricSchema.safeParse(rubric).success).toBe(false); + }); +}); + +describe("benchmark case schema", () => { + it("accepts every committed case", () => { + for (const id of manifestFixture().case_ids) { + expect(parseBenchmarkCase(caseFixture(id)).ok, id).toBe(true); + } + }); + + it("rejects an unknown tag", () => { + const value = caseFixture(); + value.tags = ["not-a-real-tag"]; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("accepts only tags from the closed vocabulary", () => { + for (const tag of BENCHMARK_TAGS) { + const value = caseFixture(); + value.tags = [tag]; + // Variant cases additionally require the permutation tag, so the check + // is scoped to the tag vocabulary itself. + value.variant_of = null; + value.variant_kind = null; + expect(parseBenchmarkCase(value).ok, tag).toBe(true); + } + }); + + it("rejects a malformed case ID", () => { + const value = caseFixture(); + value.case_id = "case-1"; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects malformed candidate IDs", () => { + const value = caseFixture(); + value.input.candidates[0].id = "Has Spaces And Capitals"; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects duplicate candidate IDs", () => { + const value = caseFixture(); + value.input.candidates[1].id = value.input.candidates[0].id; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("accepts duplicate display names when the IDs are distinct", () => { + const value = caseFixture("case-015"); + const names = value.input.candidates.map((candidate) => candidate.name); + expect(new Set(names).size).toBeLessThan(names.length); + expect(parseBenchmarkCase(value).ok).toBe(true); + }); + + it("requires the synthetic-data policy metadata", () => { + const withoutSynthetic = caseFixture(); + withoutSynthetic.synthetic = false; + expect(parseBenchmarkCase(withoutSynthetic).ok).toBe(false); + + const withoutPolicy = caseFixture(); + withoutPolicy.data_policy = "real"; + expect(parseBenchmarkCase(withoutPolicy).ok).toBe(false); + }); + + it("rejects fewer than the minimum candidates", () => { + const value = caseFixture(); + value.input.candidates = value.input.candidates.slice(0, 1); + value.deterministic_expectations.expected_candidate_ids = value.input.candidates.map((c) => c.id); + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects more than the maximum candidates", () => { + const value = caseFixture(); + value.input.candidates = Array.from({ length: 11 }, (_, index) => ({ + id: `candidate-${index}`, + name: `Candidate ${index}`, + description: "A fictional candidate description.", + })); + value.deterministic_expectations.expected_candidate_ids = value.input.candidates.map((c) => c.id); + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects more scenarios than the shared contract allows", () => { + const value = caseFixture(); + value.input.scenarios = Array.from({ length: 6 }, (_, index) => `Scenario ${index}.`); + value.deterministic_expectations.required_scenario_coverage = value.input.scenarios; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects expected_candidate_ids that disagree with the candidate list", () => { + const value = caseFixture(); + value.deterministic_expectations.expected_candidate_ids = ["ghost"]; + const parsed = parseBenchmarkCase(value); + expect(parsed.ok).toBe(false); + expect(parsed.issues.join(" ")).toContain("expected_candidate_ids"); + }); + + it("rejects a pair count that does not match the candidate count", () => { + const value = caseFixture("case-015"); + value.deterministic_expectations.expected_pair_count = 3; + const parsed = parseBenchmarkCase(value); + expect(parsed.ok).toBe(false); + expect(parsed.issues.join(" ")).toContain("expected_pair_count must be 6"); + }); + + it("requires 4 logical stages with pairing and 3 without", () => { + const pairing = caseFixture("case-015"); + pairing.deterministic_expectations.required_stage_count = 3; + expect(parseBenchmarkCase(pairing).ok).toBe(false); + + const noPairing = caseFixture("case-001"); + noPairing.deterministic_expectations.required_stage_count = 4; + expect(parseBenchmarkCase(noPairing).ok).toBe(false); + }); + + it("rejects expected_best_pair_ids when pairing is disabled", () => { + const value = caseFixture("case-001"); + value.deterministic_expectations.expected_best_pair_ids = ["nadia-brookfield", "owen-kestrel"]; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects a winner expectation that names an unknown candidate", () => { + const value = caseFixture(); + value.deterministic_expectations.allowed_winner_ids = ["nobody"]; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects a candidate that is both allowed and forbidden", () => { + const value = caseFixture(); + const id = value.input.candidates[0].id; + value.deterministic_expectations.allowed_winner_ids = [id]; + value.deterministic_expectations.forbidden_winner_ids = [id]; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("allows a case to make no winner claim at all", () => { + const value = caseFixture("case-008"); + expect(value.deterministic_expectations.allowed_winner_ids).toBeNull(); + expect(parseBenchmarkCase(value).ok).toBe(true); + }); + + it("requires a fake-provider score plan for every candidate", () => { + const value = caseFixture(); + delete value.fake_provider_plan.candidate_scores[value.input.candidates[0].id]; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects a fake-provider plan scoring an unknown candidate", () => { + const value = caseFixture(); + value.fake_provider_plan.candidate_scores["ghost-candidate"] = { default: 5 }; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects a scenario override index beyond the case's scenarios", () => { + const value = caseFixture("case-001"); + value.fake_provider_plan.scenario_weight_deltas = { 4: { domain_expertise: 5 } }; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("requires variant_of and variant_kind to be set together", () => { + const value = caseFixture("case-011"); + value.variant_kind = null; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("requires a variant to carry the permutation tag", () => { + const value = caseFixture("case-011"); + value.tags = value.tags.filter((tag) => tag !== "permutation"); + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("rejects a case that is a variant of itself", () => { + const value = caseFixture("case-011"); + value.variant_of = value.case_id; + expect(parseBenchmarkCase(value).ok).toBe(false); + }); + + it("defaults known_defects to an empty list", () => { + const value = caseFixture("case-002"); + delete value.known_defects; + const parsed = benchmarkCaseSchema.parse(value); + expect(parsed.known_defects).toEqual([]); + }); + + it("rejects a known defect with a malformed ID or a missing reference", () => { + const badId = caseFixture("case-001"); + badId.known_defects[0].id = "oops"; + expect(parseBenchmarkCase(badId).ok).toBe(false); + + const noReference = caseFixture("case-001"); + delete noReference.known_defects[0].reference; + expect(parseBenchmarkCase(noReference).ok).toBe(false); + }); +}); + +describe("pipeline compatibility probe", () => { + it("accepts the shape the benchmark was written against", () => { + expect( + assertPipelineCompatibility({ + criteriaKeys: Array.from({ length: 7 }, (_, index) => `criterion_${index}`), + maxLogicalStages: 4, + declaredVersion: EVAL_PIPELINE_VERSION, + }), + ).toBe(true); + }); + + it("refuses a different criterion count", () => { + expect(() => + assertPipelineCompatibility({ criteriaKeys: ["only_one"], maxLogicalStages: 4 }), + ).toThrow(/expected 7 scoring criteria/); + }); + + it("refuses a different logical stage ceiling", () => { + expect(() => + assertPipelineCompatibility({ + criteriaKeys: Array.from({ length: 7 }, (_, index) => `c${index}`), + maxLogicalStages: 5, + }), + ).toThrow(/expected 4 maximum logical stages/); + }); + + it("refuses a benchmark written for a different pipeline generation", () => { + expect(() => + assertPipelineCompatibility({ + criteriaKeys: Array.from({ length: 7 }, (_, index) => `c${index}`), + maxLogicalStages: 4, + declaredVersion: "v1-ancient", + }), + ).toThrow(/requires pipeline/); + }); +}); + +describe("artifact policy scanning", () => { + it("passes clean, repository-relative content", () => { + expect( + findArtifactPolicyViolations(JSON.stringify({ path: "evals/datasets/decision-benchmark-v1" })), + ).toEqual([]); + }); + + it("detects an OpenAI-shaped key", () => { + const violations = findArtifactPolicyViolations('{"k":"sk-abcdefghijklmnopqrstuvwxyz012345"}'); + expect(violations.join(" ")).toContain("possible secret"); + }); + + it("detects a bearer token and an authorization header", () => { + expect(findArtifactPolicyViolations('"Bearer abcdefghijklmnopqrstuvwx"').length).toBeGreaterThan(0); + expect(findArtifactPolicyViolations('{"authorization": "x"}').length).toBeGreaterThan(0); + }); + + it("detects a unix absolute path", () => { + const violations = findArtifactPolicyViolations('{"dir":"/Users/someone/project"}'); + expect(violations.join(" ")).toContain("absolute path"); + }); + + it("detects a file:// URL", () => { + expect(findArtifactPolicyViolations('"file:///tmp/x"').length).toBeGreaterThan(0); + }); + + it("throws rather than writing a dirty artifact", () => { + expect(() => + assertArtifactIsPolicyClean({ key: "sk-abcdefghijklmnopqrstuvwxyz012345" }, "summary.json"), + ).toThrow(/Refusing to write summary.json/); + }); +}); + +describe("shared helpers", () => { + it("canonicalises pair keys independent of order", () => { + expect(canonicalPairKey("b", "a")).toBe(canonicalPairKey("a", "b")); + }); + + it("never marks a numeric delta as significant", () => { + expect(numericDelta(1, 5)).toEqual({ + baseline: 1, + candidate: 5, + delta: 4, + significance: "not_assessed", + }); + }); + + it("reports a null delta when either side is unavailable", () => { + expect(numericDelta(null, 5).delta).toBeNull(); + expect(numericDelta(1, undefined).delta).toBeNull(); + }); +}); diff --git a/evals/schemas/evaluationReport.js b/evals/schemas/evaluationReport.js new file mode 100644 index 0000000..1af9baa --- /dev/null +++ b/evals/schemas/evaluationReport.js @@ -0,0 +1,196 @@ +/** + * @file Human-review and comparison report schemas (Phase 3A evaluation harness). + * + * Two deliberate design constraints are encoded here: + * + * 1. **Dimension-level scores are never discarded.** A report MAY carry an + * aggregate for convenience, but the per-dimension scores are required, so + * "the explanation quality was 3.1" can always be unpacked into which + * dimension was weak. Collapsing qualitative judgment into one opaque + * number is exactly the failure this phase exists to avoid. + * + * 2. **A comparison never claims statistical significance.** The verdict + * vocabulary is `improved | regressed | unchanged | inconclusive`, and + * numeric deltas (cost, tokens, duration) are reported with an explicit + * `significance: "not_assessed"` marker. Two runs of a benchmark this size + * cannot support a significance claim, so the schema does not offer a + * field in which to make one. + */ +import { z } from "zod"; +import { benchmarkCaseIdSchema } from "./benchmarkCase.js"; + +export const EVALUATION_REPORT_SCHEMA_VERSION = "1.0.0"; + +export const COMPARISON_VERDICTS = Object.freeze([ + "improved", + "regressed", + "unchanged", + "inconclusive", + "baseline_change_required", +]); + +/** 0-4 anchored scale, plus two explicit non-scores. */ +export const humanReviewScoreSchema = z.union([ + z.number().int().min(0).max(4), + z.literal("not_applicable"), + z.literal("cannot_determine"), +]); + +export const humanReviewDimensionSchema = z + .object({ + dimension_id: z.string().min(1), + label: z.string().min(1), + what_is_judged: z.string().min(1), + anchors: z.record(z.string().min(1)), + score: humanReviewScoreSchema.nullable(), + reviewer_notes: z.string(), + }) + .strict(); + +export const humanReviewEntrySchema = z + .object({ + execution_id: z.string().min(1), + case_id: benchmarkCaseIdSchema, + scenario_index: z.number().int().min(0), + repetition: z.number().int().min(1), + /** Filled in by the reviewer, not by the harness. */ + reviewer: z.string(), + reviewed_at: z.string(), + dimensions: z.array(humanReviewDimensionSchema).min(1), + overall_notes: z.string(), + }) + .strict(); + +export const humanReviewTemplateSchema = z + .object({ + schema_version: z.string().min(1), + run_id: z.string().min(1), + benchmark_id: z.string().min(1), + benchmark_version: z.string().min(1), + rubric_version: z.string().min(1), + instructions: z.string().min(40), + scale_legend: z.record(z.string().min(1)), + entries: z.array(humanReviewEntrySchema), + }) + .strict(); + +/** + * Aggregate of a completed human review. `dimension_scores` is required and + * always retained; `aggregate_mean` is a convenience only, and is explicitly + * `null` when too few dimensions were actually scored to mean anything. + */ +export const humanReviewAggregateSchema = z + .object({ + scored_entries: z.number().int().min(0), + dimension_scores: z.record( + z + .object({ + scored_count: z.number().int().min(0), + not_applicable_count: z.number().int().min(0), + cannot_determine_count: z.number().int().min(0), + mean: z.number().min(0).max(4).nullable(), + min: z.number().int().min(0).max(4).nullable(), + max: z.number().int().min(0).max(4).nullable(), + }) + .strict(), + ), + aggregate_mean: z.number().min(0).max(4).nullable(), + aggregate_caveat: z.string().min(20), + }) + .strict(); + +const numericDeltaSchema = z + .object({ + baseline: z.number().nullable(), + candidate: z.number().nullable(), + delta: z.number().nullable(), + /** + * Always "not_assessed" in Phase 3A. A benchmark of this size, run without + * a designed repetition schedule, cannot support a significance claim. + */ + significance: z.literal("not_assessed"), + }) + .strict(); + +export const caseComparisonSchema = z + .object({ + case_id: benchmarkCaseIdSchema, + verdict: z.enum(COMPARISON_VERDICTS), + reasons: z.array(z.string().min(1)), + winner_changed: z.boolean(), + ranking_changed: z.boolean(), + best_pair_changed: z.boolean(), + structured_evidence_changed: z.boolean(), + explanation_changed: z.boolean(), + required_failures: numericDeltaSchema, + advisory_failures: numericDeltaSchema, + expected_failures: numericDeltaSchema, + schema_failures: numericDeltaSchema, + }) + .strict(); + +export const comparisonReportSchema = z + .object({ + schema_version: z.string().min(1), + generated_at: z.string().datetime(), + baseline_run_id: z.string().min(1), + candidate_run_id: z.string().min(1), + benchmark_id: z.string().min(1), + benchmark_version: z.string().min(1), + verdict: z.enum(COMPARISON_VERDICTS), + verdict_reasons: z.array(z.string().min(1)).min(1), + invariants: z + .object({ + required_failures: numericDeltaSchema, + advisory_failures: numericDeltaSchema, + expected_failures: numericDeltaSchema, + schema_failures: numericDeltaSchema, + passed_cases: numericDeltaSchema, + }) + .strict(), + defect_observations: z + .object({ + unchanged: z.array(z.string()), + disappeared: z.array(z.string()), + appeared: z.array(z.string()), + changed_signature: z.array(z.string()), + moved: z.array(z.string()), + }) + .strict(), + cost: numericDeltaSchema, + tokens: numericDeltaSchema, + duration_ms: numericDeltaSchema, + /** Rubric comparison is only populated when BOTH runs carry a review. */ + rubric: z + .object({ + compared: z.boolean(), + reason: z.string().min(1), + dimensions: z.record(numericDeltaSchema), + }) + .strict(), + stability: z + .object({ + compared: z.boolean(), + reason: z.string().min(1), + baseline_winner_agreement: z.number().min(0).max(1).nullable(), + candidate_winner_agreement: z.number().min(0).max(1).nullable(), + }) + .strict(), + winner_changes: z.array(benchmarkCaseIdSchema), + ranking_changes: z.array(benchmarkCaseIdSchema), + pair_changes: z.array(benchmarkCaseIdSchema), + cases: z.array(caseComparisonSchema), + limitations: z.array(z.string().min(1)).min(1), + }) + .strict(); + +/** Reusable helper so every numeric delta is built the same, honest way. */ +export function numericDelta(baseline, candidate) { + const bothNumeric = typeof baseline === "number" && typeof candidate === "number"; + return { + baseline: typeof baseline === "number" ? baseline : null, + candidate: typeof candidate === "number" ? candidate : null, + delta: bothNumeric ? Number((candidate - baseline).toFixed(10)) : null, + significance: "not_assessed", + }; +} diff --git a/evals/schemas/evaluationRun.js b/evals/schemas/evaluationRun.js new file mode 100644 index 0000000..053a624 --- /dev/null +++ b/evals/schemas/evaluationRun.js @@ -0,0 +1,323 @@ +/** + * @file Evaluation run and case-result schemas (Phase 3A evaluation harness). + * + * These schemas *wrap* production output — they never restate it. The decision + * response inside a case result is validated by the real public contract + * (`completedPipelineResponseSchema` from shared/contracts/decisionApi.js); + * this file adds only the benchmark metadata, grader results, human-review + * slots, and accounting the harness itself owns. + * + * Privacy rules encoded here (docs/evaluation/RUNBOOK.md, "Artifacts"): + * - no API keys, headers, or request bodies are ever recorded; + * - no machine-specific absolute path is ever recorded — paths are stored + * repository-relative; + * - only synthetic benchmark content is ever written. + */ +import { z } from "zod"; +import { benchmarkCaseIdSchema } from "./benchmarkCase.js"; + +export const EVALUATION_RUN_SCHEMA_VERSION = "1.0.0"; + +/** + * `expected_failure` is a grader that failed in exactly the way a case's + * `known_defects` list says it currently does. It does not gate the exit + * status. The inverse — a known defect that has stopped reproducing — is + * reported as a `fail`, so a fix can never land silently. + */ +export const GRADER_STATUSES = Object.freeze([ + "pass", + "fail", + "skip", + "error", + "expected_failure", +]); + +/** + * `required` graders gate the exit status: any failure makes a run fail and + * the CLI exit nonzero. `advisory` graders report a signal worth looking at + * without asserting that the pipeline is wrong. + */ +export const GRADER_SEVERITIES = Object.freeze(["required", "advisory"]); + +export const graderResultSchema = z + .object({ + grader_id: z.string().min(1), + grader_version: z.string().min(1), + severity: z.enum(GRADER_SEVERITIES), + status: z.enum(GRADER_STATUSES), + summary: z.string().min(1), + /** Stable machine-readable finding classes used for exact known-defect matching. */ + finding_codes: z.array(z.string().min(1)).default([]), + /** Structured failure identities; messages are never used as known-defect keys. */ + observations: z.array(z.record(z.unknown())).default([]), + /** Finding source of truth; details are exactly these human-readable messages. */ + findings: z.array(z.record(z.unknown())).default([]), + /** Present only when a scoped known-defect record reclassified this failure. */ + known_defect_id: z.string().min(1).optional(), + /** Exact expected-observation identities matched by this failure. */ + known_defect_observation_ids: z.array(z.string().min(1)).default([]), + /** XPASS-like signal: an expected product defect disappeared and needs review. */ + unexpected_defect_resolution: z.boolean().default(false), + /** Structured, human-readable specifics. Never raw provider payloads. */ + details: z.array(z.string().min(1)), + }) + .strict() + .superRefine((result, context) => { + const messages = result.findings.map((finding) => finding.message); + if (messages.some((message) => typeof message !== "string" || message.length === 0) || JSON.stringify(messages) !== JSON.stringify(result.details)) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["details"], + message: "Every human-readable failure detail must be derived from exactly one structured finding.", + }); + } + }); + +/** + * One pipeline execution. A case with N scenarios produces N executions, and a + * run with R repetitions produces N*R executions for that case — the harness + * never collapses them, so stability can be looked at per scenario. + */ +export const executionResultSchema = z + .object({ + execution_id: z.string().min(1), + case_id: benchmarkCaseIdSchema, + scenario_index: z.number().int().min(0), + scenario: z.string().min(1), + repetition: z.number().int().min(1), + status: z.enum(["completed", "failed", "skipped"]), + /** + * The pipeline response, stored as-is. + * + * Deliberately NOT validated against `completedPipelineResponseSchema` + * here. The whole purpose of the `contract-validity` grader is to detect a + * response that violates the public contract; if the artifact schema also + * enforced that contract, the harness would crash while recording the very + * defect it exists to find, and the finding would be lost. Contract + * validation happens in exactly one place — the grader — which reports the + * violation instead of destroying the evidence. + */ + response: z.record(z.unknown()).optional(), + /** Safe message only — never a stack trace or provider payload. */ + failure_reason: z.string().min(1).optional(), + /** A safe reason for an intentionally unstarted execution, e.g. budget guard. */ + skip_reason: z.string().min(1).optional(), + /** + * The decision content the comparison command actually diffs. Extracted + * here so a comparison never has to re-walk a full response, and so + * non-deterministic fields (request_id, timestamps, durations) are + * structurally excluded from any comparison. + */ + outcome: z + .object({ + winner_id: z.string().min(1), + ranking: z.array(z.string().min(1)), + best_pair_key: z.string().nullable(), + pairing_status: z.enum(["ok", "unavailable", "absent"]), + logical_provider_stage_count: z.number().int().min(0), + provider_attempt_count: z.number().int().min(0), + total_tokens: z.number().int().min(0), + estimated_cost_usd: z.number().nonnegative().nullable(), + duration_ms: z.number().int().min(0), + }) + .strict() + .optional(), + grader_results: z.array(graderResultSchema), + }) + .strict() + .superRefine((execution, context) => { + if (execution.status === "completed" && !execution.response) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["response"], + message: "A completed execution must carry its validated response.", + }); + } + if (execution.status === "failed" && !execution.failure_reason) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["failure_reason"], + message: "A failed execution must record why it failed.", + }); + } + if (execution.status === "skipped" && !execution.skip_reason) { + context.addIssue({ + code: z.ZodIssueCode.custom, + path: ["skip_reason"], + message: "A skipped execution must record why it was not started.", + }); + } + }); + +export const caseResultSchema = z + .object({ + case_id: benchmarkCaseIdSchema, + title: z.string().min(1), + tags: z.array(z.string().min(1)), + variant_of: benchmarkCaseIdSchema.nullable(), + variant_kind: z.string().nullable(), + executions: z.array(executionResultSchema).min(1), + /** Case-level graders run across every execution (scenario coverage). */ + grader_results: z.array(graderResultSchema), + required_failures: z.number().int().min(0), + advisory_failures: z.number().int().min(0), + passed: z.boolean(), + }) + .strict(); + +export const runManifestSchema = z + .object({ + run_id: z.string().min(1), + schema_version: z.string().min(1), + timestamp: z.string().datetime(), + mode: z.enum(["fixtures", "live"]), + benchmark_id: z.string().min(1), + benchmark_version: z.string().min(1), + benchmark_schema_version: z.string().min(1), + rubric_version: z.string().min(1), + /** Repository commit the run executed against; null outside a git repo. */ + git_commit: z.string().nullable(), + git_branch: z.string().nullable(), + provider: z.string().min(1), + model: z.string().min(1), + case_selection: z.array(benchmarkCaseIdSchema).min(1), + repetitions: z.number().int().min(1), + pairing_cases: z.number().int().min(0), + /** Aggregate accounting across every execution in the run. */ + logical_provider_stages: z.number().int().min(0), + provider_attempts: z.number().int().min(0), + input_tokens: z.number().int().min(0), + output_tokens: z.number().int().min(0), + total_tokens: z.number().int().min(0), + estimated_cost_usd: z.number().nonnegative().nullable(), + duration_ms: z.number().int().min(0), + grader_versions: z.record(z.string().min(1)), + /** Reasserts what this artifact is and is not allowed to contain. */ + artifact_policy: z + .object({ + synthetic_data_only: z.literal(true), + secrets_recorded: z.literal(false), + absolute_paths_recorded: z.literal(false), + }) + .strict(), + }) + .strict(); + +export const runSummarySchema = z + .object({ + run_id: z.string().min(1), + schema_version: z.string().min(1), + benchmark_id: z.string().min(1), + benchmark_version: z.string().min(1), + mode: z.enum(["fixtures", "live"]), + git_commit: z.string().nullable(), + repetitions: z.number().int().min(1), + case_count: z.number().int().min(0), + execution_count: z.number().int().min(0), + passed_cases: z.number().int().min(0), + failed_cases: z.number().int().min(0), + required_failures: z.number().int().min(0), + advisory_failures: z.number().int().min(0), + run_state: z.enum(["clean_pass", "pass_with_known_defects", "unexpected_failure", "baseline_change_required"]), + grader_totals: z.record( + z + .object({ + pass: z.number().int().min(0), + fail: z.number().int().min(0), + skip: z.number().int().min(0), + error: z.number().int().min(0), + expected_failure: z.number().int().min(0), + severity: z.enum(GRADER_SEVERITIES), + }) + .strict(), + ), + /** Failures attributed to a documented, pre-existing product defect. */ + expected_failures: z.number().int().min(0), + clean_pass_count: z.number().int().min(0), + affected_defect_ids: z.array(z.string().min(1)), + affected_execution_ids: z.array(z.string().min(1)), + unexpected_failures: z.number().int().min(0), + unexpected_defect_resolutions: z.number().int().min(0), + known_defect_observations: z.array(z.record(z.unknown())).default([]), + /** + * Winner agreement across repetitions of the same case+scenario. + * `insufficient_samples` when repetitions < 2 — a single run can never + * demonstrate stability, and this field says so rather than reporting a + * meaningless 100%. + */ + stability: z + .object({ + assessed: z.boolean(), + reason: z.string().min(1), + winner_agreement: z.number().min(0).max(1).nullable(), + ranking_agreement: z.number().min(0).max(1).nullable(), + }) + .strict(), + totals: z + .object({ + logical_provider_stages: z.number().int().min(0), + provider_attempts: z.number().int().min(0), + total_tokens: z.number().int().min(0), + estimated_cost_usd: z.number().nonnegative().nullable(), + duration_ms: z.number().int().min(0), + }) + .strict(), + /** Non-negotiable honesty text carried into every artifact. */ + disclaimer: z.string().min(40), + }) + .strict(); + +/** + * Conservative secret scan applied to every artifact before it is written. + * These patterns are deliberately narrow — they exist to catch an accidental + * key, not to claim the harness can detect every possible secret shape. + */ +const SECRET_PATTERNS = Object.freeze([ + { name: "openai-style key", pattern: /\bsk-[A-Za-z0-9_-]{16,}/ }, + { name: "bearer token header", pattern: /\bBearer\s+[A-Za-z0-9._-]{16,}/i }, + { name: "authorization header", pattern: /"authorization"\s*:/i }, + { name: "api key assignment", pattern: /\b(?:api[_-]?key|apikey|secret)\b\s*[:=]\s*["'][^"']{8,}/i }, +]); + +/** + * Absolute-path shapes that would leak a machine layout into an artifact. + * Repository-relative paths (`evals/datasets/...`) are unaffected. + */ +const ABSOLUTE_PATH_PATTERNS = Object.freeze([ + { name: "unix home path", pattern: /(?:^|["'\s])\/(?:Users|home|root|var|private|tmp)\// }, + { name: "windows drive path", pattern: /(?:^|["'\s])[A-Za-z]:\\\\?/ }, + { name: "file url", pattern: /file:\/\/\// }, +]); + +/** + * @param {string} serialized JSON text about to be written to an artifact. + * @returns {string[]} human-readable findings; empty means clean. + */ +export function findArtifactPolicyViolations(serialized) { + const violations = []; + for (const { name, pattern } of SECRET_PATTERNS) { + if (pattern.test(serialized)) violations.push(`possible secret (${name})`); + } + for (const { name, pattern } of ABSOLUTE_PATH_PATTERNS) { + if (pattern.test(serialized)) violations.push(`absolute path (${name})`); + } + return violations; +} + +/** + * Throws rather than writing an artifact that violates the recorded policy. + * Fail-closed is the right default here: a leaked key in a run directory is + * far worse than a failed evaluation run. + * @param {unknown} artifact + * @param {string} label + */ +export function assertArtifactIsPolicyClean(artifact, label) { + const serialized = typeof artifact === "string" ? artifact : JSON.stringify(artifact); + const violations = findArtifactPolicyViolations(serialized); + if (violations.length > 0) { + throw new Error( + `Refusing to write ${label}: ${violations.join(", ")}. ` + + "Evaluation artifacts must never contain secrets or machine-specific absolute paths.", + ); + } +} diff --git a/package.json b/package.json index dcbf1d3..fa54009 100644 --- a/package.json +++ b/package.json @@ -9,14 +9,21 @@ "typecheck": "tsc --noEmit", "build:dev": "vite build --mode development", "lint": "eslint .", - "lint:server": "eslint server/ai server/domain server/config server/pipeline server/http server.mjs", + "lint:server": "eslint server/ai server/domain server/config server/pipeline server/http server.mjs evals", "check:decision-readability": "node scripts/check-decision-source-readability.mjs", + "check:project-status": "node scripts/check-project-status.mjs", "check:unused-template": "node scripts/check-unused-template.mjs", "check:toolchain": "node scripts/check-toolchain.mjs && node scripts/check-router-toolchain.mjs", + "eval:validate": "node evals/cli/validate.mjs", + "eval:fixtures": "node evals/cli/fixtures.mjs", + "eval:live": "node evals/cli/live.mjs", + "eval:compare": "node evals/cli/compare.mjs", + "eval:update-integrity": "node evals/cli/update-integrity.mjs", "preview": "vite preview", - "test": "npm run test:frontend && npm run test:server", + "test": "npm run test:frontend && npm run test:server && npm run test:evals", "test:frontend": "vitest run --config vitest.config.ts", "test:server": "vitest run --config vitest.server.config.ts", + "test:evals": "vitest run --config vitest.evals.config.ts", "test:watch": "vitest" }, "dependencies": { diff --git a/scripts/check-project-status.mjs b/scripts/check-project-status.mjs new file mode 100644 index 0000000..73e6daa --- /dev/null +++ b/scripts/check-project-status.mjs @@ -0,0 +1,88 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; + +/** + * Current Phase 3A declarations are intentionally checked as committed text, + * not calculated by running tests. This keeps the documentation reviewable, + * deterministic, and network-free while preventing old green-baseline wording + * from returning to active status sections. + */ +const statusDocuments = [ + "docs/PROJECT_STATUS.md", + "docs/V2_ROADMAP.md", + "docs/decisions/ADR-0009-local-first-evaluation-harness.md", +]; + +const requiredCurrentStatements = [ + "pass_with_known_defects", + "fixture machinery: passed", + "12 clean cases", + "8 known-defect observations", + "4 affected executions", + "0 unexpected failures", + "0 unexpected defect resolutions", + "103 frontend tests", + "224 server tests", + "326 evaluation tests", + "653 total tests", +]; + +const stalePatterns = [ + /result:\s*passed\b/i, + /16\/16\s+(?:cases?\s+)?(?:clean\s+)?pass(?:ed|es)\b/i, + /\b629\s+(?:total\s+)?tests\b/i, + /\b309\s+(?:evaluation\s+)?tests\b/i, + /\b642\s+(?:total\s+)?tests\b/i, + /\b322\s+(?:evaluation\s+)?tests\b/i, +]; + +function isHistoricalContext(lines, index) { + const line = lines[index]; + if (/\b(historical|superseded)\b/i.test(line)) return true; + + for (let cursor = index - 1; cursor >= 0; cursor -= 1) { + if (/^#{1,6}\s/.test(lines[cursor])) { + return /\b(historical|superseded)\b/i.test(lines[cursor]); + } + } + return false; +} + +const violations = []; +const documents = await Promise.all( + statusDocuments.map(async (relativePath) => ({ + relativePath, + contents: await readFile(path.resolve(relativePath), "utf8"), + })), +); +const allCurrentStatusText = documents.map(({ contents }) => contents).join("\n"); + +for (const statement of requiredCurrentStatements) { + if (!allCurrentStatusText.toLowerCase().includes(statement)) { + violations.push(`Missing current Phase 3A declaration: "${statement}".`); + } +} + +for (const { relativePath, contents } of documents) { + const lines = contents.split(/\r?\n/); + lines.forEach((line, index) => { + for (const pattern of stalePatterns) { + if (pattern.test(line) && !isHistoricalContext(lines, index)) { + violations.push( + `${relativePath}:${index + 1} contains stale active Phase 3A status text: "${line.trim()}". Mark earlier evidence under a Historical or Superseded heading instead.`, + ); + } + } + }); +} + +if (violations.length > 0) { + console.error(`Project-status check failed: ${violations.length} documentation issue(s).`); + for (const violation of violations) console.error(`- ${violation}`); + process.exitCode = 1; +} else { + console.log( + `Project-status check passed: ${statusDocuments.length} current-status document(s) contain the committed Phase 3A baseline and no stale active totals.`, + ); +} diff --git a/scripts/check-project-status.test.js b/scripts/check-project-status.test.js new file mode 100644 index 0000000..9a54642 --- /dev/null +++ b/scripts/check-project-status.test.js @@ -0,0 +1,100 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import path from "node:path"; + +const execFileAsync = promisify(execFile); +const repoRoot = path.resolve(import.meta.dirname, ".."); +const scriptPath = path.join(repoRoot, "scripts", "check-project-status.mjs"); +const projectStatusPath = path.join(repoRoot, "docs", "PROJECT_STATUS.md"); +const roadmapPath = path.join(repoRoot, "docs", "V2_ROADMAP.md"); +const adrPath = path.join(repoRoot, "docs", "decisions", "ADR-0009-local-first-evaluation-harness.md"); +const statusPaths = [projectStatusPath, roadmapPath, adrPath]; + +async function runScript() { + return execFileAsync("node", [scriptPath], { cwd: repoRoot }); +} + +/** + * The guard intentionally reads the actual committed documents. These tests + * make focused temporary edits and restore every file after each assertion. + */ +describe("scripts/check-project-status.mjs", () => { + let originals = null; + + async function captureOriginals() { + if (originals === null) { + originals = await Promise.all(statusPaths.map((file) => readFile(file, "utf8"))); + } + } + + async function appendProjectStatus(text) { + await captureOriginals(); + await writeFile(projectStatusPath, `${originals[0]}\n${text}\n`); + } + + afterEach(async () => { + if (originals !== null) { + await Promise.all(statusPaths.map((file, index) => writeFile(file, originals[index]))); + originals = null; + } + }); + + it("passes against the committed current-state documentation", async () => { + const { stdout } = await runScript(); + expect(stdout).toContain("Project-status check passed"); + }); + + it("rejects an active result: PASSED claim", async () => { + await appendProjectStatus("result: PASSED"); + await expect(runScript()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("result: PASSED"), + }); + }); + + it("rejects active 629/309 totals", async () => { + await appendProjectStatus("Verification: 629 total tests (309 evaluation tests)."); + await expect(runScript()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("629 total tests"), + }); + }); + + it("rejects active 642/322 totals", async () => { + await appendProjectStatus("Verification: 642 total tests (322 evaluation tests)."); + await expect(runScript()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("642 total tests"), + }); + }); + + it("allows stale values in a clearly marked historical section", async () => { + await appendProjectStatus("## Historical verification (superseded)\nresult: PASSED\n629 total tests (309 evaluation tests)."); + const { stdout } = await runScript(); + expect(stdout).toContain("Project-status check passed"); + }); + + it("rejects a missing current run-state declaration", async () => { + await captureOriginals(); + await Promise.all(statusPaths.map(async (file, index) => + writeFile(file, originals[index].replaceAll("pass_with_known_defects", "baseline-pending")), + )); + await expect(runScript()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("pass_with_known_defects"), + }); + }); + + it("rejects missing current total declarations", async () => { + await captureOriginals(); + await Promise.all(statusPaths.map(async (file, index) => + writeFile(file, originals[index].replaceAll("653 total tests", "total pending")), + )); + await expect(runScript()).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("653 total tests"), + }); + }); +}); diff --git a/server/ai/providerFactory.js b/server/ai/providerFactory.js index 82143ef..a40a4ab 100644 --- a/server/ai/providerFactory.js +++ b/server/ai/providerFactory.js @@ -11,7 +11,13 @@ */ import { ConfigurationError } from "./errors.js"; -import { createOpenAIProvider, DEFAULT_OPENAI_MODEL, REASONING_EFFORT_VALUES } from "./providers/openaiProvider.js"; +import { createOpenAIProvider, REASONING_EFFORT_VALUES } from "./providers/openaiProvider.js"; +import { resolveOpenAIModel } from "../config/env.js"; + +/** Pure model resolution for callers that must validate spend before construction. */ +export function resolveProviderModel({ env = process.env } = {}) { + return resolveOpenAIModel({ env }); +} /** * @param {{ env?: Record }} [options] @@ -22,7 +28,7 @@ export function createProvider({ env = process.env } = {}) { if (!apiKey) { throw new ConfigurationError("OPENAI_API_KEY is required to construct the OpenAI provider."); } - const model = env.OPENAI_MODEL || DEFAULT_OPENAI_MODEL; + const model = resolveProviderModel({ env }); let reasoningEffort; const rawEffort = env.OPENAI_REASONING_EFFORT; diff --git a/server/config/env.js b/server/config/env.js index 95a9e09..acb2254 100644 --- a/server/config/env.js +++ b/server/config/env.js @@ -22,6 +22,7 @@ import { readFileSync, existsSync } from "fs"; import { resolve } from "path"; import { REASONING_EFFORT_VALUES } from "../ai/providers/openaiProvider.js"; +import { DEFAULT_OPENAI_MODEL } from "../ai/providers/openaiProvider.js"; import { DECISION_INPUT_LIMITS, DEFAULT_RUNTIME_MAX_CANDIDATES } from "../../shared/contracts/decisionInputLimits.js"; function parseEnvFile(path) { @@ -39,6 +40,11 @@ function parseEnvFile(path) { return result; } +/** Pure, non-constructing provider model resolution for live budget preflight. */ +export function resolveOpenAIModel({ env = process.env } = {}) { + return env.OPENAI_MODEL || DEFAULT_OPENAI_MODEL; +} + /** * @param {{cwd?: string, env?: Record}} [options] * @returns {{ loadedFiles: string[] }} which files were found and applied, diff --git a/server/pipeline/runPipeline.js b/server/pipeline/runPipeline.js index 5621f61..ca2bc9e 100644 --- a/server/pipeline/runPipeline.js +++ b/server/pipeline/runPipeline.js @@ -70,14 +70,29 @@ const DEFAULT_STAGE_TIMEOUT_MS = 90000; // and batch pairing scale with the configured maximum candidate/pair // count plus a fixed per-item overhead; decision explanation is a fixed // narrative shape regardless of run size. -const CONTEXT_ANALYSIS_MAX_TOKENS = 3000; -const CANDIDATE_SCORING_TOKENS_PER_CANDIDATE = 1100; -const CANDIDATE_SCORING_FIXED_OVERHEAD = 300; -const PAIRING_TOKENS_PER_PAIR = 380; -const PAIRING_FIXED_OVERHEAD = 200; -const DECISION_EXPLANATION_MAX_TOKENS = 2200; - -const MAX_BATCH_INTEGRITY_ATTEMPTS = 2; +// Exported for conservative evaluation planning. This is policy metadata only: +// the pipeline continues to use the same values below and has no eval import. +export const PROVIDER_COST_POLICY = Object.freeze({ + outputTokenBudgets: Object.freeze({ + contextAnalysis: 3000, + candidateScoringPerCandidate: 1100, + candidateScoringOverhead: 300, + pairingPerPair: 380, + pairingOverhead: 200, + decisionExplanation: 2200, + }), + maxBatchIntegrityExecutions: 2, + maxProviderAttemptsPerRequest: 2, +}); + +const CONTEXT_ANALYSIS_MAX_TOKENS = PROVIDER_COST_POLICY.outputTokenBudgets.contextAnalysis; +const CANDIDATE_SCORING_TOKENS_PER_CANDIDATE = PROVIDER_COST_POLICY.outputTokenBudgets.candidateScoringPerCandidate; +const CANDIDATE_SCORING_FIXED_OVERHEAD = PROVIDER_COST_POLICY.outputTokenBudgets.candidateScoringOverhead; +const PAIRING_TOKENS_PER_PAIR = PROVIDER_COST_POLICY.outputTokenBudgets.pairingPerPair; +const PAIRING_FIXED_OVERHEAD = PROVIDER_COST_POLICY.outputTokenBudgets.pairingOverhead; +const DECISION_EXPLANATION_MAX_TOKENS = PROVIDER_COST_POLICY.outputTokenBudgets.decisionExplanation; + +const MAX_BATCH_INTEGRITY_ATTEMPTS = PROVIDER_COST_POLICY.maxBatchIntegrityExecutions; // This architecture has exactly 4 logical model-backed stages by design // (context, scoring, pairing, decision) — a fixed fact about the pipeline, diff --git a/vitest.evals.config.ts b/vitest.evals.config.ts new file mode 100644 index 0000000..d4c1f7e --- /dev/null +++ b/vitest.evals.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from "vitest/config"; + +/** + * Evaluation-harness tests (Phase 3A). Kept in their own project so the + * frontend, backend, and evaluation test counts stay separately reportable, + * and so a change to the harness can be run in isolation. + * + * These tests execute the real production pipeline with offline fake + * providers. None of them makes a network request, and none of them calls + * OpenAI — a test in evals/repositoryProtection.test.js enforces that the + * harness cannot reach the provider factory from fixture mode. + */ +export default defineConfig({ + test: { + environment: "node", + globals: true, + include: ["evals/**/*.test.js"], + }, +});