From 2d6e7f95338e4c8246172fe0aec5f59d23a0df41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 09:25:32 +0000 Subject: [PATCH 01/15] UVH-IN5: show the user-value chain on /evals when it has data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chain funnel was mounted on run detail but invisible on the runs where it was the only thing to show. Two gates decide whether the insight rail exists at all — `run-insight-rail.tsx`'s emptiness check and `run-detail-view.tsx`'s `hasInsightContent` — and both counted only the triage, goal-completion and groundedness cards. A run with a derived chain and no judge or triage output rendered no rail, so the funnel inside it was never drawn. Adding the card to those checks would have traded one bug for another, which is why its exclusion was deliberate and documented: the card is a truthy fragment whose two halves each suppress themselves from the inside, so counting the NODE keeps an otherwise-empty rail alive as a full-height column of dead space on every run with no insight content at all. The gates now read a fact about the DATA instead. A probe mounted above every layout branch asks the same rollup query the funnel uses — undefined while loading, null for a run with no rollup, which is exactly the panel's own render condition — and reports one boolean both gates consume. Convex de-duplicates identical subscriptions, so asking twice costs one query. The probe lives beside the panels and carries the same ErrorBoundary they do, for the same reason: `useQuery` throws when the query is not deployed (this is still dark-shipped) or when there is no ConvexProvider, and a probe that took a run-detail page down with it would be worse than the empty rail it exists to prevent. Undeployed reads as "no funnel", which is correct. The state starts false, so a run without one never flashes an empty rail on the way to finding out. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-run-detail-funnel-visibility.md | 13 +++ .../evals/__tests__/run-insight-rail.test.tsx | 52 +++++++++- .../src/components/evals/run-detail-view.tsx | 45 ++++++++- .../src/components/evals/run-insight-rail.tsx | 95 +++++++++++-------- .../user-value-chain/StageFunnelPanels.tsx | 66 ++++++++++++- 5 files changed, 225 insertions(+), 46 deletions(-) create mode 100644 .changeset/evals-run-detail-funnel-visibility.md diff --git a/.changeset/evals-run-detail-funnel-visibility.md b/.changeset/evals-run-detail-funnel-visibility.md new file mode 100644 index 0000000000..303df22cb6 --- /dev/null +++ b/.changeset/evals-run-detail-funnel-visibility.md @@ -0,0 +1,13 @@ +--- +"@mcpjam/inspector": patch +--- + +The user-value chain is visible on `/evals` run detail when it has data + +The chain funnel was mounted on run detail but could not be seen on the runs where it was the only thing to show. Two gates decide whether the insight rail exists at all — `run-insight-rail.tsx`'s emptiness check and `run-detail-view.tsx`'s `hasInsightContent` — and both counted only the triage, goal-completion and groundedness cards. A run with a derived chain and no judge or triage output rendered no rail, so the funnel it contained was never drawn. + +Adding the chain card to those checks would have traded one bug for another, which is why the exclusion was deliberate and documented: the card is a truthy fragment whose two halves each suppress themselves from the inside, so counting the NODE would keep an otherwise-empty rail alive as a full-height column of dead space on every run with no insight content at all. + +So the gates now read a fact about the DATA instead. A probe mounted above every layout branch asks the same rollup query the funnel itself uses — `undefined` while loading, `null` for a run with no rollup, which is exactly the panel's own render condition — and reports one boolean that both gates consume. Convex de-duplicates identical subscriptions, so asking twice costs one query, and the probe carries the same `ErrorBoundary` the panels do: `useQuery` throws when the query is not deployed or when there is no `ConvexProvider`, and a probe that took the page down with it would be worse than the empty rail it exists to prevent. Undeployed reads as "no funnel", which is correct. + +The state starts `false`, so a run without a funnel never flashes an empty rail on the way to finding out. diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx b/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx index a50a998e58..263ce25f69 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx +++ b/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx @@ -129,10 +129,14 @@ describe("RunAccuracyHeroBand", () => { />, ); - expect(screen.getByRole("heading", { name: /Run run-2/i })).toBeInTheDocument(); + expect( + screen.getByRole("heading", { name: /Run run-2/i }), + ).toBeInTheDocument(); expect(screen.getByText("Failed")).toBeInTheDocument(); expect(screen.getByText(/7 passed · 3 failed ·/)).toBeInTheDocument(); - const band = screen.getByRole("heading", { name: /Run run-2/i }).closest("section"); + const band = screen + .getByRole("heading", { name: /Run run-2/i }) + .closest("section"); expect(band).toHaveTextContent("Accuracy"); expect(band).toHaveTextContent("70%"); }); @@ -174,6 +178,50 @@ describe("RunInsightRail", () => { screen.queryByRole("heading", { name: "Latency by test (p50 / p95)" }), ).not.toBeInTheDocument(); }); + + it("stays closed when the chain card is the ONLY thing passed and it has no data", () => { + // The reason the card was excluded from the emptiness check to begin + // with: the node is truthy even when both of its halves render nothing, + // so counting the node would leave a full-height column of dead space. + const { container } = render( + } + />, + ); + + expect(container).toBeEmptyDOMElement(); + expect(screen.queryByTestId("chain-slot")).not.toBeInTheDocument(); + }); + + it("opens for a run whose ONLY insight is its user-value chain", () => { + // UVH-IN5, and the bug this fixes: a run with a derived chain but no + // judge or triage output rendered no rail at all, so the chain was + // invisible on exactly the runs where it was the whole story. + render( + Funnel} + userValueChainHasContent + />, + ); + + expect(screen.getByTestId("chain-slot")).toBeInTheDocument(); + }); + + it("still opens for other insight content when the chain has none", () => { + render( + Insights} + userValueChainCard={
} + userValueChainHasContent={false} + />, + ); + + expect(screen.getByTestId("triage-slot")).toBeInTheDocument(); + // The card is still mounted — it suppresses itself from the inside. + expect(screen.getByTestId("chain-slot")).toBeInTheDocument(); + }); }); describe("RunDetailMetricsCharts", () => { diff --git a/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx b/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx index f48f7e83e8..7bd910789f 100644 --- a/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx +++ b/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx @@ -45,7 +45,10 @@ import { type JudgeCase, } from "./goal-completion-presentation"; import { RunInsightBand, type InsightSeverity } from "./run-insight-band"; -import { SuiteRunStageFunnelPanel } from "@/components/shared/user-value-chain/StageFunnelPanels"; +import { + SuiteRunStageFunnelAvailability, + SuiteRunStageFunnelPanel, +} from "@/components/shared/user-value-chain/StageFunnelPanels"; import { ExplanatoryFlowOptIn } from "@/components/shared/usage-insights/ExplanatoryFlowOptIn"; import type { InsightsScope } from "@/hooks/useUsageInsights"; import { useAvailableModels } from "@/hooks/use-available-models"; @@ -576,6 +579,14 @@ export function RunDetailView({ const embeddedInResultsSplit = hideKpiStrip; + /** + * Whether this run has a stage funnel to draw, reported by the probe below. + * + * Starts `false` so a run without one never flashes an empty rail on the way + * to finding out; the probe flips it once the rollup query resolves. + */ + const [hasStageFunnel, setHasStageFunnel] = useState(false); + const serverQualityTriage = selectedRunDetails.status === "completed" && !serverQualityUnavailable ? ( ); + /** + * Mounted unconditionally, and deliberately NOT inside the rail or the band + * it informs — it answers whether those should open, so it has to exist + * before they do. Renders nothing; costs one query, which Convex shares with + * the panel's own subscription. + */ + const stageFunnelProbe = ( + + ); + const insightRail = ( ); + /** + * The chain counts toward "is there anything to show" — but only when it + * actually has a funnel to draw. + * + * Both gates below read this ONE boolean, and it comes from a probe rather + * than from the panel, because the panel cannot answer: neither gate mounts + * it until they have already decided to open. Counting the card itself + * instead would keep a rail alive on every run with no insight content at + * all, since the node is truthy whether or not it renders anything — which + * is why it was excluded from these checks in the first place, and why the + * fix has to be data-driven rather than a matter of adding the node. + */ const hasInsightContent = Boolean( serverQualityTriage || goalCompletionPanel || groundednessPanel || - actionableFindingsPanel, + actionableFindingsPanel || + hasStageFunnel, ); const triageFixCount = useMemo( @@ -1041,6 +1079,9 @@ export function RunDetailView({ omitIterationList && "px-3 py-3", )} > + {/* Renders nothing. Sits above every layout branch below because all of + them gate on the answer it reports. */} + {stageFunnelProbe} {onExportTraces || pluginSubmissionVersions.length > 0 || onShare ? ( // Always-on run-level actions — placed here (not the accuracy hero) so // they survive the folded run-detail layout that hides the hero. diff --git a/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx b/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx index 904f7ccea5..969ef91bc6 100644 --- a/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx +++ b/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx @@ -180,8 +180,8 @@ export function RunAccuracyHeroBand({ stats.total > 0 ? normalizeRunPassRatePercent(stats.passRate) : run.summary - ? normalizeRunPassRatePercent(run.summary.passRate) - : null; + ? normalizeRunPassRatePercent(run.summary.passRate) + : null; const trendChips = useMemo(() => { if (runTrendData.length < 2) return { points: [], hiddenCount: 0 }; @@ -251,10 +251,7 @@ export function RunAccuracyHeroBand({ {runClient || runServers.length > 0 ? (
{runClient ? ( - + ) : null} {visibleServers.map((name) => ( ); - const recentRunsBlock = hasRecentRuns && !hideRecentRuns ? ( -
-
-

Recent runs

- {trendChips.hiddenCount > 0 ? ( -

- Last {RUN_TREND_CHIP_LIMIT} of {runTrendData.length} -

- ) : null} -
-
- {trendChips.points.map((point) => ( - - ))} + const recentRunsBlock = + hasRecentRuns && !hideRecentRuns ? ( +
+
+

Recent runs

+ {trendChips.hiddenCount > 0 ? ( +

+ Last {RUN_TREND_CHIP_LIMIT} of {runTrendData.length} +

+ ) : null} +
+
+ {trendChips.points.map((point) => ( + + ))} +
-
- ) : null; + ) : null; // With run identity: title/stats and recent runs share one row; accuracy on the right. if (includeRunIdentity) { @@ -380,8 +378,8 @@ export function shouldShowRunAccuracyHero({ stats.total > 0 ? normalizeRunPassRatePercent(stats.passRate) : run.summary - ? normalizeRunPassRatePercent(run.summary.passRate) - : null; + ? normalizeRunPassRatePercent(run.summary.passRate) + : null; return passRatePercent !== null; } @@ -417,6 +415,7 @@ export function RunInsightRail({ goalCompletionCard, groundednessCard, userValueChainCard, + userValueChainHasContent = false, className, embedded = false, }: { @@ -433,18 +432,38 @@ export function RunInsightRail({ * diagram beside it is bought per pass and waits for a click. Same traces, * different price, so different affordance. * - * Deliberately NOT part of the emptiness check below. Both halves render - * nothing of their own when there is no derived chain and no analyzable - * cohort, and the node itself is truthy either way — counting it would keep - * an otherwise-empty rail alive as a full-height column of dead space on - * every run with no insight content at all. + * The NODE is deliberately still not part of the emptiness check below. + * Both halves render nothing of their own when there is no derived chain and + * no analyzable cohort, yet the node itself is truthy either way — counting + * it would keep an otherwise-empty rail alive as a full-height column of + * dead space on every run with no insight content at all. + * + * What the check reads instead is `userValueChainHasContent`, a fact about + * the DATA rather than about the node. That is what lets a run whose only + * insight is its chain open the rail — the case this card was previously + * invisible on — without reintroducing the empty column. */ userValueChainCard?: ReactNode; + /** + * Whether the chain card will actually draw something. + * + * Supplied by the caller, which asks a probe mounted outside this rail: the + * rail cannot ask the card, because it does not mount the card until it has + * already decided to exist. + */ + userValueChainHasContent?: boolean; className?: string; /** Flush layout inside the run-detail split (shared dividers, no card gaps). */ embedded?: boolean; }) { - if (!triageCard && !goalCompletionCard && !groundednessCard) return null; + if ( + !triageCard && + !goalCompletionCard && + !groundednessCard && + !userValueChainHasContent + ) { + return null; + } return (
, ); expect(getByTestId("sibling").textContent).toBe("the rest of the page"); }); @@ -158,7 +160,7 @@ describe("SwarmRunStageFunnelPanels — the query answers", () => { + />, ); expect(container.innerHTML).toBe(""); }); @@ -168,7 +170,7 @@ describe("SwarmRunStageFunnelPanels — the query cannot answer", () => { it("renders nothing instead of throwing", () => { queryThrows(); const { container } = render( - + , ); expect(container.textContent).toBe(""); }); @@ -179,8 +181,103 @@ describe("SwarmRunStageFunnelPanels — the query cannot answer", () => {
the rest of the page -
+
, ); expect(getByTestId("sibling").textContent).toBe("the rest of the page"); }); }); + +describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () => { + it("reports true only when the rollup actually answered", () => { + convex.useQuery.mockReturnValue(SUMMARY); + const onChange = vi.fn(); + render( + , + ); + expect(onChange).toHaveBeenCalledWith("run-1", true); + }); + + it.each([ + ["still loading", undefined], + ["a run with no rollup", null], + ])("reports false while %s", (_label, value) => { + convex.useQuery.mockReturnValue(value); + const onChange = vi.fn(); + render( + , + ); + expect(onChange).toHaveBeenCalledWith("run-1", false); + }); + + it("renders nothing and never reports when the query throws", () => { + // The dark-ship state. Undeployed must read as "no funnel", and the probe + // must not take the run-detail page down with it. + queryThrows(); + const onChange = vi.fn(); + const { container } = render( +
+ the rest of the page + +
, + ); + expect(onChange).not.toHaveBeenCalled(); + expect(container.textContent).toBe("the rest of the page"); + }); + + it("names the run each answer is about, so a stale one is detectable", () => { + // The run selector reuses one view across runs. An answer that did not + // name its run could not be told apart from the previous run's, and a + // stale `true` would open an empty rail on the run you switched to. + convex.useQuery.mockReturnValue(SUMMARY); + const onChange = vi.fn(); + const { rerender } = render( + , + ); + expect(onChange).toHaveBeenLastCalledWith("run-1", true); + + convex.useQuery.mockReturnValue(null); + rerender( + , + ); + expect(onChange).toHaveBeenLastCalledWith("run-2", false); + }); + + it("re-arms after a failing run: a later run is still probed", () => { + // An ErrorBoundary that has caught stays in its fallback for the life of + // the element, so the boundary is keyed by run. Without the key, one + // transient failure would hide the chain on every run after it. + queryThrows(); + const onChange = vi.fn(); + const { rerender } = render( + , + ); + expect(onChange).not.toHaveBeenCalled(); + + convex.useQuery.mockReturnValue(SUMMARY); + rerender( + , + ); + expect(onChange).toHaveBeenCalledWith("run-2", true); + }); +}); From 8e64ebedf29a7fd7134d492cb044959ca8bf3807 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 10:37:22 +0000 Subject: [PATCH 03/15] UVH-IN1: file a failed tool-call assertion at selection, not user value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `stepsToPromptTurns` promotes only `toolCalledWith` into `expectedToolCalls`, where the selection matcher grades it. The other three tool assertions — `toolCalledAtLeastOnce`, `toolNeverCalled`, `firstToolWas` — fall through to per-turn checks and reach the analyzer as predicate results, which `deriveUserValue` graded as user value. So a case asserting "tool get_project was never called" reported `userValue: failed / predicateFailed` with `selection: passed`. The chain said the user did not get what they asked for; what actually happened is the model picked the wrong tool. Across one audited prod window that single mis-routing is most of the gap between 2 selection failures and 12 user-value ones — the report card blaming the wrong stage on exactly the failures operators most need attributed. The three kinds now route to `selection`, and they do NOT share applicability: toolCalledAtLeastOnce failure -> missingToolCall expects a call firstToolWas failure -> unexpectedToolCall expects a call toolNeverCalled failure -> unexpectedToolCall expects NO call `toolNeverCalled` is the asymmetry that makes this a matrix rather than a set: a case whose only tool assertion forbids a call expects none, and turning `call` on for it would demand evidence of the very thing the case exists to rule out. Passing rows route too, not only failing ones — a case whose only selection assertion is "never call the admin tool", which did not call it, has MEASURED selection and found it sound. And they are routed, not copied: a failure filed at both stages would double-count one defect and make `firstFailedStage` depend on which stage a reader looked at first. For the same reason `buildStageAuthoredCase` stops counting them in `assertionCount`, so a `toolNeverCalled`-only case no longer reports a permanent user-value gap no author could close. `toolCalledWith` is deliberately untouched: already matcher-graded, and re-reading its point-in-time predicate row would let a raw residual contradict the adjudicated verdict. The discriminator was always present at runtime — `PredicateResult` carries the whole predicate — and a cast in `finalize-iteration` was erasing it. It now crosses with the row, and a row WITHOUT one grades exactly as before, which is why this bump changed no recorded row in the historical-parity corpus (only its version stamp moved). `STAGE_REASONS` does not move, so the backend mirror needs no re-pin. Verdicts, gate exit codes and pass/fail counts are unchanged: nothing in `iteration-verdict.ts`, the gate layers or the tallies reads stage rows — confirmed by the CLI suite (1182 tests) and the runner-parity snapshots, whose only diff is the version stamp. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- ...-tool-call-predicates-file-at-selection.md | 28 +++ .../__snapshots__/runner-parity.test.ts.snap | 84 ++++----- .../evals/__tests__/stage-inputs.test.ts | 81 +++++++++ .../services/evals/finalize-iteration.ts | 11 +- .../server/services/evals/stage-inputs.ts | 43 ++++- sdk/src/contract/index.ts | 2 + sdk/src/contract/stage-derivation.ts | 166 ++++++++++++++++-- sdk/src/eval-result-mapping.ts | 21 ++- sdk/tests/eval-run-decision-summary.test.ts | 10 +- .../eval-run-decision-summary-fixtures.json | 24 +-- sdk/tests/fixtures/parity/v1/MANIFEST.json | 56 +++--- .../fixtures/parity/v1/iteration-0001.json | 2 +- .../fixtures/parity/v1/iteration-0002.json | 4 +- .../fixtures/parity/v1/iteration-0003.json | 2 +- .../fixtures/parity/v1/iteration-0004.json | 2 +- .../fixtures/parity/v1/iteration-0005.json | 4 +- .../fixtures/parity/v1/iteration-0006.json | 4 +- .../fixtures/parity/v1/iteration-0007.json | 2 +- .../fixtures/parity/v1/iteration-0008.json | 4 +- .../fixtures/parity/v1/iteration-0009.json | 2 +- .../fixtures/parity/v1/iteration-0010.json | 2 +- .../fixtures/parity/v1/iteration-0011.json | 2 +- .../fixtures/parity/v1/iteration-0012.json | 2 +- .../fixtures/parity/v1/iteration-0013.json | 2 +- .../fixtures/parity/v1/iteration-0014.json | 2 +- .../fixtures/parity/v1/iteration-0015.json | 2 +- .../fixtures/parity/v1/iteration-0016.json | 4 +- .../fixtures/parity/v1/iteration-0017.json | 2 +- .../fixtures/parity/v1/iteration-0018.json | 2 +- .../fixtures/parity/v1/iteration-0019.json | 4 +- .../fixtures/stage-analytics-golden.json | 2 +- sdk/tests/stage-derivation.test.ts | 158 +++++++++++++++++ 32 files changed, 601 insertions(+), 135 deletions(-) create mode 100644 .changeset/evals-tool-call-predicates-file-at-selection.md diff --git a/.changeset/evals-tool-call-predicates-file-at-selection.md b/.changeset/evals-tool-call-predicates-file-at-selection.md new file mode 100644 index 0000000000..fedd831762 --- /dev/null +++ b/.changeset/evals-tool-call-predicates-file-at-selection.md @@ -0,0 +1,28 @@ +--- +"@mcpjam/sdk": patch +"@mcpjam/inspector": patch +--- + +A tool-call assertion that fails now files at `selection`, not `userValue` + +`stepsToPromptTurns` promotes only `toolCalledWith` into `expectedToolCalls`, where the selection matcher grades it. The other three tool assertions — `toolCalledAtLeastOnce`, `toolNeverCalled`, `firstToolWas` — fall through to per-turn checks and arrive at the analyzer as predicate results, which `deriveUserValue` graded as user value. + +So a case asserting "tool `get_project` was never called" reported `userValue: failed / predicateFailed` with `selection: passed`. The chain said the user did not get what they asked for; what actually happened is the model picked the wrong tool. Across one audited prod window that single mis-routing is most of the gap between 2 selection failures and 12 user-value ones — the report card blaming the wrong stage on the failures operators most need to attribute. + +The three kinds now route to `selection`, and they do **not** share applicability: + +| predicate | failure files as | expects a call? | +|---|---|---| +| `toolCalledAtLeastOnce` | `missingToolCall` | yes | +| `firstToolWas` | `unexpectedToolCall` | yes | +| `toolNeverCalled` | `unexpectedToolCall` | **no** | + +`toolNeverCalled` is the asymmetry that makes this a matrix rather than a set: a case whose only tool assertion forbids a call expects none, and turning `call` on for it would demand evidence of the very thing the case exists to rule out. + +Passing rows route too, not just failing ones. A case whose only selection assertion is "never call the admin tool", which did not call it, has *measured* selection and found it sound — reporting that as `notMeasured` understates what the run established. And they are **routed, not copied**: a failure filed at both stages would double-count one defect and make `firstFailedStage` depend on which stage a reader looked at first. For the same reason `buildStageAuthoredCase` stops counting them in `assertionCount`, so a `toolNeverCalled`-only case no longer reports a permanent user-value gap that no author could close. + +`toolCalledWith` is deliberately untouched: it is already matcher-graded, and re-reading its point-in-time predicate row here would let a raw residual contradict the adjudicated verdict. + +Two mechanical notes. The predicate discriminator was always present at runtime — `PredicateResult` carries the whole predicate — and was being erased by a cast in `finalize-iteration`; it now crosses with the row, and a row *without* one grades exactly as before, which is why the analyzer bump to 6 changed no recorded row in the historical-parity corpus. `STAGE_REASONS` does not move (the routing re-uses `missingToolCall` and `unexpectedToolCall`), so the backend mirror needs no re-pin. + +Verdicts, gate exit codes and pass/fail counts are unchanged: nothing in `iteration-verdict.ts`, the gate layers or the tallies reads stage rows. This does widen the D7 metadata-attribution judge's candidate population, since it gates on `firstFailedStage === "selection"` — intended, and the reason more selection failures now reach it is that they were always selection failures. diff --git a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap index 3fd9cea577..f7c36139d7 100644 --- a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap +++ b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap @@ -181,7 +181,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -210,7 +210,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -417,7 +417,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "stepIndex": 1, }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -446,7 +446,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -747,7 +747,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -776,7 +776,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -977,7 +977,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -1006,7 +1006,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -1246,7 +1246,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -1275,7 +1275,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -1494,7 +1494,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -1523,7 +1523,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -1720,7 +1720,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -1749,7 +1749,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -1999,7 +1999,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -2028,7 +2028,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -2201,7 +2201,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo "reason": "no widget render observations recorded", }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -2230,7 +2230,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -2430,7 +2430,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -2459,7 +2459,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -2616,7 +2616,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -2645,7 +2645,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -2801,7 +2801,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -2830,7 +2830,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -3040,7 +3040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -3069,7 +3069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -3242,7 +3242,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -3271,7 +3271,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -3437,7 +3437,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -3466,7 +3466,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -3617,7 +3617,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -3646,7 +3646,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -3816,7 +3816,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi "reason": "widget rendered (1/1 observation(s))", }, ], - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -3845,7 +3845,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -4040,7 +4040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -4069,7 +4069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -4294,7 +4294,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -4323,7 +4323,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -4503,7 +4503,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 0, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -4532,7 +4532,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { @@ -4659,7 +4659,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "stageMeasurements": { "rows": [ { @@ -4688,7 +4688,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, }, "stageResults": [ { diff --git a/mcpjam-inspector/server/services/evals/__tests__/stage-inputs.test.ts b/mcpjam-inspector/server/services/evals/__tests__/stage-inputs.test.ts index ba53d9f1ff..cd83d6a911 100644 --- a/mcpjam-inspector/server/services/evals/__tests__/stage-inputs.test.ts +++ b/mcpjam-inspector/server/services/evals/__tests__/stage-inputs.test.ts @@ -131,6 +131,87 @@ describe("buildStageAuthoredCase", () => { }); }); + // ── UVH-IN1: the per-kind tool-call predicate matrix ───────────────────── + // + // The three kinds do NOT share applicability, and treating them as one group + // gets a case wrong in both directions: a positive assertion that leaves + // `call` inapplicable, or a forbidden-call assertion that demands evidence + // of the very call it exists to rule out. + + const toolPredicateStep = (id: string, type: string): TestStep => + ({ + id, + kind: "assert", + assertion: { type, toolName: "get_project" }, + }) as TestStep; + + it.each(["toolCalledAtLeastOnce", "firstToolWas"])( + "%s expects a tool call and is not a user-value assertion", + (type) => { + const result = buildStageAuthoredCase({ + test: {}, + steps: [toolPredicateStep("a1", type)], + caseNeedsModel: true, + }); + expect(result).toEqual({ + mode: "model_driven", + expectsToolCall: true, + expectsWidgetRender: false, + // Routed to `selection`, so it cannot also be what grades user value. + assertionCount: 0, + }); + } + ); + + it("toolNeverCalled does NOT expect a tool call", () => { + // The asymmetry that makes this a matrix rather than a set. A case whose + // only tool assertion forbids a call expects none; turning `call` on would + // demand evidence of the thing the case exists to forbid. + const result = buildStageAuthoredCase({ + test: {}, + steps: [toolPredicateStep("a1", "toolNeverCalled")], + caseNeedsModel: true, + }); + expect(result).toEqual({ + mode: "model_driven", + expectsToolCall: false, + expectsWidgetRender: false, + assertionCount: 0, + }); + }); + + it("excludes tool-call predicates from assertionCount, keeping the rest", () => { + const result = buildStageAuthoredCase({ + test: { + successPredicates: [ + { type: "toolCalledAtLeastOnce", toolName: "get_project" }, + { type: "responseContains", needle: "Refunded" }, + ], + }, + steps: [ + toolPredicateStep("a1", "toolNeverCalled"), + predicateAssertStep("a2"), + ], + caseNeedsModel: true, + }); + // Two user-value assertions survive: `responseContains` and `noToolErrors`. + expect(result.assertionCount).toBe(2); + expect(result.expectsToolCall).toBe(true); + }); + + it("a widget assertion is never mistaken for a predicate kind", () => { + // Widget assertions are keyed by `kind`, not `type`, so reading `.type` + // off one must not accidentally match a selection kind. + const result = buildStageAuthoredCase({ + test: {}, + steps: [widgetAssertStep("a1")], + caseNeedsModel: true, + }); + expect(result.assertionCount).toBe(1); + expect(result.expectsWidgetRender).toBe(true); + expect(result.expectsToolCall).toBe(false); + }); + it("carries isNegativeTest through only when the case sets it", () => { expect( "isNegativeTest" in diff --git a/mcpjam-inspector/server/services/evals/finalize-iteration.ts b/mcpjam-inspector/server/services/evals/finalize-iteration.ts index 3fe6f7b723..62292841cb 100644 --- a/mcpjam-inspector/server/services/evals/finalize-iteration.ts +++ b/mcpjam-inspector/server/services/evals/finalize-iteration.ts @@ -32,6 +32,7 @@ import { type EvalSuiteFileToolPolicy, type StageAuthoredCase, type StageEvidence, + type StagePredicateResultLike, type StageResultRow, type StageSetupSignals, type IterationStatus as ContractIterationStatus, @@ -139,10 +140,12 @@ function buildStageEvidence(args: { ...(hasPrompts ? { prompts: args.prompts } : {}), ...(args.predicateResults?.length ? { - predicateResults: args.predicateResults as ReadonlyArray<{ - passed?: boolean; - reason?: string; - }>, + // The `predicate` discriminator crosses with the row. It was always + // present at runtime — `PredicateResult` carries the whole predicate + // — and this cast used to drop it, which is why UVH-IN1's routing + // could not tell a tool-selection assertion from a user-value one. + predicateResults: + args.predicateResults as ReadonlyArray, } : {}), ...(args.widgetRenderObservations?.length diff --git a/mcpjam-inspector/server/services/evals/stage-inputs.ts b/mcpjam-inspector/server/services/evals/stage-inputs.ts index 0aa6b58c97..9556b686c1 100644 --- a/mcpjam-inspector/server/services/evals/stage-inputs.ts +++ b/mcpjam-inspector/server/services/evals/stage-inputs.ts @@ -14,6 +14,8 @@ import { isAssertStep, + isPositiveToolCallPredicateKind, + isSelectionPredicateKind, isToolCallStep, isWidgetAssertion, type StageAuthoredCase, @@ -49,22 +51,55 @@ export function buildStageAuthoredCase(args: { const steps = args.steps ?? []; const turns = args.turns ?? []; + const assertSteps = steps.filter(isAssertStep); + + /** The predicate `type` an assertion carries, when it is a transcript one. */ + const predicateKind = (assertion: unknown): string | undefined => + isWidgetAssertion(assertion as never) + ? undefined + : (assertion as { type?: unknown })?.type as string | undefined; + + const assertedPredicateKinds = [ + ...assertSteps.map((s) => predicateKind(s.assertion)), + ...(args.test.successPredicates ?? []).map(predicateKind), + ]; + + // UVH-IN1. A case asserting "call tool X" expects a call just as surely as + // one authoring `expectedToolCalls` does, and before this its assertion set + // `call` to `notApplicable` — a stage the case plainly exercises reported as + // one it does not. + // + // `toolNeverCalled` is deliberately NOT among them: a case whose only tool + // assertion forbids a call expects none, and turning `call` on for it would + // demand evidence of the very thing the case exists to rule out. const expectsToolCall = (args.test.expectedToolCalls?.length ?? 0) > 0 || turns.some((t) => (t.expectedToolCalls?.length ?? 0) > 0) || - steps.some(isToolCallStep); + steps.some(isToolCallStep) || + assertedPredicateKinds.some(isPositiveToolCallPredicateKind); - const assertSteps = steps.filter(isAssertStep); const expectsWidgetRender = args.test.caseType === "widget_probe" || assertSteps.some((s) => isWidgetAssertion(s.assertion)); // What could speak to "the user's actual request was satisfied": authored // predicates, an expected output to compare against, and assert steps. + // + // Tool-call assertions are EXCLUDED, because UVH-IN1 routes their results to + // `selection`. Counting them here would leave `userValue` applicable with + // nothing left to grade it, so a case whose only assertion is + // "never call the admin tool" would report a permanent `notMeasured` + // user-value gap that no author could ever close. + const gradesUserValue = (kind: string | undefined) => + !isSelectionPredicateKind(kind); + const assertionCount = - (args.test.successPredicates?.length ?? 0) + + (args.test.successPredicates ?? []).filter((p) => + gradesUserValue(predicateKind(p)) + ).length + (args.test.expectedOutput !== undefined ? 1 : 0) + - assertSteps.length; + assertSteps.filter((s) => gradesUserValue(predicateKind(s.assertion))) + .length; return { mode: args.caseNeedsModel ? "model_driven" : "model_free", diff --git a/sdk/src/contract/index.ts b/sdk/src/contract/index.ts index 9725213137..90d9ca1acd 100644 --- a/sdk/src/contract/index.ts +++ b/sdk/src/contract/index.ts @@ -165,6 +165,8 @@ export { STAGE_METADATA_KEYS, STAGE_REASONS, deriveStageResults, + isPositiveToolCallPredicateKind, + isSelectionPredicateKind, stageDerivationSchema, stageDerivationToMetadata, stageReasonSchema, diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index fa6ddb8545..37c9542cc3 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -64,8 +64,14 @@ import { * is not persisted cannot be recomputed selectively, which is the entire * reason `sessionReadiness` stamps `READINESS_ANALYZER_VERSION` on every * record it writes. + * + * 6 (UVH-IN1): tool-call predicate results are routed to `selection` instead + * of falling to `userValue` as `predicateFailed`. `STAGE_REASONS` does not + * move — the routing re-uses `missingToolCall` and `unexpectedToolCall` — so + * the backend mirror needs no re-pin for this bump. Rows derived under 5 are + * identifiable as stale and can be recomputed selectively. */ -export const STAGE_ANALYZER_VERSION = 5; +export const STAGE_ANALYZER_VERSION = 6; /** * Why a stage landed where it did. @@ -228,8 +234,65 @@ export type StagePromptSummaryLike = { export type StagePredicateResultLike = { passed?: boolean; reason?: string; + /** + * The predicate that produced this row, when the producer kept it. + * + * Optional because it is genuinely absent on older rows and on producers + * that never carried it. A row without it is graded exactly as before — + * user-value evidence — so widening this type changes nothing on its own. + */ + predicate?: { type?: string; toolName?: string }; +}; + +/** + * Predicate kinds that are evidence about TOOL SELECTION, not user value. + * + * `stepsToPromptTurns` promotes only `toolCalledWith` into `expectedToolCalls`, + * where the selection matcher grades it. The three kinds below fall through to + * per-turn checks and arrive here as predicate results, so before UVH-IN1 a + * case asserting "tool X was never called" filed its failure at `userValue` + * with `predicateFailed` — the chain reporting that the user did not get what + * they wanted, when what actually happened is that the model picked the wrong + * tool. In one prod audit that alone accounted for the gap between 2 selection + * failures and 12 user-value ones. + * + * `toolCalledWith` is deliberately absent: it is already matcher-graded, and + * re-reading its point-in-time predicate row here would let a raw residual + * contradict the adjudicated verdict the matcher path produces. + */ +const SELECTION_PREDICATE_REASONS: Record = { + /** A required call never happened — the same fact `missing` reports. */ + toolCalledAtLeastOnce: "missingToolCall", + /** Something else went first: a call we did not expect, in that position. */ + firstToolWas: "unexpectedToolCall", + /** A forbidden tool was called. */ + toolNeverCalled: "unexpectedToolCall", }; +/** + * Kinds that assert a call WILL happen, so they make `call` applicable. + * + * `toolNeverCalled` is deliberately excluded: a case whose only tool assertion + * is "never call X" expects no call at all, and turning `call` on for it would + * demand evidence of something the case exists to forbid. + */ +const POSITIVE_TOOL_CALL_PREDICATE_KINDS = new Set([ + "toolCalledAtLeastOnce", + "firstToolWas", +]); + +/** True when this predicate row is selection evidence rather than user value. */ +export function isSelectionPredicateKind(kind: string | undefined): boolean { + return kind !== undefined && kind in SELECTION_PREDICATE_REASONS; +} + +/** True when this predicate kind asserts that a tool call will occur. */ +export function isPositiveToolCallPredicateKind( + kind: string | undefined +): boolean { + return kind !== undefined && POSITIVE_TOOL_CALL_PREDICATE_KINDS.has(kind); +} + export type StageToolErrorLike = { kind?: string; toolName?: string; @@ -638,12 +701,40 @@ function selectionNeedsExplicitEvidence( return !prompts.some((p) => nonEmpty(p.expectedToolCalls)); } +/** Tool-selection predicate rows, in author order. */ +function selectionPredicates(e: StageEvidence): StagePredicateResultLike[] { + return (e.predicateResults ?? []).filter((r) => + isSelectionPredicateKind(r.predicate?.type) + ); +} + +/** + * The reason a set of failed selection predicates is filed under. + * + * A missing REQUIRED call outranks an unexpected one: "the tool you needed was + * never called" is the more specific and more actionable of the two, and a case + * can fail both at once (a required call absent while a forbidden one fired). + */ +function selectionPredicateReason( + failed: readonly StagePredicateResultLike[] +): StageReason { + return failed.some( + (r) => SELECTION_PREDICATE_REASONS[r.predicate?.type ?? ""] === "missingToolCall" + ) + ? "missingToolCall" + : "unexpectedToolCall"; +} + function deriveSelection( e: StageEvidence, authored: StageAuthoredCase ): StageResultRow { const prompts = e.prompts ?? []; - if (selectionNeedsExplicitEvidence(authored, prompts)) { + const predicates = selectionPredicates(e); + // Authored tool-call predicates ARE explicit evidence about selection, so a + // case carrying them is never sent down the "nothing adjudicates this" path + // below even when it authored no `expectedToolCalls`. + if (predicates.length === 0 && selectionNeedsExplicitEvidence(authored, prompts)) { // No trace at all outranks both branches below, the same way it does in // `deriveCall` and `deriveResponse`. "The run recorded no trace" and "a // sink existed and captured nothing" are different facts, and reporting @@ -670,6 +761,23 @@ function deriveSelection( promptIndexes: promptIndexes(missing), }); } + } + + // Authored tool-call predicates, after the matcher's most specific verdict + // and before its tolerated-extras logic. A failed predicate is a definite, + // author-stated fact about which tool the model picked; the `unexpected` + // branch below is the one that has to decide whether extras were tolerated. + const failedPredicates = predicates.filter((r) => r.passed === false); + if (failedPredicates.length > 0) { + return row( + "selection", + "failed", + selectionPredicateReason(failedPredicates), + boundedPredicateReasons(failedPredicates) + ); + } + + if (prompts.length > 0) { const unexpected = prompts.filter((p) => nonEmpty(p.unexpected)); if (unexpected.length > 0) { // Extras are a failure ONLY when the turn's own verdict says so. @@ -704,6 +812,13 @@ function deriveSelection( } return row("selection", "passed", "observed"); } + // Predicates that all PASSED are evidence too, not just blame: a case whose + // only selection assertion is "never call the admin tool" and which did not + // call it has measured selection and found it sound. Reporting that as + // `notMeasured` would understate what the run actually established. + if (predicates.length > 0) { + return row("selection", "passed", "observed"); + } if (e.traceAbsent) return row("selection", "notMeasured", "traceAbsent"); if (e.traceLacksSpanChannel) { return row("selection", "notMeasured", "executorEmitsNoSpans"); @@ -812,21 +927,22 @@ function deriveUserValue(e: StageEvidence): StageResultRow { if (e.evaluatorErrored) { return row("userValue", "notMeasured", "evaluatorError"); } - const results = e.predicateResults ?? []; + // Tool-call predicates are ROUTED to `selection`, not copied into it: a + // failure filed in both places would double-count one defect and, worse, + // make `firstFailedStage` depend on which stage the reader looked at first. + // What is left here is what actually speaks to the user's ask. + const results = (e.predicateResults ?? []).filter( + (r) => !isSelectionPredicateKind(r.predicate?.type) + ); if (results.length > 0) { const failed = results.filter((r) => r.passed === false); if (failed.length > 0) { - return row("userValue", "failed", "predicateFailed", { - predicateReasons: failed - .map((r) => r.reason) - .filter((r): r is string => typeof r === "string") - .slice(0, MAX_EVIDENCE_REASONS) - .map((r) => - r.length > MAX_EVIDENCE_REASON_CHARS - ? `${r.slice(0, MAX_EVIDENCE_REASON_CHARS - 1)}\u2026` - : r - ), - }); + return row( + "userValue", + "failed", + "predicateFailed", + boundedPredicateReasons(failed) + ); } return row("userValue", "passed", "observed"); } @@ -868,6 +984,28 @@ function deriveUserValue(e: StageEvidence): StageResultRow { return row("userValue", "notMeasured", "noEvidenceCaptured"); } +/** + * Predicate reasons under the row's evidence caps. + * + * Extracted so `selection` and `userValue` bound their reasons identically — + * two copies of the same slice-and-ellipsis would be free to drift, and the + * cap is what keeps a row from carrying an unbounded model-authored string. + */ +function boundedPredicateReasons( + rows: readonly StagePredicateResultLike[] +): StageEvidenceRefs | undefined { + const bounded = rows + .map((r) => r.reason) + .filter((r): r is string => typeof r === "string" && r.trim().length > 0) + .slice(0, MAX_EVIDENCE_REASONS) + .map((r) => + r.length > MAX_EVIDENCE_REASON_CHARS + ? `${r.slice(0, MAX_EVIDENCE_REASON_CHARS - 1)}…` + : r + ); + return bounded.length > 0 ? { predicateReasons: bounded } : undefined; +} + /** * Judge reasons under the SAME caps predicate reasons already obey. * diff --git a/sdk/src/eval-result-mapping.ts b/sdk/src/eval-result-mapping.ts index 1125ae3215..c86364e503 100644 --- a/sdk/src/eval-result-mapping.ts +++ b/sdk/src/eval-result-mapping.ts @@ -23,6 +23,8 @@ import { import { buildHostSnapshotMetadata } from "./host-config/internal.js"; import { deriveStageResults, + isPositiveToolCallPredicateKind, + isSelectionPredicateKind, stageDerivationToMetadata, } from "./contract/stage-derivation.js"; import { attachStageMeasurements } from "./contract/stage-measurements.js"; @@ -887,16 +889,29 @@ function deriveSdkStageResults(args: { ...(caseIdentity?.isNegativeTest !== undefined ? { isNegativeTest: caseIdentity.isNegativeTest } : {}), + // UVH-IN1's matrix, mirrored: a positive tool-call predicate expects a + // call, `toolNeverCalled` does not. This path builds its own authored + // case rather than calling `buildStageAuthoredCase` (it has no steps or + // turns to read, and its `mode` and `expectsWidgetRender` are fixed by + // what the SDK path can observe), so the matrix has to be applied twice + // — a pre-existing divergence this PR keeps in step rather than widens. expectsToolCall: (expectedToolCalls?.length ?? 0) > 0 || - caseIdentity?.isNegativeTest === true, + caseIdentity?.isNegativeTest === true || + (predicates ?? []).some((p) => + isPositiveToolCallPredicateKind(p?.type) + ), // Render observations are not carried on the SDK path, so a case is // never treated as asserting a widget render here — claiming otherwise // would demand evidence this path cannot produce and report every SDK // run's `response` as an evidence gap. + // + // Tool-call predicates are excluded for the same reason they are on the + // server: their results are routed to `selection`, so counting them here + // would leave `userValue` applicable with nothing left to grade it. assertionCount: - (predicates?.length ?? 0) + - (caseIdentity?.expectedOutput !== undefined ? 1 : 0), + (predicates ?? []).filter((p) => !isSelectionPredicateKind(p?.type)) + .length + (caseIdentity?.expectedOutput !== undefined ? 1 : 0), }, evidence: buildSdkStageEvidence(iteration, trace), iteration: { diff --git a/sdk/tests/eval-run-decision-summary.test.ts b/sdk/tests/eval-run-decision-summary.test.ts index 7872f41153..0e49825703 100644 --- a/sdk/tests/eval-run-decision-summary.test.ts +++ b/sdk/tests/eval-run-decision-summary.test.ts @@ -25,6 +25,7 @@ import { EVAL_VERDICT_DECISION_REASON_LABELS, evalRunDecisionSummarySchema, FAILURE_CATEGORY_LABELS, + STAGE_ANALYZER_VERSION, STAGE_REASON_LABELS, STAGE_STATE_LABELS, USER_VALUE_STAGE_LABELS, @@ -245,9 +246,14 @@ describe("evidence is attached to the claim it supports", () => { // Version-ahead is FLAGGED, not rejected. expect(ahead!.chain.status).toBe("verified"); if (ahead!.chain.status === "verified") { + // `known` is whatever analyzer THIS build ships, asserted by meaning + // rather than by a literal: pinning both numbers here duplicated the + // fixture comparison above and turned every analyzer bump into an edit + // in two places. `reported` stays a far-future constant so the case keeps + // exercising version-ahead as the analyzer advances. expect(ahead!.chain.analyzerVersionAhead).toEqual({ - reported: 6, - known: 5, + reported: 99, + known: STAGE_ANALYZER_VERSION, }); expect(ahead!.chain.firstFailedStage).toBe("call"); } diff --git a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json index 258df8acac..8ab14ffd43 100644 --- a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json +++ b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json @@ -1,5 +1,5 @@ { - "__readme": "Golden corpus for the canonical eval run decision summary (EVAL_RUN_DECISION_SUMMARY_SCHEMA_VERSION 1). ONE file, five consumers: the contract test (sdk/tests/eval-run-decision-summary.test.ts), the API route test (mcpjam-inspector/server/routes/v1/__tests__/eval-decision-summary.test.ts), the MCP operation test, the CLI reporter tests, and the structured-report renderer tests. That is the point: the API's answer, a client-side assembly, what the Platform MCP server hands a model, and what JSON/JUnit/HTML render are supposed to be ONE reading of a run, and the only way to prove it is to check all of them against the same rows. LOAD RULE: strip every key whose name starts with '__' before handing a row to a validator; they are fixture annotations and every object in this contract is closed, so a payload still carrying one would be rejected for the wrong reason. `expected` is GENERATED by running assembleEvalRunDecisionSummary over `input` and is asserted both ways: it must equal the assembler's output AND validate against evalRunDecisionSummarySchema. Regenerate it in the same PR as any change to the assembler, and read the diff — a change here is a change to what every surface says about a run. The embedded `verdictSummary` objects are copied verbatim from eval-verdict-policy-parity-fixtures.json, so the decisions this corpus explains are the same ones the verdict contract already pins.", + "__readme": "Golden corpus for the canonical eval run decision summary (EVAL_RUN_DECISION_SUMMARY_SCHEMA_VERSION 1). ONE file, five consumers: the contract test (sdk/tests/eval-run-decision-summary.test.ts), the API route test (mcpjam-inspector/server/routes/v1/__tests__/eval-decision-summary.test.ts), the MCP operation test, the CLI reporter tests, and the structured-report renderer tests. That is the point: the API's answer, a client-side assembly, what the Platform MCP server hands a model, and what JSON/JUnit/HTML render are supposed to be ONE reading of a run, and the only way to prove it is to check all of them against the same rows. LOAD RULE: strip every key whose name starts with '__' before handing a row to a validator; they are fixture annotations and every object in this contract is closed, so a payload still carrying one would be rejected for the wrong reason. `expected` is GENERATED by running assembleEvalRunDecisionSummary over `input` and is asserted both ways: it must equal the assembler's output AND validate against evalRunDecisionSummarySchema. Regenerate it in the same PR as any change to the assembler, and read the diff \u2014 a change here is a change to what every surface says about a run. The embedded `verdictSummary` objects are copied verbatim from eval-verdict-policy-parity-fixtures.json, so the decisions this corpus explains are the same ones the verdict contract already pins.", "cases": [ { "__name": "policyV2-passing", @@ -264,7 +264,7 @@ }, { "__name": "measured-failure-at-every-stage", - "__why": "One trial per chain stage. Each diagnostic's evidence is read from the FIRST FAILED STAGE'S ROW only — the earlier stages passed and carry spans of their own, and unioning those in would present the evidence of everything that worked as the explanation of the thing that did not.", + "__why": "One trial per chain stage. Each diagnostic's evidence is read from the FIRST FAILED STAGE'S ROW only \u2014 the earlier stages passed and carry spans of their own, and unioning those in would present the evidence of everything that worked as the explanation of the thing that did not.", "input": { "projectId": "proj-1", "run": { @@ -2183,7 +2183,7 @@ }, { "__name": "unverified-and-version-ahead", - "__why": "Two quarantine shapes. The unverified row exposes ONLY that the chain did not validate — never firstFailedStage or failureCategory, which are claims ABOUT rows that did not validate. The version-ahead row is flagged and KEPT: a derivation from a newer analyzer is still the producer's own answer, and blanking it every time the platform ships ahead of a pinned CLI would be worse.", + "__why": "Two quarantine shapes. The unverified row exposes ONLY that the chain did not validate \u2014 never firstFailedStage or failureCategory, which are claims ABOUT rows that did not validate. The version-ahead row is flagged and KEPT: a derivation from a newer analyzer is still the producer's own answer, and blanking it every time the platform ships ahead of a pinned CLI would be worse. The ahead row is stamped 99 rather than CURRENT+1 on purpose: a fixture that picks the next integer stops exercising version-ahead the moment the analyzer reaches it, which is exactly what happened when UVH-IN1 took the analyzer to 6.", "input": { "projectId": "proj-1", "run": { @@ -2304,7 +2304,7 @@ } ], "error": "arguments rejected", - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 99, "caseId": "c_ahead", "stageResults": [ { @@ -2466,10 +2466,10 @@ ], "firstFailedStage": "call", "failureCategory": "arguments", - "analyzerVersion": 6, + "analyzerVersion": 99, "analyzerVersionAhead": { - "reported": 6, - "known": 5 + "reported": 99, + "known": 6 } }, "expected": { @@ -2581,7 +2581,7 @@ }, { "__name": "mixed-repetitions-case-passes-by-threshold", - "__why": "One case, four trials, two of them failing — and the case PASSES because its pass rate meets its threshold. The counts say 1/1 case variants passed while two diagnostics are listed: diagnostics are trials beneath the aggregate and never a second tally of cases. Observed stability and mixedVerdict come from the decision, never from these rows.", + "__why": "One case, four trials, two of them failing \u2014 and the case PASSES because its pass rate meets its threshold. The counts say 1/1 case variants passed while two diagnostics are listed: diagnostics are trials beneath the aggregate and never a second tally of cases. Observed stability and mixedVerdict come from the decision, never from these rows.", "input": { "projectId": "proj-1", "run": { @@ -3232,7 +3232,7 @@ }, { "__name": "mixed-repetitions-case-fails-by-threshold", - "__why": "The same shape at a stricter threshold: one failing trial drops the case under 1.0 and the run fails. Only the decision decides that — this contract copies it.", + "__why": "The same shape at a stricter threshold: one failing trial drops the case under 1.0 and the run fails. Only the decision decides that \u2014 this contract copies it.", "input": { "projectId": "proj-1", "run": { @@ -3973,7 +3973,7 @@ }, { "__name": "inconclusive-evaluator-errors-above-ceiling", - "__why": "The grader failed too often for the run to describe the server. Same rule, different reason — and the diagnostics still list the trials, because evidence under a withheld verdict is still evidence.", + "__why": "The grader failed too often for the run to describe the server. Same rule, different reason \u2014 and the diagnostics still list the trials, because evidence under a withheld verdict is still evidence.", "input": { "projectId": "proj-1", "run": { @@ -4659,7 +4659,7 @@ }, { "__name": "legacy-cancelled-run-is-notEstablished", - "__why": "A cancelled LEGACY run: its stored counts describe the iterations it happened to record, so gating on them is fail-open. A policy-v2 run is deliberately NOT resolved this way — its validity phase is where lifecycle enters the verdict.", + "__why": "A cancelled LEGACY run: its stored counts describe the iterations it happened to record, so gating on them is fail-open. A policy-v2 run is deliberately NOT resolved this way \u2014 its validity phase is where lifecycle enters the verdict.", "input": { "projectId": "proj-1", "run": { @@ -4857,7 +4857,7 @@ }, { "__name": "partial-diagnostics-page", - "__why": "A page reached with more to come. `complete: false` plus a cursor is the difference between 'these are the failures' and 'these are some of them' — a partial page that claimed completeness would let a reader conclude a run failed in exactly two places from a sample.", + "__why": "A page reached with more to come. `complete: false` plus a cursor is the difference between 'these are the failures' and 'these are some of them' \u2014 a partial page that claimed completeness would let a reader conclude a run failed in exactly two places from a sample.", "input": { "projectId": "proj-1", "run": { diff --git a/sdk/tests/fixtures/parity/v1/MANIFEST.json b/sdk/tests/fixtures/parity/v1/MANIFEST.json index 27ccd301eb..36accbbcea 100644 --- a/sdk/tests/fixtures/parity/v1/MANIFEST.json +++ b/sdk/tests/fixtures/parity/v1/MANIFEST.json @@ -27,105 +27,105 @@ "maxIterations": 200, "maxBytes": 2097152 }, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "iterations": 19, - "bytes": 34197, - "corpusDigest": "4f17ce85965ce7936bbdec0e312603d2a0eff38e44398d6ac2d54915d5c4ddaa", + "bytes": 34215, + "corpusDigest": "c7f1250abada9fa52fb97ba4f319a462a3b991635f8449098b9384316e8eb725", "files": [ { "path": "iteration-0001.json", - "sha256": "076689979922f8552605b5100948768be772c8cef908f875159af04fad1858bc", + "sha256": "876c5de542862ab7f283b22047fca6219c1ada1e79b92c38465681dcad3fd413", "bytes": 1964 }, { "path": "iteration-0002.json", - "sha256": "2c598b7921499f5ca473822a5beec21c0d36935500847c3ca54aa2f585018aa4", - "bytes": 2251 + "sha256": "34bb23511674ac81015db145c30c4ee2d4375dde57f99a74cc701bd03d894e73", + "bytes": 2254 }, { "path": "iteration-0003.json", - "sha256": "e9fed47343bd9d727cec5044f440cd64d53a04433e84fc90356f4f645bd25976", + "sha256": "190cc8a5ab82c7722d0cb7348e1203fa653c1b33fdd41e7e4ea7ec248a4b0825", "bytes": 1798 }, { "path": "iteration-0004.json", - "sha256": "3eddea72004de2a2e34927f167475c15a20f48af5c8c08e0053ef60a28adf444", + "sha256": "41991c9fb492da1115b910a4b1883dba1510b6fb3760dc01b427e9873b81cb7d", "bytes": 1986 }, { "path": "iteration-0005.json", - "sha256": "f47a95d4d63ca5cf3d222456d3c8823137ab13ab4b511b05ef652cd96d3fdb41", - "bytes": 1930 + "sha256": "7524ebc08cb7e8351e4c0a1b412662b86d67cc9cd93ce69ab7ffa2919c6d5acc", + "bytes": 1933 }, { "path": "iteration-0006.json", - "sha256": "3bdd0e74ec55e53e80de91452bfa44cbbe95fb33766bd2926c1d0736105c3c59", - "bytes": 1507 + "sha256": "166feef51f3e4d32418db629d9cffcc56ebbbec7076ef67f13af9d9be4a56389", + "bytes": 1510 }, { "path": "iteration-0007.json", - "sha256": "f93d562f1c41c1859d2c8d7134e9275da5e5e150fac7755828be1e4cbfcd3745", + "sha256": "b68d6a1328e876848f113bb4556a8b946c59da8814e0e6097b713a10dc1f9978", "bytes": 1512 }, { "path": "iteration-0008.json", - "sha256": "9a8eb766fb4bec0696837c67b5e381eb20ce921999e545dad7361cedf5cba568", - "bytes": 1199 + "sha256": "93e4a85c9025714a523fd6054670809a3c66f92ae7cf54854a8caaa758625e19", + "bytes": 1202 }, { "path": "iteration-0009.json", - "sha256": "039da928c30b4ffaf46b72eb524a7ea80c76a07132bab69174d4c9f1fd7363ed", + "sha256": "2a6d13e4860fa43bb0a9b1c3b50a168573a5af977e2dd95f5f39ef889d131def", "bytes": 1583 }, { "path": "iteration-0010.json", - "sha256": "f5439500104be98a73b3835f13a2b609eb570bb95fa43215476abd53939d9f48", + "sha256": "4d1f7c73569af6d02060e6daee1ce03db6d05d28b599d56fc4619c8ba31248d1", "bytes": 1602 }, { "path": "iteration-0011.json", - "sha256": "57de9895b743bbed2c301e1947c9cf0a2057b2a755b7a042b4c1d34150c28bad", + "sha256": "9a2f6f8b830fecd33090db11320916bd84f3316fe556588d30a246bf71170d8a", "bytes": 2201 }, { "path": "iteration-0012.json", - "sha256": "9130a28c8c5cf5e01283104361d5dd1b21e59b946a0b70f605d101c3f986d5ef", + "sha256": "1f72db4968e1c1cd0557f5203eaef17ce0edb8581a09321dc48a576913df060c", "bytes": 1616 }, { "path": "iteration-0013.json", - "sha256": "804499955bf8cf73ded64a69ef8a7572facc33f00b55b35f5f6de70e88787ba6", + "sha256": "c3b5f3f5a8b2c686fd16ab0023f6219f5ac40c154c615e6e1e3d17e34a2ae096", "bytes": 1878 }, { "path": "iteration-0014.json", - "sha256": "a3cd6c6b37beecde6d120388c70e79184aef170dd6de5fb7d869eb8681eee023", + "sha256": "8582ee889a1ef02960f793d54a2dffa56734afa4dfe83e3bec002cde395d317a", "bytes": 1874 }, { "path": "iteration-0015.json", - "sha256": "c9500e82c6dd4ad20313d734f2acc5883fbabad1016b8abb613b939e155b9a01", + "sha256": "7f3216d6f0de9ff0e3abbc2843eba2a6c7f084db64e4b32a488077e838554497", "bytes": 1846 }, { "path": "iteration-0016.json", - "sha256": "532bbbda0222ab79ef0bca019962baf77856c2d397a3b51f5d3f5ed06d1e63f8", - "bytes": 1818 + "sha256": "0c349683c3854101a9b2cedff0aae1888387991cf408c1d8a7e1dead1c7213d5", + "bytes": 1821 }, { "path": "iteration-0017.json", - "sha256": "dc572c391881c17bc234cd45673e1ad1ac82e572af7b6529e43c9dc11f02c69f", + "sha256": "d8dc49ccb7fb8f89d87705d9804c9f67486b9a3bd67a0348bf05a0e28bb241c8", "bytes": 1971 }, { "path": "iteration-0018.json", - "sha256": "a53bbdd11cb05a01094ed76933056f6d5349ce2be32bc8003f213ac779819d1d", + "sha256": "e4ae1861a7423b70ce555c94575f7e1f3f8176a24acfffd15c258ec05c98c6ad", "bytes": 1833 }, { "path": "iteration-0019.json", - "sha256": "c43d83a5fd324319162c0fa95d074d4b445afe2404f7e8d483f1706afd4b4a92", - "bytes": 1828 + "sha256": "521652d4d98c3f547260cff6924151d0840f466b83b9b404475cd3bcdd129f3f", + "bytes": 1831 } ], "review": { diff --git a/sdk/tests/fixtures/parity/v1/iteration-0001.json b/sdk/tests/fixtures/parity/v1/iteration-0001.json index 2247aae8e7..24542b8b91 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0001.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0001.json @@ -97,6 +97,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0002.json b/sdk/tests/fixtures/parity/v1/iteration-0002.json index 9ac31091bd..d332b41c67 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0002.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0002.json @@ -1,6 +1,6 @@ { "id": "hosted-0002", - "note": "predicate failed — the judge can never reach this row", + "note": "predicate failed \u2014 the judge can never reach this row", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -108,6 +108,6 @@ ], "firstFailedStage": "userValue", "failureCategory": "userValue", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0003.json b/sdk/tests/fixtures/parity/v1/iteration-0003.json index 12f912135d..48410e67e4 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0003.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0003.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0004.json b/sdk/tests/fixtures/parity/v1/iteration-0004.json index 5eda1cd9f5..274c83fb94 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0004.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0004.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "serverData", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0005.json b/sdk/tests/fixtures/parity/v1/iteration-0005.json index a767b84391..3287322a0b 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0005.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0005.json @@ -1,6 +1,6 @@ { "id": "hosted-0005", - "note": "transport-local error — attributed to setup, never to the server", + "note": "transport-local error \u2014 attributed to setup, never to the server", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -91,6 +91,6 @@ ], "firstFailedStage": "call", "failureCategory": "setup", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0006.json b/sdk/tests/fixtures/parity/v1/iteration-0006.json index 63d813c689..277f42979e 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0006.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0006.json @@ -1,6 +1,6 @@ { "id": "hosted-0006", - "note": "connect failed — a setup abort measures nothing", + "note": "connect failed \u2014 a setup abort measures nothing", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -65,6 +65,6 @@ ], "firstFailedStage": "connection", "failureCategory": "setup", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0007.json b/sdk/tests/fixtures/parity/v1/iteration-0007.json index 1d023f19df..d860076a7f 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0007.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0007.json @@ -67,6 +67,6 @@ ], "firstFailedStage": "discovery", "failureCategory": "setup", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0008.json b/sdk/tests/fixtures/parity/v1/iteration-0008.json index 5d3a87d728..6f02493fe8 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0008.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0008.json @@ -1,6 +1,6 @@ { "id": "hosted-0008", - "note": "no trace at all — an untraced run is not judgeable", + "note": "no trace at all \u2014 an untraced run is not judgeable", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -55,6 +55,6 @@ "reason": "traceAbsent" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0009.json b/sdk/tests/fixtures/parity/v1/iteration-0009.json index d57951a035..9913726138 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0009.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0009.json @@ -75,6 +75,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0010.json b/sdk/tests/fixtures/parity/v1/iteration-0010.json index 3b5b6eee43..e379c75f12 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0010.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0010.json @@ -77,6 +77,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0011.json b/sdk/tests/fixtures/parity/v1/iteration-0011.json index 6801825e70..91ea3b1fcc 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0011.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0011.json @@ -106,6 +106,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0012.json b/sdk/tests/fixtures/parity/v1/iteration-0012.json index d7311d5dbb..e6b15e2a5f 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0012.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0012.json @@ -80,6 +80,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0013.json b/sdk/tests/fixtures/parity/v1/iteration-0013.json index f2f1d66985..7a352c0eb3 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0013.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0013.json @@ -94,6 +94,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0014.json b/sdk/tests/fixtures/parity/v1/iteration-0014.json index 6bc24a6a76..4205601f87 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0014.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0014.json @@ -91,6 +91,6 @@ ], "firstFailedStage": "response", "failureCategory": "serverData", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0015.json b/sdk/tests/fixtures/parity/v1/iteration-0015.json index 5c8718ad5b..ed0b7a94f1 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0015.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0015.json @@ -90,6 +90,6 @@ } ], "failureCategory": "evaluator", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0016.json b/sdk/tests/fixtures/parity/v1/iteration-0016.json index 7aefbbaa59..0a24b6152b 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0016.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0016.json @@ -1,6 +1,6 @@ { "id": "hosted-0016", - "note": "no authored assertions — nothing about user value was measured", + "note": "no authored assertions \u2014 nothing about user value was measured", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -88,6 +88,6 @@ "reason": "notAuthored" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0017.json b/sdk/tests/fixtures/parity/v1/iteration-0017.json index 76ef131a8c..08ea3ba05f 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0017.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0017.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "arguments", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0018.json b/sdk/tests/fixtures/parity/v1/iteration-0018.json index 053f65035a..a360771c7d 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0018.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0018.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0019.json b/sdk/tests/fixtures/parity/v1/iteration-0019.json index aee3cac7c2..73d0e3cfb8 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0019.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0019.json @@ -1,6 +1,6 @@ { "id": "hosted-0019", - "note": "assertions authored but nothing captured — the row a judge can fill", + "note": "assertions authored but nothing captured \u2014 the row a judge can fill", "origin": "synthetic", "authored": { "mode": "model_driven", @@ -88,6 +88,6 @@ "reason": "noEvidenceCaptured" } ], - "stageAnalyzerVersion": 5 + "stageAnalyzerVersion": 6 } } diff --git a/sdk/tests/fixtures/stage-analytics-golden.json b/sdk/tests/fixtures/stage-analytics-golden.json index 0bfdf6fa0a..4dabf0dfcd 100644 --- a/sdk/tests/fixtures/stage-analytics-golden.json +++ b/sdk/tests/fixtures/stage-analytics-golden.json @@ -11,7 +11,7 @@ "runCompletedAt": 1700000100000, "sourceIterationCount": 7, "sourceMaxUpdatedAt": 1700000090000, - "stageAnalyzerVersion": 5, + "stageAnalyzerVersion": 6, "measurementsSchemaVersion": 1, "materializationState": "provisional", "createdAt": 1700000150000, diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index d7d634d605..fe65667261 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -781,6 +781,164 @@ describe("userValue", () => { }); }); +// ── UVH-IN1: tool-call predicates are SELECTION evidence ───────────────────── +// +// `stepsToPromptTurns` promotes only `toolCalledWith` into `expectedToolCalls`, +// so these three kinds arrive as predicate results and used to be graded as +// user value — the chain saying the user did not get what they asked for, when +// what happened is the model picked the wrong tool. + +describe("tool-call predicates route to selection", () => { + /** One predicate row, with the discriminator the producer now preserves. */ + const pred = (type: string, passed: boolean, reason = `${type} says so`) => ({ + passed, + reason, + predicate: { type, toolName: "get_project" }, + }); + + /** No matcher evidence at all — the predicate is the only selection signal. */ + const predicateOnly = (rows: ReturnType[]) => + derive({ + evidence: { spans: [toolSpan()], predicateResults: rows }, + }); + + test.each([ + ["toolCalledAtLeastOnce", "missingToolCall"], + ["firstToolWas", "unexpectedToolCall"], + ["toolNeverCalled", "unexpectedToolCall"], + ])("%s fails at selection with %s", (kind, reason) => { + const { stageResults, firstFailedStage, failureCategory } = predicateOnly([ + pred(kind, false), + ]); + + expect(stateOf(stageResults, "selection")).toMatchObject({ + state: "failed", + reason, + evidence: { predicateReasons: [`${kind} says so`] }, + }); + expect(firstFailedStage).toBe("selection"); + // The D7 metadata judge gates on exactly this, so routing here widens its + // candidate population — intended, and named in the PR. + expect(failureCategory).toBe("selection"); + // Routed, not copied: filing it in both places would double-count one + // defect and make `firstFailedStage` depend on read order. + expect(stateOf(stageResults, "userValue").reason).not.toBe( + "predicateFailed" + ); + }); + + test.each(["toolCalledAtLeastOnce", "firstToolWas", "toolNeverCalled"])( + "%s that PASSED is selection evidence, not silence", + (kind) => { + const { stageResults } = predicateOnly([pred(kind, true)]); + expect(stateOf(stageResults, "selection")).toMatchObject({ + state: "passed", + reason: "observed", + }); + } + ); + + test("a missing required call outranks a forbidden one that fired", () => { + // Both can fail at once. "The tool you needed was never called" is the + // more specific and more actionable of the two. + const { stageResults } = predicateOnly([ + pred("toolNeverCalled", false), + pred("toolCalledAtLeastOnce", false), + ]); + expect(stateOf(stageResults, "selection").reason).toBe("missingToolCall"); + }); + + test("a row with NO discriminator is still graded as user value", () => { + // Backward compatibility, and the reason this bump changed no recorded + // row in the parity corpus: producers that never carried the predicate — + // and every row stored before UVH-IN1 — grade exactly as before. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + predicateResults: [{ passed: false, reason: "no discriminator" }], + }, + }); + expect(stateOf(stageResults, "userValue")).toMatchObject({ + state: "failed", + reason: "predicateFailed", + }); + }); + + test("toolCalledWith is deliberately left to the matcher", () => { + // It is already promoted to `expectedToolCalls` and adjudicated there. + // Re-reading its point-in-time predicate row here would let a raw residual + // contradict the verdict the matcher path produced. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [cleanTurn], + predicateResults: [pred("toolCalledWith", false)], + }, + }); + expect(stateOf(stageResults, "selection").state).toBe("passed"); + expect(stateOf(stageResults, "userValue")).toMatchObject({ + state: "failed", + reason: "predicateFailed", + }); + }); + + test("MIXED: a matcher `missing` outranks a predicate failure", () => { + // The matcher's verdict is the most specific signal about selection, and + // it is fatal in every match mode. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [ + { ...cleanTurn, missing: [{ toolName: "fetch_order" }], passed: false }, + ], + predicateResults: [pred("toolNeverCalled", false)], + }, + }); + expect(stateOf(stageResults, "selection")).toMatchObject({ + state: "failed", + reason: "missingToolCall", + evidence: { promptIndexes: [0] }, + }); + }); + + test("MIXED: a predicate failure surfaces even when every turn passed", () => { + // The conflicting case. The turn tolerated its extras, so the matcher path + // would have reported `selection: passed` and the authored assertion would + // have been filed as a user-value failure instead. + const { stageResults, firstFailedStage } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [cleanTurn], + predicateResults: [pred("firstToolWas", false)], + }, + }); + expect(stateOf(stageResults, "selection")).toMatchObject({ + state: "failed", + reason: "unexpectedToolCall", + }); + expect(firstFailedStage).toBe("selection"); + }); + + test("user-value predicates still reach userValue alongside a routed one", () => { + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [cleanTurn], + predicateResults: [ + pred("toolNeverCalled", true), + { passed: false, reason: "expected 'Refunded' on screen" }, + ], + }, + }); + expect(stateOf(stageResults, "selection").state).toBe("passed"); + expect(stateOf(stageResults, "userValue")).toMatchObject({ + state: "failed", + reason: "predicateFailed", + evidence: { predicateReasons: ["expected 'Refunded' on screen"] }, + }); + }); +}); + // ── precedence ─────────────────────────────────────────────────────────────── describe("precedence when signals conflict", () => { From 0d7e526fb2b90cf6e91e26b96be59071e1f662ec Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 10:48:58 +0000 Subject: [PATCH 04/15] UVH-IN7: an observed tool error reaches response, even unauthored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A case can author only transcript predicates — nothing about tools at all — and still have a tool fail on the server during the run. `response` was `notApplicable` for such a case, because applicability was decided purely from what the case AUTHORED. So when `failOnToolError` failed the trial on exactly that tool error, the chain had every applicable stage green and no row able to say why the verdict was red. That is the disagreement class found in prod: the legacy verdict failing on a recovered tool error the chain could not represent. It was never a verdict bug — it was a chain with no vocabulary for what happened. An observed errored tool call now makes `response` measurable on its own, filing `failed / toolError` under `serverData` like any other server-answered error. Applicability and the deriver share ONE predicate (`hasObservedToolFailure`) rather than two copies of the condition, so a stage cannot be switched on by one rule and then found empty by the other. Two boundaries are deliberate: - A span carrying an `mcpErrorCode` never reached the server's handler, so it stays a setup fact and does NOT turn the stage on. Attributing our own transport failure to the server is the mis-attribution this module exists to prevent. - A case with no tool failure is unchanged: `response` stays `notApplicable`, because there is still nothing for it to decide. This is the one evidence-driven entry in the applicability table, and it is a POSITIVE observation rather than a gap — which is what keeps the surrounding rule intact. A stage turned on by observed evidence cannot then be reported as an evidence gap, because the deriver holds the very span that turned it on. Analyzer 6 -> 7; `STAGE_REASONS` unmoved, so no mirror re-pin. Verdicts and gate exit codes unchanged (CLI 1182 tests); runner-parity snapshots differ only in the version stamp, verified by filtering the diff. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- ...ls-observed-tool-error-reaches-response.md | 18 ++++ .../__snapshots__/runner-parity.test.ts.snap | 84 ++++++++-------- sdk/src/contract/stage-derivation.ts | 77 ++++++++++++--- .../eval-run-decision-summary-fixtures.json | 2 +- sdk/tests/fixtures/parity/v1/MANIFEST.json | 42 ++++---- .../fixtures/parity/v1/iteration-0001.json | 2 +- .../fixtures/parity/v1/iteration-0002.json | 2 +- .../fixtures/parity/v1/iteration-0003.json | 2 +- .../fixtures/parity/v1/iteration-0004.json | 2 +- .../fixtures/parity/v1/iteration-0005.json | 2 +- .../fixtures/parity/v1/iteration-0006.json | 2 +- .../fixtures/parity/v1/iteration-0007.json | 2 +- .../fixtures/parity/v1/iteration-0008.json | 2 +- .../fixtures/parity/v1/iteration-0009.json | 2 +- .../fixtures/parity/v1/iteration-0010.json | 2 +- .../fixtures/parity/v1/iteration-0011.json | 2 +- .../fixtures/parity/v1/iteration-0012.json | 2 +- .../fixtures/parity/v1/iteration-0013.json | 2 +- .../fixtures/parity/v1/iteration-0014.json | 2 +- .../fixtures/parity/v1/iteration-0015.json | 2 +- .../fixtures/parity/v1/iteration-0016.json | 2 +- .../fixtures/parity/v1/iteration-0017.json | 2 +- .../fixtures/parity/v1/iteration-0018.json | 2 +- .../fixtures/parity/v1/iteration-0019.json | 2 +- .../fixtures/stage-analytics-golden.json | 2 +- sdk/tests/stage-derivation.test.ts | 99 ++++++++++++++++++- 26 files changed, 265 insertions(+), 97 deletions(-) create mode 100644 .changeset/evals-observed-tool-error-reaches-response.md diff --git a/.changeset/evals-observed-tool-error-reaches-response.md b/.changeset/evals-observed-tool-error-reaches-response.md new file mode 100644 index 0000000000..b9e4eddc17 --- /dev/null +++ b/.changeset/evals-observed-tool-error-reaches-response.md @@ -0,0 +1,18 @@ +--- +"@mcpjam/sdk": patch +"@mcpjam/inspector": patch +--- + +An observed tool error reaches `response`, even when the case authored nothing about tools + +A case can author only transcript predicates — nothing about tools at all — and still have a tool fail on the server during the run. `response` was `notApplicable` for such a case, because applicability was decided purely from what the case authored. So when `failOnToolError` failed the trial on exactly that tool error, the chain had **every applicable stage green and no row able to say why the verdict was red**. + +That is the shape of a disagreement class found in prod: the legacy verdict failing on a recovered tool error the chain could not represent. It was never a verdict bug — it was a chain that had no vocabulary for what happened. + +An observed errored tool call now makes `response` measurable on its own, filing `failed / toolError` under `serverData` like any other server-answered error. Applicability and the deriver share one predicate (`hasObservedToolFailure`) rather than two copies of the condition, so a stage cannot be switched on by one rule and then found empty by the other. + +Two boundaries are deliberate. A span carrying an `mcpErrorCode` never reached the server's handler, so it stays a setup fact and does **not** turn the stage on — attributing our own transport failure to the server is the mis-attribution this module exists to prevent. And a case with no tool failure at all is unchanged: `response` stays `notApplicable`, because there is still nothing for it to decide. + +This is the one evidence-driven entry in the applicability table, and it is a _positive observation_ rather than a gap, which is what keeps the surrounding rule intact: a stage turned on by observed evidence cannot then be reported as an evidence gap, because the deriver holds the very span that turned it on. + +Analyzer 6 → 7. `STAGE_REASONS` does not move (`toolError` already existed), so the backend mirror needs no re-pin. diff --git a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap index f7c36139d7..84d97181b0 100644 --- a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap +++ b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap @@ -181,7 +181,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -210,7 +210,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -417,7 +417,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "stepIndex": 1, }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -446,7 +446,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -747,7 +747,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -776,7 +776,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -977,7 +977,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -1006,7 +1006,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -1246,7 +1246,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -1275,7 +1275,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -1494,7 +1494,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -1523,7 +1523,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -1720,7 +1720,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -1749,7 +1749,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -1999,7 +1999,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -2028,7 +2028,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -2201,7 +2201,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo "reason": "no widget render observations recorded", }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -2230,7 +2230,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -2430,7 +2430,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -2459,7 +2459,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -2616,7 +2616,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -2645,7 +2645,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -2801,7 +2801,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -2830,7 +2830,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -3040,7 +3040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -3069,7 +3069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -3242,7 +3242,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -3271,7 +3271,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -3437,7 +3437,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -3466,7 +3466,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -3617,7 +3617,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -3646,7 +3646,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -3816,7 +3816,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi "reason": "widget rendered (1/1 observation(s))", }, ], - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -3845,7 +3845,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -4040,7 +4040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -4069,7 +4069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -4294,7 +4294,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -4323,7 +4323,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -4503,7 +4503,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 0, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -4532,7 +4532,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { @@ -4659,7 +4659,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "stageMeasurements": { "rows": [ { @@ -4688,7 +4688,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, }, "stageResults": [ { diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index 37c9542cc3..94d337a704 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -70,8 +70,14 @@ import { * move — the routing re-uses `missingToolCall` and `unexpectedToolCall` — so * the backend mirror needs no re-pin for this bump. Rows derived under 5 are * identifiable as stale and can be recomputed selectively. + * + * 7 (UVH-IN7): an OBSERVED errored tool call makes `response` measurable even + * on a case that authored nothing about tools. Before this, such a run had + * every applicable stage green while its legacy verdict failed on exactly + * that tool error — a disagreement the chain had no row to express. Re-uses + * `toolError`, so again no mirror re-pin. */ -export const STAGE_ANALYZER_VERSION = 6; +export const STAGE_ANALYZER_VERSION = 7; /** * Why a stage landed where it did. @@ -532,11 +538,36 @@ const LIFECYCLE_STOPPED: ReadonlySet = /** * Which stages this case can say anything about at all. * - * Computed BEFORE any evidence is read, so an inapplicable stage can never be - * reported as an evidence gap. + * Computed before any evidence is INTERPRETED, so an inapplicable stage can + * never be reported as an evidence gap. One entry (`response`) additionally + * reads a single positive observation — see `hasObservedToolFailure`. + */ +/** + * Did a tool call come back an ERROR the server is answerable for? + * + * Deliberately the exact condition `deriveResponse` decides `toolError` on — + * a content error, or an errored span carrying no MCP error code (a domain + * error reported the protocol-correct way). Written as one predicate used by + * both so applicability and the deriver cannot drift into a state where a + * stage is switched on by one rule and then found empty by the other. + * + * Transport-local failures are excluded by that same shared condition: a span + * with an `mcpErrorCode` never reached the server's handler, so it is a setup + * fact, not the server's answer. */ +function hasObservedToolFailure(e: StageEvidence): boolean { + const contentErrors = (e.toolErrors ?? []).some( + (t) => t.kind === "content-error" + ); + const domainFailed = (e.spans ?? []).some( + (s) => isToolSpan(s) && spanFailed(s) && typeof s.mcpErrorCode !== "number" + ); + return contentErrors || domainFailed; +} + function applicability( - authored: StageAuthoredCase + authored: StageAuthoredCase, + evidence: StageEvidence ): Record { // A case that expects no tool call but IS a negative case still exercises // `call`: proving no call happened is the assertion. @@ -562,7 +593,25 @@ function applicability( // even when it authors no expected tool call — `deriveResponse` reads the // render observations directly. Gating this on `callApplies` alone would // make `renderFailed` unreachable for a pure render probe. - response: callApplies || authored.expectsWidgetRender === true, + // + // UVH-IN7: an OBSERVED errored tool call does the same. A case can author + // nothing about tools — only predicates over the transcript — and still + // have a tool fail on the server during the run. `notApplicable` there + // says "this case has nothing for `response` to decide", which is false + // the moment a call came back an error, and it is how the chain ended up + // unable to represent a class of run whose legacy verdict failed on + // exactly that tool error: every applicable stage green, the verdict red, + // and no stage able to say why. + // + // This is the ONE evidence-driven entry in this table, and it is a + // POSITIVE observation rather than a gap — which is what keeps the rule + // above intact. A stage turned on by observed evidence cannot then be + // reported as an evidence gap: `deriveResponse` has the very span that + // turned it on. + response: + callApplies || + authored.expectsWidgetRender === true || + hasObservedToolFailure(evidence), // D8: a real ask makes `userValue` applicable even with nothing authored // to grade it. `notApplicable` would say "there was nothing to satisfy", // which is false the moment someone asked for something. @@ -719,7 +768,8 @@ function selectionPredicateReason( failed: readonly StagePredicateResultLike[] ): StageReason { return failed.some( - (r) => SELECTION_PREDICATE_REASONS[r.predicate?.type ?? ""] === "missingToolCall" + (r) => + SELECTION_PREDICATE_REASONS[r.predicate?.type ?? ""] === "missingToolCall" ) ? "missingToolCall" : "unexpectedToolCall"; @@ -734,7 +784,10 @@ function deriveSelection( // Authored tool-call predicates ARE explicit evidence about selection, so a // case carrying them is never sent down the "nothing adjudicates this" path // below even when it authored no `expectedToolCalls`. - if (predicates.length === 0 && selectionNeedsExplicitEvidence(authored, prompts)) { + if ( + predicates.length === 0 && + selectionNeedsExplicitEvidence(authored, prompts) + ) { // No trace at all outranks both branches below, the same way it does in // `deriveCall` and `deriveResponse`. "The run recorded no trace" and "a // sink existed and captured nothing" are different facts, and reporting @@ -880,15 +933,15 @@ function deriveResponse( e: StageEvidence, authored: StageAuthoredCase ): StageResultRow { - const contentErrors = (e.toolErrors ?? []).filter( - (t) => t.kind === "content-error" - ); // An errored tool span with NO code is a domain error reported the // protocol-correct way: the server answered, with unusable data. const domainFailed = (e.spans ?? []).filter( (s) => isToolSpan(s) && spanFailed(s) && typeof s.mcpErrorCode !== "number" ); - if (contentErrors.length > 0 || domainFailed.length > 0) { + // `hasObservedToolFailure` is the same condition, and it is what makes this + // stage applicable on a case that authored nothing about tools — the two + // must stay one rule, or a stage could be switched on and then found empty. + if (hasObservedToolFailure(e)) { return row("response", "failed", "toolError", { spanIds: spanIds(domainFailed).slice(0, 5), }); @@ -1095,7 +1148,7 @@ export function deriveStageResults( input: StageDerivationInput ): StageDerivation { const { authored, evidence, iteration, policy } = input; - const applies = applicability(authored); + const applies = applicability(authored, evidence); const inapplicable = (stage: UserValueStage) => row(stage, "notApplicable", "notAuthored"); diff --git a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json index 8ab14ffd43..110060d901 100644 --- a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json +++ b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json @@ -2469,7 +2469,7 @@ "analyzerVersion": 99, "analyzerVersionAhead": { "reported": 99, - "known": 6 + "known": 7 } }, "expected": { diff --git a/sdk/tests/fixtures/parity/v1/MANIFEST.json b/sdk/tests/fixtures/parity/v1/MANIFEST.json index 36accbbcea..049f146b9f 100644 --- a/sdk/tests/fixtures/parity/v1/MANIFEST.json +++ b/sdk/tests/fixtures/parity/v1/MANIFEST.json @@ -27,104 +27,104 @@ "maxIterations": 200, "maxBytes": 2097152 }, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "iterations": 19, "bytes": 34215, - "corpusDigest": "c7f1250abada9fa52fb97ba4f319a462a3b991635f8449098b9384316e8eb725", + "corpusDigest": "878f6d376c5370c13aa5c35e73ee7b9ea96b41bb4467f791892245ca7a819321", "files": [ { "path": "iteration-0001.json", - "sha256": "876c5de542862ab7f283b22047fca6219c1ada1e79b92c38465681dcad3fd413", + "sha256": "0dbc450eea310da28f20936a9237ce6680602d1e60c27158f713464dbdb536b7", "bytes": 1964 }, { "path": "iteration-0002.json", - "sha256": "34bb23511674ac81015db145c30c4ee2d4375dde57f99a74cc701bd03d894e73", + "sha256": "002a69070e2692549173a0e15203d95d948ee48a4914d583aa507e9fa08b63a4", "bytes": 2254 }, { "path": "iteration-0003.json", - "sha256": "190cc8a5ab82c7722d0cb7348e1203fa653c1b33fdd41e7e4ea7ec248a4b0825", + "sha256": "d5e3dacda886713ae9d7280de56b2b11d712dd3cdaebdf7ddf97be825b6a8b69", "bytes": 1798 }, { "path": "iteration-0004.json", - "sha256": "41991c9fb492da1115b910a4b1883dba1510b6fb3760dc01b427e9873b81cb7d", + "sha256": "a0a8f56d21e166dca2bed0b9ab2b70437fa8f775b295c6d24e72a0d0da55d7bc", "bytes": 1986 }, { "path": "iteration-0005.json", - "sha256": "7524ebc08cb7e8351e4c0a1b412662b86d67cc9cd93ce69ab7ffa2919c6d5acc", + "sha256": "355cf2c77f69aeabb3a3c204ec69e951a34cdb4dc60dbd7d9f128c5f51d8b297", "bytes": 1933 }, { "path": "iteration-0006.json", - "sha256": "166feef51f3e4d32418db629d9cffcc56ebbbec7076ef67f13af9d9be4a56389", + "sha256": "59865c00ee14c90d289961d613235c9a6b7b342ba8b82443f13e0259713cda38", "bytes": 1510 }, { "path": "iteration-0007.json", - "sha256": "b68d6a1328e876848f113bb4556a8b946c59da8814e0e6097b713a10dc1f9978", + "sha256": "85fdd897c33cf41ca5789c46f79c9491d2abe9701b2d7e00631d4ed65cf37f03", "bytes": 1512 }, { "path": "iteration-0008.json", - "sha256": "93e4a85c9025714a523fd6054670809a3c66f92ae7cf54854a8caaa758625e19", + "sha256": "936168e81f242bc01f19d5ea6218901821533cf8be7247cbe32eaab6346a0578", "bytes": 1202 }, { "path": "iteration-0009.json", - "sha256": "2a6d13e4860fa43bb0a9b1c3b50a168573a5af977e2dd95f5f39ef889d131def", + "sha256": "b34b97aa9c370e656e9bc69b81d54eff38646f0e4741b631d261722cfcad1775", "bytes": 1583 }, { "path": "iteration-0010.json", - "sha256": "4d1f7c73569af6d02060e6daee1ce03db6d05d28b599d56fc4619c8ba31248d1", + "sha256": "e77240046df9a0c992cd155a471cff741fb4772faeed1c388d99502d259f1ea6", "bytes": 1602 }, { "path": "iteration-0011.json", - "sha256": "9a2f6f8b830fecd33090db11320916bd84f3316fe556588d30a246bf71170d8a", + "sha256": "ef36ec48540c87d6100429d4641a17c219af841be25c4ce1541c20e6878eac9e", "bytes": 2201 }, { "path": "iteration-0012.json", - "sha256": "1f72db4968e1c1cd0557f5203eaef17ce0edb8581a09321dc48a576913df060c", + "sha256": "2383fbad01107d300decb40fed1211d4a7696b71272d1835b770bff654366c58", "bytes": 1616 }, { "path": "iteration-0013.json", - "sha256": "c3b5f3f5a8b2c686fd16ab0023f6219f5ac40c154c615e6e1e3d17e34a2ae096", + "sha256": "2c24f860be94f279b3acd94845898b3419533e2a95511f992bf5aa68a29860fd", "bytes": 1878 }, { "path": "iteration-0014.json", - "sha256": "8582ee889a1ef02960f793d54a2dffa56734afa4dfe83e3bec002cde395d317a", + "sha256": "fb0eb96bc62f721bfd35027f12f78fbae3cbb8798a04111c3bc1588ff2fb487c", "bytes": 1874 }, { "path": "iteration-0015.json", - "sha256": "7f3216d6f0de9ff0e3abbc2843eba2a6c7f084db64e4b32a488077e838554497", + "sha256": "12f5c4dc972aabd9bc01c8e095426064ad9c48fdcafd90d6918590ba5f64b6b6", "bytes": 1846 }, { "path": "iteration-0016.json", - "sha256": "0c349683c3854101a9b2cedff0aae1888387991cf408c1d8a7e1dead1c7213d5", + "sha256": "7d9f4f5dc8080c598fe8b2eaf93c83763550ffed7ae62562685a347a2625ae29", "bytes": 1821 }, { "path": "iteration-0017.json", - "sha256": "d8dc49ccb7fb8f89d87705d9804c9f67486b9a3bd67a0348bf05a0e28bb241c8", + "sha256": "e30bd72bd1596bc4a21f74aed893a46f9b07db67373514b4fe31f457c7d836f4", "bytes": 1971 }, { "path": "iteration-0018.json", - "sha256": "e4ae1861a7423b70ce555c94575f7e1f3f8176a24acfffd15c258ec05c98c6ad", + "sha256": "a46e1490142534f2c8484639df05dfea300ca0e2dad2cfd0d8df7a4355f38cf0", "bytes": 1833 }, { "path": "iteration-0019.json", - "sha256": "521652d4d98c3f547260cff6924151d0840f466b83b9b404475cd3bcdd129f3f", + "sha256": "5285863ba6cb805ee0e866ceb26ee811b5052dba7dfb69477ac1b5b78bc5b6a7", "bytes": 1831 } ], diff --git a/sdk/tests/fixtures/parity/v1/iteration-0001.json b/sdk/tests/fixtures/parity/v1/iteration-0001.json index 24542b8b91..02b72ee663 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0001.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0001.json @@ -97,6 +97,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0002.json b/sdk/tests/fixtures/parity/v1/iteration-0002.json index d332b41c67..671e58dd23 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0002.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0002.json @@ -108,6 +108,6 @@ ], "firstFailedStage": "userValue", "failureCategory": "userValue", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0003.json b/sdk/tests/fixtures/parity/v1/iteration-0003.json index 48410e67e4..0bd186321c 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0003.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0003.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0004.json b/sdk/tests/fixtures/parity/v1/iteration-0004.json index 274c83fb94..643b425a4d 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0004.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0004.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "serverData", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0005.json b/sdk/tests/fixtures/parity/v1/iteration-0005.json index 3287322a0b..4a0b054d2b 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0005.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0005.json @@ -91,6 +91,6 @@ ], "firstFailedStage": "call", "failureCategory": "setup", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0006.json b/sdk/tests/fixtures/parity/v1/iteration-0006.json index 277f42979e..3b76d6e3e8 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0006.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0006.json @@ -65,6 +65,6 @@ ], "firstFailedStage": "connection", "failureCategory": "setup", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0007.json b/sdk/tests/fixtures/parity/v1/iteration-0007.json index d860076a7f..12d81432c4 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0007.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0007.json @@ -67,6 +67,6 @@ ], "firstFailedStage": "discovery", "failureCategory": "setup", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0008.json b/sdk/tests/fixtures/parity/v1/iteration-0008.json index 6f02493fe8..260069371d 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0008.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0008.json @@ -55,6 +55,6 @@ "reason": "traceAbsent" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0009.json b/sdk/tests/fixtures/parity/v1/iteration-0009.json index 9913726138..d6bd8710c9 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0009.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0009.json @@ -75,6 +75,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0010.json b/sdk/tests/fixtures/parity/v1/iteration-0010.json index e379c75f12..1e8d5b7bc3 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0010.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0010.json @@ -77,6 +77,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0011.json b/sdk/tests/fixtures/parity/v1/iteration-0011.json index 91ea3b1fcc..3b6278a1c8 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0011.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0011.json @@ -106,6 +106,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0012.json b/sdk/tests/fixtures/parity/v1/iteration-0012.json index e6b15e2a5f..078ef955ea 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0012.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0012.json @@ -80,6 +80,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0013.json b/sdk/tests/fixtures/parity/v1/iteration-0013.json index 7a352c0eb3..70d227da83 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0013.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0013.json @@ -94,6 +94,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0014.json b/sdk/tests/fixtures/parity/v1/iteration-0014.json index 4205601f87..dc327d7faa 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0014.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0014.json @@ -91,6 +91,6 @@ ], "firstFailedStage": "response", "failureCategory": "serverData", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0015.json b/sdk/tests/fixtures/parity/v1/iteration-0015.json index ed0b7a94f1..d78b7171e1 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0015.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0015.json @@ -90,6 +90,6 @@ } ], "failureCategory": "evaluator", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0016.json b/sdk/tests/fixtures/parity/v1/iteration-0016.json index 0a24b6152b..b34d8605e1 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0016.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0016.json @@ -88,6 +88,6 @@ "reason": "notAuthored" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0017.json b/sdk/tests/fixtures/parity/v1/iteration-0017.json index 08ea3ba05f..3d1923830b 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0017.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0017.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "arguments", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0018.json b/sdk/tests/fixtures/parity/v1/iteration-0018.json index a360771c7d..2c48c2506d 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0018.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0018.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0019.json b/sdk/tests/fixtures/parity/v1/iteration-0019.json index 73d0e3cfb8..39b266116a 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0019.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0019.json @@ -88,6 +88,6 @@ "reason": "noEvidenceCaptured" } ], - "stageAnalyzerVersion": 6 + "stageAnalyzerVersion": 7 } } diff --git a/sdk/tests/fixtures/stage-analytics-golden.json b/sdk/tests/fixtures/stage-analytics-golden.json index 4dabf0dfcd..5c4466c109 100644 --- a/sdk/tests/fixtures/stage-analytics-golden.json +++ b/sdk/tests/fixtures/stage-analytics-golden.json @@ -11,7 +11,7 @@ "runCompletedAt": 1700000100000, "sourceIterationCount": 7, "sourceMaxUpdatedAt": 1700000090000, - "stageAnalyzerVersion": 6, + "stageAnalyzerVersion": 7, "measurementsSchemaVersion": 1, "materializationState": "provisional", "createdAt": 1700000150000, diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index fe65667261..8b68edccf7 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -781,6 +781,99 @@ describe("userValue", () => { }); }); +// ── UVH-IN7: an observed tool error makes `response` measurable ────────────── +// +// The disagreement class this closes: a case authors only transcript +// predicates, so `call` and `response` were `notApplicable`; a tool errors on +// the server during the run; `failOnToolError` fails the legacy verdict. Every +// applicable stage green, the verdict red, and no row able to say why. + +describe("an observed tool error reaches response, even unauthored", () => { + /** Authors predicates only — nothing about tools at all. */ + const predicateOnlyCase = { + mode: "model_driven" as const, + expectsToolCall: false, + assertionCount: 1, + }; + + const erroredToolSpan = () => ({ + ...toolSpan(), + id: "span-err", + status: "error", + }); + + test("a recovered tool error fails response as toolError / serverData", () => { + const { stageResults, firstFailedStage, failureCategory } = derive({ + authored: predicateOnlyCase, + evidence: { + spans: [erroredToolSpan()], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + + expect(stateOf(stageResults, "response")).toMatchObject({ + state: "failed", + reason: "toolError", + }); + expect(firstFailedStage).toBe("response"); + expect(failureCategory).toBe("serverData"); + }); + + test("the chain no longer goes all-green while the verdict fails", () => { + // The exact shape of the 8 prod trials: every stage the case authored + // passed, so nothing in the chain contradicted a red verdict. + const { stageResults } = derive({ + authored: predicateOnlyCase, + evidence: { + spans: [erroredToolSpan()], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + expect(stageResults.some((r) => r.state === "failed")).toBe(true); + }); + + test("a transport-local error does NOT turn the stage on", () => { + // A span carrying an MCP error code never reached the server's handler, + // so it is a setup fact rather than the server's answer — and turning + // `response` on for it would attribute our own failure to the server. + const { stageResults } = derive({ + authored: predicateOnlyCase, + evidence: { + spans: [{ ...erroredToolSpan(), mcpErrorCode: -32601 }], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + expect(stateOf(stageResults, "response").state).toBe("notApplicable"); + }); + + test("no errored span leaves an unauthored response inapplicable", () => { + // The floor is unchanged: a case that authors nothing about tools and saw + // no tool failure still has nothing for `response` to decide. + const { stageResults } = derive({ + authored: predicateOnlyCase, + evidence: { + spans: [toolSpan()], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + expect(stateOf(stageResults, "response").state).toBe("notApplicable"); + }); + + test("an authored case is unaffected — it was already applicable", () => { + const { stageResults } = derive({ + evidence: { + spans: [erroredToolSpan()], + prompts: [cleanTurn], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + expect(stateOf(stageResults, "response")).toMatchObject({ + state: "failed", + reason: "toolError", + }); + }); +}); + // ── UVH-IN1: tool-call predicates are SELECTION evidence ───────────────────── // // `stepsToPromptTurns` promotes only `toolCalledWith` into `expectedToolCalls`, @@ -889,7 +982,11 @@ describe("tool-call predicates route to selection", () => { evidence: { spans: [toolSpan()], prompts: [ - { ...cleanTurn, missing: [{ toolName: "fetch_order" }], passed: false }, + { + ...cleanTurn, + missing: [{ toolName: "fetch_order" }], + passed: false, + }, ], predicateResults: [pred("toolNeverCalled", false)], }, From 257230a1e42cef3c1ae4c6ffe7ea37c6d0699694 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 19:26:39 +0000 Subject: [PATCH 05/15] UVH-IN5: report "no funnel" when the probe fails, not silence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the run-binding fix. Keying the ErrorBoundary by run re-arms the probe across a run CHANGE, but it does nothing for the other half: a query that throws AFTER answering for the run still on screen renders the fallback silently, so the caller keeps the last good `true` and holds the rail open over a funnel that is no longer there. `onError` now reports `onChange(suiteRunId, false)`, which makes a failure say exactly what the dark-ship case already says — no funnel. The boundary already accepted the hook; nothing new was needed to reach it. The dark-ship test's assertion moves from "never reports" to "reports false", and a success-to-error regression for the SAME run covers the case the key cannot. Mutation-checked: removing `onError` fails those two plus the re-arm test, and nothing else. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../user-value-chain/StageFunnelPanels.tsx | 13 +++++++- .../__tests__/StageFunnelPanels.test.tsx | 32 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx b/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx index 53e8a30ff2..0eb335b40d 100644 --- a/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx +++ b/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx @@ -186,7 +186,18 @@ export function SuiteRunStageFunnelAvailability({ // for the life of the element, so an unkeyed one would swallow the probe // for every LATER run too: one transient failure would hide the chain on // every run after it until the whole view remounted. - + // + // `onError` covers the other half of that: a query that throws AFTER + // answering for the run still on screen renders the fallback silently, so + // without this the caller would keep the last good `true` and hold the + // rail open over a funnel that is no longer there. Reporting `false` from + // here makes the failure say the same thing the dark-ship case does — + // no funnel. + onChange(suiteRunId, false)} + > ); diff --git a/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx b/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx index 63454ab1dd..b0b416d283 100644 --- a/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx +++ b/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx @@ -215,7 +215,7 @@ describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () expect(onChange).toHaveBeenCalledWith("run-1", false); }); - it("renders nothing and never reports when the query throws", () => { + it("reports false and renders nothing when the query throws", () => { // The dark-ship state. Undeployed must read as "no funnel", and the probe // must not take the run-detail page down with it. queryThrows(); @@ -229,10 +229,35 @@ describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () />
, ); - expect(onChange).not.toHaveBeenCalled(); + expect(onChange).toHaveBeenCalledWith("run-1", false); expect(container.textContent).toBe("the rest of the page"); }); + it("clears a previous answer when the SAME run's probe then fails", () => { + // The boundary key re-arms the probe across runs, but a query that throws + // after answering for the run still on screen renders the fallback + // silently. Without `onError` the caller would keep the last good `true` + // and hold the rail open over a funnel that is no longer there. + convex.useQuery.mockReturnValue(SUMMARY); + const onChange = vi.fn(); + const { rerender } = render( + , + ); + expect(onChange).toHaveBeenLastCalledWith("run-1", true); + + queryThrows(); + rerender( + , + ); + expect(onChange).toHaveBeenLastCalledWith("run-1", false); + }); + it("names the run each answer is about, so a stale one is detectable", () => { // The run selector reuses one view across runs. An answer that did not // name its run could not be told apart from the previous run's, and a @@ -269,7 +294,8 @@ describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () onChange={onChange} />, ); - expect(onChange).not.toHaveBeenCalled(); + // The failing run reports "no funnel" rather than staying silent. + expect(onChange).toHaveBeenLastCalledWith("run-1", false); convex.useQuery.mockReturnValue(SUMMARY); rerender( From eaae5e4ed522e35258268aee97ef65c322c2fe1a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 19:42:17 +0000 Subject: [PATCH 06/15] UVH-IN2: attribute a model-provider failure to us, not to the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 20 trials in one audited prod window failed on "credit balance too low… Anthropic API". The step errored, the asserts were skipped, and the chain ended `userValue: notMeasured / noEvidenceCaptured` with NO failure category at all — our provider's outage presented as an unattributed server failure, in a report card whose entire job is to say whose side broke. The layer that failed is now tagged at the catch site and carried into the chain. `providerError` marks the stages a model-call failure left blank, and `categoryFor` files the run under `setup` — the existing bucket for our own side breaking, so no new category was needed. CLASSIFIED STRUCTURALLY, never by reading the message. `drive-hosted-eval-turn.ts` already knows which layer it is in: `failTurn` is the engine's stream, `mapThrownTurnError` names its own call site, and "pre-turn setup" is the one that never reached the model. A text classifier would be one provider's wording away from mis-attributing a whole class of run. The engine's `code`/`httpStatus` ride along as diagnostics and are deliberately NOT part of the decision. Three boundaries are deliberate: - Only BLANK rows are re-labelled. A provider dying at turn 4 does not un-observe turns 1-3, so a stage with its own evidence keeps its row. - Never `failed`. A run that could not be attempted has measured nothing about the server, and inflating a server failure rate with our own outage is the mis-attribution this reason exists to prevent. - A broken grader still outranks it; `evaluator` is never folded in. `providerError` is broader than its name — outage, exhausted credits, rate limit, our own spend guardrails — and what they share is that OUR side of the call broke. That is said in the reason's own docblock rather than left for a reader to infer. Analyzer 7 -> 8. This bump moves `STAGE_REASONS`; the backend mirror already carries the member (UVH-BE1 shipped it deliberately ahead), so nothing quarantines during the deploy window. Stated limitation, unchanged: the legacy verdict still counts these trials failed. Verdict population is a customer gate change and stays deferred. SDK 6891 tests, inspector server 700, CLI 1182 — gate exit codes unchanged. Parity corpus and snapshots differ only in the version stamp, verified by filtering the diff. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-provider-error-attribution.md | 24 ++ .../server/services/evals-runner.ts | 252 ++++++++++-------- .../__snapshots__/runner-parity.test.ts.snap | 84 +++--- .../services/evals/drive-hosted-eval-turn.ts | 92 +++++-- .../services/evals/finalize-iteration.ts | 86 +++--- .../server/services/evals/step-executor.ts | 52 +++- sdk/src/contract/decision-labels.ts | 2 + sdk/src/contract/index.ts | 1 + sdk/src/contract/stage-derivation.ts | 80 +++++- .../eval-run-decision-summary-fixtures.json | 2 +- sdk/tests/fixtures/parity/v1/MANIFEST.json | 42 +-- .../fixtures/parity/v1/iteration-0001.json | 2 +- .../fixtures/parity/v1/iteration-0002.json | 2 +- .../fixtures/parity/v1/iteration-0003.json | 2 +- .../fixtures/parity/v1/iteration-0004.json | 2 +- .../fixtures/parity/v1/iteration-0005.json | 2 +- .../fixtures/parity/v1/iteration-0006.json | 2 +- .../fixtures/parity/v1/iteration-0007.json | 2 +- .../fixtures/parity/v1/iteration-0008.json | 2 +- .../fixtures/parity/v1/iteration-0009.json | 2 +- .../fixtures/parity/v1/iteration-0010.json | 2 +- .../fixtures/parity/v1/iteration-0011.json | 2 +- .../fixtures/parity/v1/iteration-0012.json | 2 +- .../fixtures/parity/v1/iteration-0013.json | 2 +- .../fixtures/parity/v1/iteration-0014.json | 2 +- .../fixtures/parity/v1/iteration-0015.json | 2 +- .../fixtures/parity/v1/iteration-0016.json | 2 +- .../fixtures/parity/v1/iteration-0017.json | 2 +- .../fixtures/parity/v1/iteration-0018.json | 2 +- .../fixtures/parity/v1/iteration-0019.json | 2 +- .../fixtures/stage-analytics-golden.json | 2 +- sdk/tests/stage-derivation.test.ts | 73 +++++ 32 files changed, 554 insertions(+), 276 deletions(-) create mode 100644 .changeset/evals-provider-error-attribution.md diff --git a/.changeset/evals-provider-error-attribution.md b/.changeset/evals-provider-error-attribution.md new file mode 100644 index 0000000000..6093f9c9b0 --- /dev/null +++ b/.changeset/evals-provider-error-attribution.md @@ -0,0 +1,24 @@ +--- +"@mcpjam/sdk": patch +"@mcpjam/inspector": patch +--- + +A model-provider failure is attributed to us, not filed against the server + +20 trials in one audited prod window failed on "credit balance too low… Anthropic API". The step errored, the asserts were skipped, and the chain ended `userValue: notMeasured / noEvidenceCaptured` with **no failure category at all** — our provider's outage presented as an unattributed server failure, in a report card whose entire job is to say whose side broke. + +The layer that failed is now tagged at the catch site and carried into the chain. `providerError` marks the stages a model-call failure left blank, and `categoryFor` files the run under `setup` — the existing bucket for our own side breaking, so no new category was needed. + +**Classified structurally, never by reading the message.** `drive-hosted-eval-turn.ts` already knows which layer it is in: `failTurn` is the engine's stream, `mapThrownTurnError` names its own call site, and "pre-turn setup" is the one that never reached the model. A text classifier would be one provider's wording away from mis-attributing a whole class of run. The engine's `code` and `httpStatus` ride along as diagnostics for a reader, and are deliberately _not_ part of the decision. + +Three boundaries are deliberate: + +- **Only blank rows are re-labelled.** A provider dying at turn 4 does not un-observe turns 1–3, so a stage with its own evidence keeps its own row. +- **Never `failed`.** A run that could not be attempted has measured nothing about the server, and inflating a server failure rate with our own outage is the mis-attribution this reason exists to prevent. +- **A broken grader still outranks it.** `evaluator` is never folded into another category. + +`providerError` is broader than its name: it covers a provider outage, an exhausted credit balance, a rate limit, and our own spend guardrails. What they share is that _our_ side of the call broke, which is the only distinction the chain needs to stop blaming the server. That is stated in the reason's own docblock. + +Analyzer 7 → 8. This bump moves `STAGE_REASONS`, and the backend mirror already carries the member — it shipped deliberately ahead of this change — so nothing quarantines during the deploy window. + +**Stated limitation, unchanged:** the legacy verdict still counts these trials failed. Changing verdict population is a customer gate change and remains deferred behind its own product decision and release note. diff --git a/mcpjam-inspector/server/services/evals-runner.ts b/mcpjam-inspector/server/services/evals-runner.ts index b7c635b60f..c9222d9cec 100644 --- a/mcpjam-inspector/server/services/evals-runner.ts +++ b/mcpjam-inspector/server/services/evals-runner.ts @@ -429,7 +429,7 @@ export function runFrozenSkillOptions(run: { run.pinnedHarnessSkills === null ? "null" : typeof run.pinnedHarnessSkills - })` + })`, ); } return { @@ -496,7 +496,7 @@ export function resolveIterationSkillsSource(args: { * never wrote. Change both, or neither. */ function scoreMatchOptionsFor( - test: Pick + test: Pick, ): Record { return resolveMatchOptions(undefined, test.matchOptions) as unknown as Record< string, @@ -650,13 +650,13 @@ export type EvalIterationOutcome = { export function narrowToolsToAdvertised( allTools: PrepareChatV2Result["allTools"], progressivePlan: ProgressiveToolPlan, - discoveryState: ToolDiscoveryState + discoveryState: ToolDiscoveryState, ): PrepareChatV2Result["allTools"] { if (!progressivePlan.enabled) { return allTools; } const advertisedNames = new Set( - resolveActiveToolNames(progressivePlan, discoveryState) + resolveActiveToolNames(progressivePlan, discoveryState), ); const narrowed: PrepareChatV2Result["allTools"] = {}; for (const [name, tool] of Object.entries(allTools)) { @@ -769,12 +769,12 @@ type TraceSnapshotKind = "step_finish" | "turn_finish" | "failure"; function getServerLabelForEvalError( serverId: string, - environment: RunEvalSuiteOptions["config"]["environment"] | undefined + environment: RunEvalSuiteOptions["config"]["environment"] | undefined, ): string { const binding = environment?.serverBindings?.find( (entry) => entry.projectServerId === serverId || - entry.projectServerId?.toLowerCase() === serverId.toLowerCase() + entry.projectServerId?.toLowerCase() === serverId.toLowerCase(), ); return binding?.serverName || serverId; } @@ -819,7 +819,7 @@ function throwSetupPhaseError(args: { }): never { const serverLabel = getServerLabelForEvalError( args.serverId, - args.environment + args.environment, ); if (isMissingRuntimeServerError(args.error) || args.phase === "connection") { throw new EvalSetupPhaseError({ @@ -893,7 +893,7 @@ async function getEvalToolsForAiSdkOrThrow(args: { const tools = toolOptions ? await args.mcpClientManager.getToolsForAiSdk( [serverId], - toolOptions + toolOptions, ) : await args.mcpClientManager.getToolsForAiSdk([serverId]); const endedAt = now(); @@ -939,7 +939,7 @@ async function getEvalToolsForAiSdkOrThrow(args: { firstError.push({ error, serverId, phase: "discovery" }); return null; } - }) + }), ); if (observer) { @@ -987,7 +987,7 @@ export function resolveConfiguredServerIds(args: { const availableServerIdsSet = new Set(availableServerIds); const availableServerIdByLowercase = new Map( - availableServerIds.map((serverId) => [serverId.toLowerCase(), serverId]) + availableServerIds.map((serverId) => [serverId.toLowerCase(), serverId]), ); const projectServerIdByName = new Map(); const serverNameByProjectServerId = new Map(); @@ -1025,7 +1025,7 @@ export function resolveConfiguredServerIds(args: { : availableServerIdByLowercase.get(trimmedServerRef.toLowerCase()) ?? (() => { const projectServerId = projectServerIdByName.get( - trimmedServerRef.toLowerCase() + trimmedServerRef.toLowerCase(), ); if (projectServerId) { return ( @@ -1037,7 +1037,7 @@ export function resolveConfiguredServerIds(args: { } const serverName = serverNameByProjectServerId.get( - trimmedServerRef.toLowerCase() + trimmedServerRef.toLowerCase(), ); if (serverName) { return ( @@ -1132,11 +1132,11 @@ function resolvePinnedServerKey( pinned: PinnedToolCall, environment: RunEvalSuiteOptions["config"]["environment"] | undefined, selectedServers: string[], - mcpClientManager: MCPClientManager + mcpClientManager: MCPClientManager, ): string | undefined { const connected = new Set(selectedServers); const candidates = [pinned.serverId, pinned.serverName].filter( - (ref): ref is string => !!ref + (ref): ref is string => !!ref, ); for (const candidate of candidates) { const [resolved] = resolveConfiguredServerIds({ @@ -1155,12 +1155,12 @@ function resolvePinnedServerKey( function buildPromptTraceSummaries( evaluation: MultiTurnEvaluationResult, - turnCheckResults: PredicateResult[] = [] + turnCheckResults: PredicateResult[] = [], ): PromptTraceSummary[] { return evaluation.promptSummaries.map((summary) => { const perTurn = turnCheckResults.filter( (r) => - r.scope?.kind === "turn" && r.scope.promptIndex === summary.promptIndex + r.scope?.kind === "turn" && r.scope.promptIndex === summary.promptIndex, ); return { promptIndex: summary.promptIndex, @@ -1190,7 +1190,7 @@ function buildPromptTraceSummaries( mismatchedArguments: Array.from(mismatchedArguments).filter( (key) => JSON.stringify(mismatch.expectedArgs?.[key]) !== - JSON.stringify(mismatch.actualArgs?.[key]) + JSON.stringify(mismatch.actualArgs?.[key]), ), }; }), @@ -1233,7 +1233,7 @@ function extractToolCallsFromConversation(params: { (toolCall) => toolCall.toolName === name && JSON.stringify(toolCall.arguments) === - JSON.stringify(argumentsValue) + JSON.stringify(argumentsValue), ); if (!alreadyAdded) { toolsCalled.push({ @@ -1258,7 +1258,7 @@ function extractToolCallsFromConversation(params: { (toolCall) => toolCall.toolName === toolName && JSON.stringify(toolCall.arguments) === - JSON.stringify(argumentsValue) + JSON.stringify(argumentsValue), ); if (!alreadyAdded) { toolsCalled.push({ @@ -1282,12 +1282,12 @@ function extractToolCallsExcludingPolicyBlocks( steps?: ReadonlyArray; messages: ModelMessage[]; }, - blockedToolCallIds: ReadonlySet + blockedToolCallIds: ReadonlySet, ): ToolCall[] { return extractToolCallsFromConversation(params).filter( (toolCall) => toolCall.toolCallId === undefined || - !blockedToolCallIds.has(toolCall.toolCallId) + !blockedToolCallIds.has(toolCall.toolCallId), ); } @@ -1297,7 +1297,7 @@ function toolCallIdentity(toolCall: ToolCall): string { function mergeToolCalls( existingToolCalls: ToolCall[], - incomingToolCalls: ToolCall[] + incomingToolCalls: ToolCall[], ): ToolCall[] { const seen = new Set(existingToolCalls.map(toolCallIdentity)); const merged = [...existingToolCalls]; @@ -1331,14 +1331,14 @@ function appendPartialToolCallsToPrompt(params: { } const existingToolCalls = Array.isArray( - params.toolsCalledByPrompt[params.promptIndex] + params.toolsCalledByPrompt[params.promptIndex], ) ? params.toolsCalledByPrompt[params.promptIndex]! : []; params.toolsCalledByPrompt[params.promptIndex] = mergeToolCalls( existingToolCalls, - partialToolCalls + partialToolCalls, ); } @@ -1377,7 +1377,7 @@ function buildTraceSnapshotEvent(params: { snapshotKind: params.snapshotKind, trace: sanitizeForConvexTransport(trace), actualToolCalls: sanitizeForConvexTransport( - toStreamToolCalls(params.actualToolCalls) + toStreamToolCalls(params.actualToolCalls), ), usage: { inputTokens: params.usage.inputTokens ?? 0, @@ -1397,7 +1397,7 @@ function buildTraceSnapshotEvent(params: { * (already steps-shaped) — and prefers an existing `steps` array if present. */ function snapshotWithStepsForConvex( - snapshot: Record + snapshot: Record, ): Record { if ( !snapshot || @@ -1441,7 +1441,7 @@ async function createIterationDirectly( }; iterationNumber: number; startedAt: number; - } + }, ): Promise { try { const result = await convexClient.mutation( @@ -1449,11 +1449,11 @@ async function createIterationDirectly( { testCaseId: params.testCaseId, testCaseSnapshot: sanitizeForConvexTransport( - snapshotWithStepsForConvex(params.testCaseSnapshot) + snapshotWithStepsForConvex(params.testCaseSnapshot), ), iterationNumber: params.iterationNumber, startedAt: params.startedAt, - } + }, ); return result?.iterationId as string | undefined; @@ -1555,10 +1555,10 @@ async function persistRunSetupFailure(args: { try { const details = (await args.convexClient.query( "testSuites:getTestSuiteRunDetails" as any, - { runId: args.runId } + { runId: args.runId }, )) as { iterations?: Array> } | null; return (details?.iterations ?? []).filter( - (row) => row.status === "pending" + (row) => row.status === "pending", ); } catch (readError) { logger.warn("[evals] Failed to read pending setup iterations", { @@ -1581,7 +1581,7 @@ async function persistRunSetupFailure(args: { : undefined; const test = args.tests.find( (candidate) => - candidate.testCaseId && candidate.testCaseId === row.testCaseId + candidate.testCaseId && candidate.testCaseId === row.testCaseId, ); const snapshot = row.testCaseSnapshot as | { query?: string; expectedToolCalls?: unknown[] } @@ -1616,7 +1616,7 @@ async function persistRunSetupFailure(args: { recorder: args.recorder, convexClient: args.convexClient, }); - }) + }), ); }; @@ -1633,7 +1633,7 @@ async function persistRunSetupFailure(args: { try { await args.convexClient.mutation( "testSuites:markSetupPendingIterationsFailed" as any, - { runId: args.runId, error: args.errorMessage } + { runId: args.runId, error: args.errorMessage }, ); } catch (cleanupError) { logger.warn("[evals] Failed to mark residual setup iterations failed", { @@ -1823,7 +1823,7 @@ const buildModelDefinition = (test: EvalTestCase): ModelDefinition => { function lookupProviderApiKey( modelApiKeys: Record | undefined, - provider: string + provider: string, ): string | undefined { return modelApiKeys?.[provider] ?? modelApiKeys?.[provider.toLowerCase()]; } @@ -1853,7 +1853,7 @@ function resolveEvalModelRuntime(args: { const provider = args.modelDefinition.provider; if (!apiKey && provider !== "ollama" && provider !== "custom") { throw new Error( - `Missing API key for provider ${args.test.provider} (test: ${args.test.title})` + `Missing API key for provider ${args.test.provider} (test: ${args.test.title})`, ); } @@ -1869,14 +1869,14 @@ function resolveEvalModelRuntime(args: { } function hasExplicitModelApiKeys( - modelApiKeys: Record | undefined + modelApiKeys: Record | undefined, ): boolean { return Boolean(modelApiKeys && Object.keys(modelApiKeys).length > 0); } function resolveOrgTargetForEval( test: EvalTestCase, - explicitTarget?: ResolveOrgModelConfigTarget + explicitTarget?: ResolveOrgModelConfigTarget, ): ResolveOrgModelConfigTarget | undefined { if (explicitTarget) return explicitTarget; const maybeProjectId = (test as { projectId?: unknown }).projectId; @@ -1926,7 +1926,7 @@ async function resolveOrgByokEvalRuntime(args: { target, providerKey, String(args.modelDefinition.id), - { bearerToken: args.convexAuthToken } + { bearerToken: args.convexAuthToken }, ); if (runtime.runtimeLocation === "cloud") { return { kind: "cloud", providerKey: runtime.providerKey, target }; @@ -1941,15 +1941,15 @@ async function resolveOrgByokEvalRuntime(args: { // PR6: single hosted wrapper for both modes (emit optional). Owns the browser // harness lifecycle (try/finally guarantees Chromium teardown on every exit). const runHostedIteration = async ( - params: RunIterationBackendParams & { emit?: StreamEmit } + params: RunIterationBackendParams & { emit?: StreamEmit }, ): Promise => { // First pinned turn's per-call render-budget override (mirrors the local // runner's harness creation). const pinnedRenderTimeoutMs = resolveEvalTestCase( - params.test + params.test, ).promptTurns.find( (t) => - isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number" + isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number", )?.pinnedToolCall?.renderTimeoutMs; const browser = await createBrowserSessionContext({ model: params.test.model, @@ -1997,7 +1997,7 @@ async function findIterationIdForTimeout(args: { try { const response = await args.convexClient.query( "testSuites:getTestSuiteRunDetails" as any, - { runId: args.runId } + { runId: args.runId }, ); const iterations = response?.iterations ?? []; const matching = iterations.find((iteration: any) => { @@ -2207,7 +2207,7 @@ const executeTestCase = async (params: { runner: () => Promise, precreatedIterationId: string | undefined, runIndex: number, - timeoutTest: EvalTestCase = normalizedTest + timeoutTest: EvalTestCase = normalizedTest, ): Promise => { if (abortSignal?.aborted) { const reason = abortSignal.reason; @@ -2292,8 +2292,8 @@ const executeTestCase = async (params: { }), undefined, runIndex, - normalizedTest - ) + normalizedTest, + ), ); } return outcomes; @@ -2302,11 +2302,11 @@ const executeTestCase = async (params: { const modelDefinition = buildModelDefinition(test); const resolvedModelId = getCanonicalModelId( String(modelDefinition.id), - modelDefinition.provider + modelDefinition.provider, ); const isJamModel = isHostedCatalogModel( resolvedModelId, - modelDefinition.provider + modelDefinition.provider, ); const orgByokRuntime = isJamModel ? undefined @@ -2366,7 +2366,7 @@ const executeTestCase = async (params: { { runIndex, error: error instanceof Error ? error.message : String(error), - } + }, ); precreatedIterationIds.push(undefined); } @@ -2429,7 +2429,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test + test, ); outcomes.push(iterationOutcome); continue; @@ -2491,7 +2491,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test + test, ); outcomes.push(iterationOutcome); continue; @@ -2545,7 +2545,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test + test, ); outcomes.push(iterationOutcome); } @@ -2556,7 +2556,7 @@ const executeTestCase = async (params: { // Thin batch wrapper (no `emit`) — preserves the call site in // `runEvalSuiteWithAiSdk` and tests with zero churn. const runTestCase = ( - params: Omit[0], "emit"> + params: Omit[0], "emit">, ) => executeTestCase(params); export const runEvalSuiteWithAiSdk = async ({ @@ -2602,7 +2602,7 @@ export const runEvalSuiteWithAiSdk = async ({ ) { logger.warn( "[evals] readOnly tool policy does not restrict the sandbox bash tool", - { suiteId } + { suiteId }, ); } @@ -2640,7 +2640,7 @@ export const runEvalSuiteWithAiSdk = async ({ const evalTasksSeam = resolveToolTaskSeam({ tasksPolicy: readTasksPolicy( - (suiteHostConfig ?? undefined) as Parameters[0] + (suiteHostConfig ?? undefined) as Parameters[0], ), surface: "eval", // Driver `timeoutMs` stays at its default — the task drive nests under @@ -2678,19 +2678,19 @@ export const runEvalSuiteWithAiSdk = async ({ const toolAnnotations: ToolAnnotationsLookup = new Map(); if (toolPolicy) { const uncachedServerIds = serverIds.filter( - (serverId) => !mcpClientManager.hasCachedToolAnnotations(serverId) + (serverId) => !mcpClientManager.hasCachedToolAnnotations(serverId), ); if (uncachedServerIds.length > 0) { throw new WebRouteError( 400, ErrorCode.VALIDATION_ERROR, `TOOL_POLICY_ANNOTATIONS_UNAVAILABLE: tool policy requires a populated annotation cache for every selected server; missing ${uncachedServerIds.join( - ", " + ", ", )}.`, { reason: "TOOL_POLICY_ANNOTATIONS_UNAVAILABLE", serverIds: uncachedServerIds, - } + }, ); } try { @@ -2711,18 +2711,18 @@ export const runEvalSuiteWithAiSdk = async ({ 400, ErrorCode.VALIDATION_ERROR, error.message, - { reason: "TOOL_POLICY_INVALID" } + { reason: "TOOL_POLICY_INVALID" }, ); } throw error; } for (const serverId of serverIds) { for (const [toolName, annotations] of Object.entries( - mcpClientManager.getAllToolAnnotations(serverId) + mcpClientManager.getAllToolAnnotations(serverId), )) { toolAnnotations.set( toolAnnotationsKey(serverId, toolName), - annotations + annotations, ); } } @@ -2735,7 +2735,7 @@ export const runEvalSuiteWithAiSdk = async ({ ? applyVisibilityPolicyAndCountSignals( tools as Record, mcpClientManager, - hostExecutionPolicy + hostExecutionPolicy, ) : undefined; const resolvedSetupSignals = setupObserver.buildSignals(); @@ -2752,7 +2752,7 @@ export const runEvalSuiteWithAiSdk = async ({ "testSuites:getTestSuiteRun" as any, { runId, - } + }, ); if (currentRun?.status === "cancelled") { @@ -2775,7 +2775,7 @@ export const runEvalSuiteWithAiSdk = async ({ // exhaust the worker. LLM-only cases are network-bound and stay unbounded. // The limiter releases a slot when each case settles, so it can't leak. const renderCheckLimit = createConcurrencyLimiter( - MAX_CONCURRENT_RENDER_CHECKS + MAX_CONCURRENT_RENDER_CHECKS, ); const runOne = (test: (typeof tests)[number]) => runTestCase({ @@ -2837,7 +2837,7 @@ export const runEvalSuiteWithAiSdk = async ({ promptTurns: resolveEvalTestCase(test).promptTurns, }) ? renderCheckLimit(() => runOne(test)) - : runOne(test) + : runOne(test), ); // Poll the run status: user cancellation, or a `timed_out` status set @@ -2851,7 +2851,7 @@ export const runEvalSuiteWithAiSdk = async ({ try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId } + { runId }, ); if (currentRun?.status === "cancelled") { abortRun(RUN_CANCELLED_ERROR); @@ -2886,7 +2886,7 @@ export const runEvalSuiteWithAiSdk = async ({ try { await convexClient.mutation( "testSuites:heartbeatTestSuiteRun" as any, - { runId } + { runId }, ); } catch (error) { logger.warn("[evals] Failed to heartbeat eval run", { @@ -2920,8 +2920,8 @@ export const runEvalSuiteWithAiSdk = async ({ throw error; } return never(); - }) - ) + }), + ), ); const allTestsSettled = Promise.allSettled(testPromises); @@ -2989,7 +2989,7 @@ export const runEvalSuiteWithAiSdk = async ({ } } summary.policyBlockedIterations += outcomes.filter( - (outcome) => (outcome.policyBlockCount ?? 0) > 0 + (outcome) => (outcome.policyBlockCount ?? 0) > 0, ).length; if (runId === null) { quickRunOutcomes.push(...outcomes); @@ -3101,7 +3101,7 @@ async function seedAndAnnotateEvalAttachments(args: { }); if (!seeded.note) return; const firstModelTurnIndex = args.promptTurns.findIndex( - (t) => !isPinnedTurn(t) + (t) => !isPinnedTurn(t), ); if (firstModelTurnIndex < 0) return; args.promptTurns[firstModelTurnIndex] = { @@ -3170,7 +3170,7 @@ const runLocalIteration = async ({ try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId } + { runId }, ); if (currentRun?.status === "cancelled") { return { @@ -3182,7 +3182,7 @@ const runLocalIteration = async ({ resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -3205,7 +3205,7 @@ const runLocalIteration = async ({ resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -3259,7 +3259,7 @@ const runLocalIteration = async ({ }); const system = withHostContextSystemPrompt( resolvedExecution.systemPrompt, - test.hostConfigOverride?.hostContext as Record | undefined + test.hostConfigOverride?.hostContext as Record | undefined, ); const temperature = resolvedExecution.temperature; const toolChoice = normalizeToolChoice(advancedConfig?.toolChoice); @@ -3276,7 +3276,7 @@ const runLocalIteration = async ({ // First pinned turn's render-budget override; applied to the shared harness. const pinnedRenderTimeoutMs = promptTurns.find( (t) => - isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number" + isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number", )?.pinnedToolCall?.renderTimeoutMs; const modelRuntime = caseNeedsModel @@ -3445,7 +3445,7 @@ const runLocalIteration = async ({ if (caseNeedsModel) { resolveHostTools( { builtInToolIds: resolvedExecution.builtInToolIds }, - null + null, ); prepared = await prepareChatV2({ mcpClientManager, @@ -3494,7 +3494,7 @@ const runLocalIteration = async ({ modelDefinition, modelRuntime!.apiKey, modelRuntime!.baseUrls, - modelRuntime!.customProviders + modelRuntime!.customProviders, ); // Reproducible evals: boot a fresh ephemeral sandbox from the suite's @@ -3519,7 +3519,7 @@ const runLocalIteration = async ({ // paid box only the backend TTL GC could reap. Fail loudly instead. if (!isComputersDataPlaneConfigured()) { throw new Error( - "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." + "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it.", ); } evalSandbox = await provisionEvalSandbox({ @@ -3530,7 +3530,7 @@ const runLocalIteration = async ({ }); if (!evalSandbox.ok) { throw new Error( - `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}` + `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}`, ); } // COMP-17: seed the case's pinned attachments into the fresh box before @@ -3560,7 +3560,7 @@ const runLocalIteration = async ({ !Object.hasOwn(browser.computerWidgetTools, toolChoice.toolName) ) { throw new Error( - `Configured tool choice '${toolChoice.toolName}' is not available for this eval run.` + `Configured tool choice '${toolChoice.toolName}' is not available for this eval run.`, ); } } @@ -3577,7 +3577,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -3613,7 +3613,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }) + }), ); }, onTurnFailure: ({ @@ -3632,7 +3632,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }) + }), ); emit({ type: "step_status", @@ -3652,7 +3652,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }) + }), ); emit({ type: "turn_finish", turnIndex }); emit({ @@ -3665,7 +3665,7 @@ const runLocalIteration = async ({ onPinnedTurn: (ctx) => emitPinnedTurnSse( { emit, withSystemPrefix, buildTraceSnapshotEvent }, - { turnIndex, ...ctx } + { turnIndex, ...ctx }, ), }) : undefined; @@ -3693,7 +3693,7 @@ const runLocalIteration = async ({ pinned, environment, selectedServers, - mcpClientManager + mcpClientManager, ), prepared, llmModel, @@ -3710,7 +3710,7 @@ const runLocalIteration = async ({ extractToolCalls: (params) => extractToolCallsExcludingPolicyBlocks( params, - toolPolicyGate?.blockedToolCallIds() ?? new Set() + toolPolicyGate?.blockedToolCallIds() ?? new Set(), ), // Per-turn streaming play-by-play (headless in batch). buildSinks: makeSinks, @@ -3754,13 +3754,13 @@ const runLocalIteration = async ({ // accounting (a click that fires a tool the case forbade SHOULD fail it). const toolsCalledByPromptWithWidgets = mergeToolCallsByPromptIndex( acc.toolsCalledByPrompt, - widgetToolCallsByPromptIndex(browser.browserInteractionSteps) + widgetToolCallsByPromptIndex(browser.browserInteractionSteps), ); // Per-turn predicate results from step assert execution facts (not a // re-evaluation of promptTurns.checks — avoids duplicates vs executeSteps). const turnCheckResults = resolveTurnCheckResultsFromStepExecution( stepState, - steps + steps, ); const failOnToolError = (advancedConfig as { failOnToolError?: boolean } | undefined) @@ -3808,7 +3808,7 @@ const runLocalIteration = async ({ ? acc.accumulatedUsage : undefined, renderObservations: summarizeRenderObservations( - browser.widgetRenderObservations + browser.widgetRenderObservations, ), toolErrors: acc.pinnedToolErrors, iterationError: acc.iterationError, @@ -3822,7 +3822,7 @@ const runLocalIteration = async ({ }); const promptTraceSummaries = buildPromptTraceSummaries( evaluation, - turnCheckResults + turnCheckResults, ); // Reflect the gated verdict (match AND tool-error gate AND predicates) in // the returned evaluation so totals built from `evaluation.passed` agree @@ -3929,13 +3929,13 @@ const runLocalIteration = async ({ ? narrowToolsToAdvertised( selectionToolsForFinish, selectionDiscoveryForFinish.progressivePlan, - selectionDiscoveryForFinish.discoveryState + selectionDiscoveryForFinish.discoveryState, ) : selectionToolsForFinish, } : {}), }); - // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. + // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. // // At `enforce` the iteration's result is the conjunction of the boolean // pipeline and the gating score rows, computed inside @@ -3977,7 +3977,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -4032,7 +4032,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions + test.matchOptions, ); // Suite summary aggregates `evaluation.passed` (see runEvalSuiteWithAiSdk). // The persisted iteration is hard-coded `passed: false` below, but the @@ -4069,7 +4069,7 @@ const runLocalIteration = async ({ totalTokens: acc.accumulatedUsage.totalTokens, }, prompts: promptTraceSummaries, - }) + }), ); emit({ type: "error", @@ -4152,7 +4152,7 @@ const runLocalIteration = async ({ ? narrowToolsToAdvertised( selectionToolsForFinish, selectionDiscoveryForFinish.progressivePlan, - selectionDiscoveryForFinish.discoveryState + selectionDiscoveryForFinish.discoveryState, ) : selectionToolsForFinish, } @@ -4242,7 +4242,7 @@ const runHostedIterationWithBrowser = async ( }: RunIterationBackendParams & { emit?: StreamEmit; }, - browser: BrowserSessionContext + browser: BrowserSessionContext, ): Promise => { const resolvedTest = resolveEvalTestCase(test); const toolPolicyGate = resolveEnforcementGate({ @@ -4261,7 +4261,7 @@ const runHostedIterationWithBrowser = async ( try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId } + { runId }, ); if (currentRun?.status === "cancelled") { return { @@ -4273,7 +4273,7 @@ const runHostedIterationWithBrowser = async ( resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -4296,7 +4296,7 @@ const runHostedIterationWithBrowser = async ( resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions + test.matchOptions, ), passed: false, }, @@ -4337,7 +4337,7 @@ const runHostedIterationWithBrowser = async ( }); const systemPrompt = withHostContextSystemPrompt( resolvedExecution.systemPrompt, - test.hostConfigOverride?.hostContext as Record | undefined + test.hostConfigOverride?.hostContext as Record | undefined, ); const temperature = resolvedExecution.temperature; const toolChoice = normalizeToolChoice(advancedConfig?.toolChoice); @@ -4429,7 +4429,7 @@ const runHostedIterationWithBrowser = async ( { builtInToolIds: resolvedExecution.builtInToolIds }, builtInTarget && "projectId" in builtInTarget ? { authHeader: convexAuthToken, projectId: builtInTarget.projectId } - : null + : null, ); // ── Harness execution inputs, resolved once per iteration. // @@ -4541,7 +4541,7 @@ const runHostedIterationWithBrowser = async ( throw new Error( pinnedEnvironmentId ? "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." - : "This eval runs on a harness, which boots a disposable computer per iteration, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." + : "This eval runs on a harness, which boots a disposable computer per iteration, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it.", ); } evalSandbox = await provisionEvalSandbox({ @@ -4552,7 +4552,7 @@ const runHostedIterationWithBrowser = async ( }); if (!evalSandbox.ok) { throw new Error( - `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}` + `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}`, ); } // COMP-17: seed the case's pinned attachments before exposing `bash` @@ -4613,7 +4613,7 @@ const runHostedIterationWithBrowser = async ( promptTurns, [], test.isNegativeTest, - test.matchOptions + test.matchOptions, ); failedEvaluation.passed = false; return { @@ -4654,7 +4654,7 @@ const runHostedIterationWithBrowser = async ( promptTurns, toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions + test.matchOptions, ), iterationId: undefined, }); @@ -4667,6 +4667,10 @@ const runHostedIterationWithBrowser = async ( let iterationError: string | undefined = undefined; let iterationErrorDetails: string | undefined = undefined; + /** Which layer raised `iterationError`, when the executor classified it. */ + let iterationStepError: + | { source?: "model" | "setup"; code?: string; httpStatus?: number } + | undefined = undefined; const capturedSpans: EvalTraceSpan[] = []; // PR 4d review fix (Codex P2 / Cursor Medium): see hoist above the // `prepareChatV2` try. @@ -4681,7 +4685,7 @@ const runHostedIterationWithBrowser = async ( extractToolCalls: (messages) => extractToolCallsExcludingPolicyBlocks( { messages }, - toolPolicyGate?.blockedToolCallIds() ?? new Set() + toolPolicyGate?.blockedToolCallIds() ?? new Set(), ), buildTraceSnapshotEvent, }) @@ -4800,7 +4804,7 @@ const runHostedIterationWithBrowser = async ( extractToolCalls: (messages) => extractToolCallsExcludingPolicyBlocks( { messages }, - toolPolicyGate?.blockedToolCallIds() ?? new Set() + toolPolicyGate?.blockedToolCallIds() ?? new Set(), ), acc: { messageHistory, @@ -4815,7 +4819,7 @@ const runHostedIterationWithBrowser = async ( pinned, environment, selectedServers, - mcpClientManager + mcpClientManager, ), pinnedToolErrors, ...(emit @@ -4823,7 +4827,7 @@ const runHostedIterationWithBrowser = async ( emitPinnedTurn: (payload: PinnedTurnSsePayload) => emitPinnedTurnSse( { emit, withSystemPrefix, buildTraceSnapshotEvent }, - payload + payload, ), } : {}), @@ -4860,6 +4864,15 @@ const runHostedIterationWithBrowser = async ( if (result.iterationError) { iterationError = result.iterationError; iterationErrorDetails = result.iterationErrorDetails; + if (result.errorSource) { + iterationStepError = { + source: result.errorSource, + ...(result.errorCode ? { code: result.errorCode } : {}), + ...(typeof result.errorHttpStatus === "number" + ? { httpStatus: result.errorHttpStatus } + : {}), + }; + } } // Pinned setup failure (server not connected) — drives `status:"setup_failed"` // below, mirroring the local runner. @@ -4873,7 +4886,7 @@ const runHostedIterationWithBrowser = async ( // calls whether authored as an expected tool call or a predicate. const toolsCalledByPromptWithWidgets = mergeToolCallsByPromptIndex( toolsCalledByPrompt, - widgetToolCallsByPromptIndex(browser.browserInteractionSteps) + widgetToolCallsByPromptIndex(browser.browserInteractionSteps), ); const failOnToolError = (advancedConfig as { failOnToolError?: boolean } | undefined) @@ -4891,7 +4904,7 @@ const runHostedIterationWithBrowser = async ( // Per-turn predicate results from step assert execution (hosted parity). const turnCheckResults = resolveTurnCheckResultsFromStepExecution( stepState, - steps + steps, ); const effectivePredicates = test.successPredicates?.length ? test.successPredicates @@ -4914,7 +4927,7 @@ const runHostedIterationWithBrowser = async ( trace: traceForGate, usage: hasReportedUsage(accumulatedUsage) ? accumulatedUsage : undefined, renderObservations: summarizeRenderObservations( - browser.widgetRenderObservations + browser.widgetRenderObservations, ), toolErrors: pinnedToolErrors, iterationError, @@ -4928,7 +4941,7 @@ const runHostedIterationWithBrowser = async ( }); const promptTraceSummaries = buildPromptTraceSummaries( evaluation, - turnCheckResults + turnCheckResults, ); // Reflect the gated verdict (match AND tool-error gate AND predicates) in the // returned evaluation so totals built from `evaluation.passed` agree with the @@ -4965,6 +4978,9 @@ const runHostedIterationWithBrowser = async ( : {}), spans: capturedSpans, prompts: promptTraceSummaries, + // UVH-IN2: the layer that raised the fatal error, so the chain can say a + // provider outage was ours rather than filing it against the server. + ...(iterationStepError ? { stepError: iterationStepError } : {}), ...(widgetSnapshots ? { widgetSnapshots } : {}), // Browser-rendered MCP App eval (PR 14): hosted-path browser artifacts // (see the non-stream backend runner). @@ -5013,7 +5029,7 @@ const runHostedIterationWithBrowser = async ( selectionTools: narrowToolsToAdvertised( prepared.allTools, prepared.progressivePlan, - prepared.discoveryState + prepared.discoveryState, ), }); // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. @@ -5054,5 +5070,5 @@ const runHostedIterationWithBrowser = async ( export const streamTestCase = ( params: Omit[0], "emit"> & { emit: StreamEmit; - } + }, ) => executeTestCase(params); diff --git a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap index 84d97181b0..cc43dd7138 100644 --- a/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap +++ b/mcpjam-inspector/server/services/evals/__tests__/__snapshots__/runner-parity.test.ts.snap @@ -181,7 +181,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -210,7 +210,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -417,7 +417,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h "stepIndex": 1, }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -446,7 +446,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -747,7 +747,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -776,7 +776,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -977,7 +977,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -1006,7 +1006,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-batch p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -1246,7 +1246,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -1275,7 +1275,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -1494,7 +1494,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -1523,7 +1523,7 @@ exports[`runner parity (golden Convex payload + event sequence) > hosted-stream }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -1720,7 +1720,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -1749,7 +1749,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ca }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -1999,7 +1999,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -2028,7 +2028,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch in }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -2201,7 +2201,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo "reason": "no widget render observations recorded", }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -2230,7 +2230,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mo }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -2430,7 +2430,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -2459,7 +2459,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch mu }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -2616,7 +2616,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -2645,7 +2645,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch ne }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -2801,7 +2801,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -2830,7 +2830,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch pr }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -3040,7 +3040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -3069,7 +3069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -3242,7 +3242,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -3271,7 +3271,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -3437,7 +3437,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -3466,7 +3466,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch to }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -3617,7 +3617,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -3646,7 +3646,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch un }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -3816,7 +3816,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi "reason": "widget rendered (1/1 observation(s))", }, ], - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -3845,7 +3845,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-batch wi }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -4040,7 +4040,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h "missingCount": 0, "multiTurn": true, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -4069,7 +4069,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream h }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -4294,7 +4294,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m "missingCount": 0, "multiTurn": true, "outputTokens": 4, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -4323,7 +4323,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream m }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -4503,7 +4503,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 0, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -4532,7 +4532,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { @@ -4659,7 +4659,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p "mismatchCount": 0, "missingCount": 0, "outputTokens": 2, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "stageMeasurements": { "rows": [ { @@ -4688,7 +4688,7 @@ exports[`runner parity (golden Convex payload + event sequence) > local-stream p }, ], "schemaVersion": 1, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, }, "stageResults": [ { diff --git a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts index b7fdb45a0e..43e48d26ad 100644 --- a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts +++ b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts @@ -64,6 +64,27 @@ export type HostedEvalTurnOutcome = kind: "failed"; iterationError: string; iterationErrorDetails?: string; + /** + * WHICH LAYER failed, decided by the catch site rather than by reading + * the message. + * + * `model` means the model-call layer — the engine's stream, or a throw + * escaping the assistant turn. A failure there is ours or our + * provider's: an outage, an exhausted credit balance, a spend guardrail. + * It says nothing about the MCP server under test, which is exactly why + * the chain must not file it as an unattributed server failure. + * + * `setup` is pre-turn work that never reached the model. + * + * Deliberately NOT derived from the error text. A message classifier + * would be one provider's wording away from silently mis-attributing a + * whole class of run, and the catch site already knows the answer. + */ + errorSource?: "model" | "setup"; + /** The engine's structured code, when the failure carried one. */ + errorCode?: string; + /** HTTP status, when the failure came from a non-OK response. */ + errorHttpStatus?: number; }; /** Stream-runner SSE concerns, layered over the shared skeleton per turn. */ @@ -279,16 +300,10 @@ const truncateError = (message: string): string => export const MAX_WIDGET_FOLLOWUP_TURNS = 3; export async function driveHostedEvalTurn( - params: DriveHostedEvalTurnParams + params: DriveHostedEvalTurnParams, ): Promise { - const { - promptIndex, - browser, - prepared, - acc, - isAborted, - abortSignal, - } = params; + const { promptIndex, browser, prepared, acc, isAborted, abortSignal } = + params; const logSuffix = params.logSuffix ?? ""; // Browser-rendered MCP App eval (PR 14): stamp collected artifacts with @@ -315,7 +330,7 @@ export async function driveHostedEvalTurn( ? params.toolPolicyGate.wrap(mergedTools) : mergedTools, traceCtx, - promptIndex + promptIndex, ); // Push the user prompt into `messageHistory` BEFORE the engine call so a @@ -340,8 +355,9 @@ export async function driveHostedEvalTurn( // parent's already-committed calls end so the post-turn reconcile below // replaces only THIS turn's live entries (the stream runner's `onToolCall` // populates the array live) without wiping the parent's. - const promptToolsCalled: ToolCall[] = (acc.toolsCalledByPrompt[promptIndex] ??= - []); + const promptToolsCalled: ToolCall[] = (acc.toolsCalledByPrompt[ + promptIndex + ] ??= []); const promptToolsBaseline = promptToolsCalled.length; // Built inside the pre-turn try below; `{}` until then so the failure @@ -362,14 +378,14 @@ export async function driveHostedEvalTurn( // failure branches below (CodeRabbit, PR 2610). const mapThrownTurnError = ( error: unknown, - failedStage: string + failedStage: string, ): HostedEvalTurnOutcome => { if ( isAborted() || (error instanceof Error && error.name === "AbortError") ) { logger.debug( - `[evals] backend iteration${logSuffix} aborted due to cancellation` + `[evals] backend iteration${logSuffix} aborted due to cancellation`, ); return { kind: "cancelled" }; } @@ -395,7 +411,16 @@ export async function driveHostedEvalTurn( ...(iterationErrorDetails ? { iterationErrorDetails } : {}), }; sinks.onTurnFailure?.(failure); - return { kind: "failed", ...failure }; + // `failedStage` already names the layer; "pre-turn setup" is the one call + // site that never reached the model. + return { + kind: "failed" as const, + ...failure, + errorSource: + failedStage === "pre-turn setup" + ? ("setup" as const) + : ("model" as const), + }; }; // Pre-turn setup that can genuinely throw: the Chromium widget dismissal @@ -455,7 +480,7 @@ export async function driveHostedEvalTurn( systemPrompt: EVAL_WIDGET_MODEL_CONTEXT ? withWidgetContextSystemPrompt( prepared.enhancedSystemPrompt, - browser.browserInteractionSteps + browser.browserInteractionSteps, ) : prepared.enhancedSystemPrompt, ...(prepared.resolvedTemperature != null @@ -529,8 +554,7 @@ export async function driveHostedEvalTurn( // opt-out and a truthy check would erase it. ...(params.modelVisibleMcpToolResults !== undefined ? { - modelVisibleMcpToolResults: - params.modelVisibleMcpToolResults, + modelVisibleMcpToolResults: params.modelVisibleMcpToolResults, } : {}), ...(params.respectToolVisibility !== undefined @@ -583,7 +607,7 @@ export async function driveHostedEvalTurn( // aborted run as a verdict failure. if (isAborted()) { logger.debug( - `[evals] backend iteration${logSuffix} aborted mid-turn; skipping record` + `[evals] backend iteration${logSuffix} aborted mid-turn; skipping record`, ); return { kind: "cancelled" }; } @@ -650,7 +674,7 @@ export async function driveHostedEvalTurn( // generic fallbacks. const failTurn = ( fallbackError: string, - logLine: string + logLine: string, ): HostedEvalTurnOutcome => { const failure = lastEngineError ? { @@ -660,7 +684,19 @@ export async function driveHostedEvalTurn( : { iterationError: fallbackError }; logger.error(logLine); sinks.onTurnFailure?.(failure); - return { kind: "failed", ...failure }; + // Every path through here is the engine's stream failing, so the layer is + // known without inspecting anything. The structured code and status ride + // along when the engine captured them — they are diagnostics, never the + // basis for the classification. + return { + kind: "failed" as const, + ...failure, + errorSource: "model" as const, + ...(lastEngineError?.code ? { errorCode: lastEngineError.code } : {}), + ...(typeof lastEngineError?.httpStatus === "number" + ? { errorHttpStatus: lastEngineError.httpStatus } + : {}), + }; }; if (!turnResult.turnTrace) { @@ -669,16 +705,16 @@ export async function driveHostedEvalTurn( `[evals] runAssistantTurn${logSuffix} returned no turnTrace (engine runSucceeded=false); treating as cycle failure (messagesGrew=${ newMessages.length > 0 }, engineError=${ - lastEngineError ? (lastEngineError.code ?? "uncoded") : "none" - })` + lastEngineError ? lastEngineError.code ?? "uncoded" : "none" + })`, ); } if (newMessages.length === 0) { return failTurn( "Backend step returned no content (stream error or empty response)", `[evals] runAssistantTurn${logSuffix} produced no new messages this turn; treating as cycle failure (engineError=${ - lastEngineError ? (lastEngineError.code ?? "uncoded") : "none" - })` + lastEngineError ? lastEngineError.code ?? "uncoded" : "none" + })`, ); } // Cursor / Codex review fix: filter to backend step / LLM failure spans @@ -691,7 +727,7 @@ export async function driveHostedEvalTurn( (span) => span.status === "error" && span.category !== "tool" && - !(span as { toolCallId?: string }).toolCallId + !(span as { toolCallId?: string }).toolCallId, ); if (stepErrorSpan) { return failTurn( @@ -699,8 +735,8 @@ export async function driveHostedEvalTurn( `[evals] runAssistantTurn${logSuffix} turnTrace has non-tool error-status span; treating as cycle failure (span=${ stepErrorSpan.name } category=${stepErrorSpan.category} engineError=${ - lastEngineError ? (lastEngineError.code ?? "uncoded") : "none" - })` + lastEngineError ? lastEngineError.code ?? "uncoded" : "none" + })`, ); } diff --git a/mcpjam-inspector/server/services/evals/finalize-iteration.ts b/mcpjam-inspector/server/services/evals/finalize-iteration.ts index 62292841cb..e667ec09d9 100644 --- a/mcpjam-inspector/server/services/evals/finalize-iteration.ts +++ b/mcpjam-inspector/server/services/evals/finalize-iteration.ts @@ -92,7 +92,7 @@ type PolicyBlockRecord = { reason?: unknown }; * summary reason when multiple policy blocks occur. */ function getIterationPolicyReason( - policyBlocks: ReadonlyArray + policyBlocks: ReadonlyArray, ): string | undefined { const reason = policyBlocks[0]?.reason; return typeof reason === "string" ? reason : undefined; @@ -125,6 +125,8 @@ function buildStageEvidence(args: { * failure would leave `call`/`response` looking unmeasured. */ toolErrors?: unknown[]; + /** Which layer raised a fatal step error, when the runner classified it. */ + stepError?: StageEvidence["stepError"]; toolSignals?: ToolExposureSignals; setupSignals?: StageSetupSignals; /** Advisory judge evidence. Absent on the first pass; see {@link buildStageMetadata}. */ @@ -159,6 +161,7 @@ function buildStageEvidence(args: { }>, } : {}), + ...(args.stepError ? { stepError: args.stepError } : {}), ...(args.toolSignals ? { toolSignals: args.toolSignals } : {}), ...(args.setupSignals ? { setupSignals: args.setupSignals } : {}), ...(args.judgeEvidence ? { judgeEvidence: args.judgeEvidence } : {}), @@ -198,6 +201,12 @@ export function buildStageMetadata(args: { predicateResults?: unknown[]; widgetRenderObservations?: RunnerWidgetRenderObservation[]; stageToolErrors?: unknown[]; + /** + * Which layer raised a fatal step error, when the runner classified it. + * Reported by the catch site, never parsed out of `error` — see + * `StageStepErrorLike`. + */ + stepError?: StageEvidence["stepError"]; toolSignals?: ToolExposureSignals; setupSignals?: StageSetupSignals; /** @@ -234,6 +243,7 @@ export function buildStageMetadata(args: { predicateResults: args.predicateResults, widgetRenderObservations: args.widgetRenderObservations, toolErrors: args.stageToolErrors, + ...(args.stepError ? { stepError: args.stepError } : {}), toolSignals: args.toolSignals, setupSignals: args.setupSignals, ...(args.judgeEvidence ? { judgeEvidence: args.judgeEvidence } : {}), @@ -250,7 +260,7 @@ export function buildStageMetadata(args: { /** A predicate row the score projection can key a criterion off. */ function isHostedPredicateResult( - value: unknown + value: unknown, ): value is HostedPredicateResultLike { if (typeof value !== "object" || value === null) return false; const row = value as { predicate?: unknown; passed?: unknown }; @@ -263,7 +273,7 @@ function isHostedPredicateResult( /** Read only the matcher fields the projection needs, typed rather than cast. */ function narrowEvaluation( - evaluation: Record + evaluation: Record, ): HostedEvaluationLike { const list = (key: string): readonly unknown[] | undefined => { const value = evaluation[key]; @@ -353,7 +363,7 @@ function buildScoreMetadata(args: { } { if (args.mode === "off") return { keys: {} }; const predicateResults = (args.predicateResults ?? []).filter( - isHostedPredicateResult + isHostedPredicateResult, ); const { scores, evaluationConfig } = buildHostedScoreContract({ ...(predicateResults.length ? { predicateResults } : {}), @@ -419,7 +429,7 @@ function buildScoreMetadata(args: { ...(typeof args.stageMetadata.stageAnalyzerVersion === "number" ? { stageAnalyzerVersion: args.stageMetadata.stageAnalyzerVersion } : {}), - } + }, ); // The emitter is only REACHED on disagreement, so a spy on it counts // mismatches rather than comparisons — that is what makes @@ -441,14 +451,20 @@ function buildScoreMetadata(args: { * first-pass mismatch means the score projection disagrees with `passed`. */ function readUserValueRow( - stageMetadata: Record -): { state: StageResultRow["state"]; reason: StageResultRow["reason"] } | undefined { + stageMetadata: Record, +): + | { state: StageResultRow["state"]; reason: StageResultRow["reason"] } + | undefined { const rows = stageMetadata.stageResults; if (!Array.isArray(rows)) return undefined; for (const row of rows) { if (typeof row !== "object" || row === null) continue; const candidate = row as Partial; - if (candidate.stage === "userValue" && candidate.state && candidate.reason) { + if ( + candidate.stage === "userValue" && + candidate.state && + candidate.reason + ) { return { state: candidate.state, reason: candidate.reason }; } } @@ -473,7 +489,9 @@ function buildSelectionToolCatalogMetadata(args: { if (!Array.isArray(rows)) return {}; const selectionRow = rows.find( (row): row is Partial => - typeof row === "object" && row !== null && (row as { stage?: unknown }).stage === "selection" + typeof row === "object" && + row !== null && + (row as { stage?: unknown }).stage === "selection", ); if (selectionRow?.state !== "failed") return {}; @@ -483,12 +501,14 @@ function buildSelectionToolCatalogMetadata(args: { // and folding them in could fill the catalog's cap before the turn that // actually caused the failure is ever considered. const failingPrompts = prompts.filter( - (p) => (p.missing?.length ?? 0) > 0 || (p.unexpected?.length ?? 0) > 0 + (p) => (p.missing?.length ?? 0) > 0 || (p.unexpected?.length ?? 0) > 0, ); const expectedToolNames = failingPrompts .flatMap((p) => p.missing ?? []) .map((t) => t.toolName) - .filter((name): name is string => typeof name === "string" && name.length > 0); + .filter( + (name): name is string => typeof name === "string" && name.length > 0, + ); // `unexpected` names FIRST, then the rest of the turn's actual calls: // `buildSelectionToolCatalog`'s cap is shared across both roles, and for // an `unexpectedToolCall` failure (e.g. `maxExtraToolCalls: 0`, six @@ -503,11 +523,15 @@ function buildSelectionToolCatalogMetadata(args: { const unexpectedToolNames = failingPrompts .flatMap((p) => p.unexpected ?? []) .map((t) => t.toolName) - .filter((name): name is string => typeof name === "string" && name.length > 0); + .filter( + (name): name is string => typeof name === "string" && name.length > 0, + ); const otherActualToolNames = failingPrompts .flatMap((p) => p.actualToolCalls ?? []) .map((t) => t.toolName) - .filter((name): name is string => typeof name === "string" && name.length > 0); + .filter( + (name): name is string => typeof name === "string" && name.length > 0, + ); const actualToolNames = [...unexpectedToolNames, ...otherActualToolNames]; if (expectedToolNames.length === 0 && actualToolNames.length === 0) { return {}; @@ -594,6 +618,11 @@ export function buildIterationFinishParams(args: { * them unless they are threaded here. */ stageToolErrors?: unknown[]; + /** + * Which layer raised the fatal step error, forwarded from the executor. + * Absent when nothing fatal happened, or when the caller could not say. + */ + stepError?: StageEvidence["stepError"]; /** Execution-layer policy blocks; persisted as metadata, never a failure. */ policyBlocks?: PolicyBlockRecord[]; /** Non-fatal policy configuration warnings, persisted for run consumers. */ @@ -693,10 +722,7 @@ export function buildIterationFinishParams(args: { selectionTools, } = args; const gradingMode = args.gradingMode ?? resolveGradingEngineMode(); - const persistedSpans = [ - ...(setupSpans ?? []), - ...(spans ?? []), - ]; + const persistedSpans = [...(setupSpans ?? []), ...(spans ?? [])]; const stageMetadata = buildStageMetadata({ ...(stageCase ? { stageCase } : {}), spans, @@ -705,6 +731,7 @@ export function buildIterationFinishParams(args: { predicateResults, widgetRenderObservations, stageToolErrors, + ...(args.stepError ? { stepError: args.stepError } : {}), toolSignals, setupSignals, policy: @@ -741,14 +768,13 @@ export function buildIterationFinishParams(args: { // would silently switch D7's catalog capture back OFF for the cohort that // has progressed furthest. The predicate is what keeps "dual_write and // above" in one place. - const selectionToolCatalogMetadata = - isDualWrite(gradingMode) - ? buildSelectionToolCatalogMetadata({ - stageMetadata, - prompts, - selectionTools, - }) - : {}; + const selectionToolCatalogMetadata = isDualWrite(gradingMode) + ? buildSelectionToolCatalogMetadata({ + stageMetadata, + prompts, + selectionTools, + }) + : {}; // THE FLIP, and the ONE DIRECTION IT MAY MOVE. // @@ -1013,8 +1039,8 @@ export async function finalizeEvalIteration( iterationStatus === "cancelled" ? "eval_cancelled" : isCycleFailure - ? "eval_failed" - : "eval_completed"; + ? "eval_failed" + : "eval_completed"; // PR 13: emit per-iteration browser-eval observability from the runner-local // arrays (covers both the stream + non-stream paths via this shared choke @@ -1069,8 +1095,7 @@ export async function finalizeEvalIteration( // before any turn landed. With turns already written, re-sending // would overwrite turn 0 (W1 always writes at promptIndex: 0) and // orphan turns 1..N. See persist-eval-trace.ts for the contract. - const useW1Fallback = - fanout.persisted === false && fanout.turnsWritten === 0; + const useW1Fallback = fanout.persisted === false && fanout.turnsWritten === 0; if (fanout.persisted === false) { logger.warn( useW1Fallback @@ -1116,8 +1141,7 @@ export async function finalizeEvalIteration( : {}), ...(widgetSnapshots?.length ? { - widgetSnapshots: - sanitizeForConvexTransport(widgetSnapshots), + widgetSnapshots: sanitizeForConvexTransport(widgetSnapshots), } : {}), // PR 6b: browser artifacts already uploaded + sanitized above; diff --git a/mcpjam-inspector/server/services/evals/step-executor.ts b/mcpjam-inspector/server/services/evals/step-executor.ts index 477863bd25..d8f1efd9c3 100644 --- a/mcpjam-inspector/server/services/evals/step-executor.ts +++ b/mcpjam-inspector/server/services/evals/step-executor.ts @@ -24,10 +24,7 @@ */ import type { ModelMessage } from "ai"; -import type { - PredicateResult, - ToolErrorRecord, -} from "@/shared/eval-matching"; +import type { PredicateResult, ToolErrorRecord } from "@/shared/eval-matching"; import { buildIterationTranscript, evaluatePredicates, @@ -161,6 +158,20 @@ export interface StepEngineOutcome { */ iterationError?: string; iterationErrorDetails?: string; + /** + * WHICH LAYER produced `iterationError`, reported by the catch site that + * raised it rather than inferred from its text. + * + * `model` is the model-call layer — our provider, not the MCP server under + * test. The chain uses it to stop filing an outage or an exhausted credit + * balance as an unattributed server failure. Absent when the caller does + * not know, which reads as "unclassified" and changes nothing. + */ + errorSource?: "model" | "setup"; + /** The engine's structured code, when the failure carried one. */ + errorCode?: string; + /** HTTP status, when the failure came from a non-OK response. */ + errorHttpStatus?: number; /** * When true, the iterationError is a SETUP failure (status:"failed"), not an * assertion failure (status:"completed"+error). Mirrors the pinned @@ -214,6 +225,10 @@ export interface StepExecutorResult { /** Set when a `prompt`/`toolCall` step reported a fatal error. */ iterationError?: string; iterationErrorDetails?: string; + /** Which layer raised `iterationError` — see `StepEngineOutcome`. */ + errorSource?: "model" | "setup"; + errorCode?: string; + errorHttpStatus?: number; /** True when `iterationError` is a setup (not assertion) failure. */ setupFailure: boolean; } @@ -222,8 +237,7 @@ export interface StepExecutorResult { export function hasWidgetDrivingStep(steps: TestStep[]): boolean { return steps.some( (s) => - isInteractStep(s) || - (isAssertStep(s) && isWidgetAssertion(s.assertion)), + isInteractStep(s) || (isAssertStep(s) && isWidgetAssertion(s.assertion)), ); } @@ -248,7 +262,8 @@ function applyOutcome( turn: number, ): void { if (outcome.messages?.length) state.messages.push(...outcome.messages); - if (outcome.toolCalls?.length) recordToolCalls(state, turn, outcome.toolCalls); + if (outcome.toolCalls?.length) + recordToolCalls(state, turn, outcome.toolCalls); if (outcome.toolErrors?.length) state.toolErrors.push(...outcome.toolErrors); if (outcome.usage) { state.usage.inputTokens += outcome.usage.inputTokens ?? 0; @@ -262,9 +277,7 @@ function snapshotTranscript(state: StepExecutionState) { const finalAssistantMessage = extractFinalAssistantMessage(state.messages); return buildIterationTranscript({ toolCalls: state.toolCalls, - ...(finalAssistantMessage !== undefined - ? { finalAssistantMessage } - : {}), + ...(finalAssistantMessage !== undefined ? { finalAssistantMessage } : {}), usage: state.usage.inputTokens || state.usage.outputTokens || @@ -360,7 +373,11 @@ async function drainAndDriveFollowUps( return undefined; } remaining -= 1; - const outcome = await handlers.onFollowUp!({ text, stepIndex, turnOrdinal: turn }); + const outcome = await handlers.onFollowUp!({ + text, + stepIndex, + turnOrdinal: turn, + }); applyOutcome(state, outcome, turn); if (outcome.iterationError) return outcome.iterationError; } @@ -387,8 +404,7 @@ async function runAssertStep( passed: outcome.ok, reason: outcome.ok ? `widget assertion "${step.assertion.kind}" passed` - : outcome.reason ?? - `widget assertion "${step.assertion.kind}" failed`, + : outcome.reason ?? `widget assertion "${step.assertion.kind}" failed`, }); return; } @@ -556,6 +572,11 @@ export async function executeSteps(args: { state, iterationError: outcome.iterationError, iterationErrorDetails: outcome.iterationErrorDetails, + ...(outcome.errorSource ? { errorSource: outcome.errorSource } : {}), + ...(outcome.errorCode ? { errorCode: outcome.errorCode } : {}), + ...(typeof outcome.errorHttpStatus === "number" + ? { errorHttpStatus: outcome.errorHttpStatus } + : {}), setupFailure: outcome.setupFailure === true, }; } @@ -593,6 +614,11 @@ export async function executeSteps(args: { state, iterationError: outcome.iterationError, iterationErrorDetails: outcome.iterationErrorDetails, + ...(outcome.errorSource ? { errorSource: outcome.errorSource } : {}), + ...(outcome.errorCode ? { errorCode: outcome.errorCode } : {}), + ...(typeof outcome.errorHttpStatus === "number" + ? { errorHttpStatus: outcome.errorHttpStatus } + : {}), setupFailure: outcome.setupFailure === true, }; } diff --git a/sdk/src/contract/decision-labels.ts b/sdk/src/contract/decision-labels.ts index e9f8736eab..56ef074076 100644 --- a/sdk/src/contract/decision-labels.ts +++ b/sdk/src/contract/decision-labels.ts @@ -102,6 +102,8 @@ export const STAGE_REASON_LABELS = Object.freeze({ blockedByPolicy: "a policy blocked the run before it could be measured", evaluatorError: "the evaluator itself failed, so the run says nothing about the server", + providerError: + "the model provider failed the call, so the run never reached the server", setupAborted: "the environment was never prepared, so the test never began", connectFailed: "the configured server was reached and initialize failed there", diff --git a/sdk/src/contract/index.ts b/sdk/src/contract/index.ts index 90d9ca1acd..cd61b701c2 100644 --- a/sdk/src/contract/index.ts +++ b/sdk/src/contract/index.ts @@ -156,6 +156,7 @@ export type { StageSetupPhaseSignal, StageSetupSignals, StageSpanLike, + StageStepErrorLike, StageToolErrorLike, } from "./stage-derivation.js"; export { diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index 94d337a704..1eff7674f5 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -76,8 +76,13 @@ import { * every applicable stage green while its legacy verdict failed on exactly * that tool error — a disagreement the chain had no row to express. Re-uses * `toolError`, so again no mirror re-pin. + * + * 8 (UVH-IN2): a model-call-layer failure is attributed to `providerError` + * instead of leaving the trial uncategorised. This one DOES move + * `STAGE_REASONS`; the backend mirror already carries the member (UVH-BE1 + * shipped it deliberately ahead of this bump), so nothing quarantines. */ -export const STAGE_ANALYZER_VERSION = 7; +export const STAGE_ANALYZER_VERSION = 8; /** * Why a stage landed where it did. @@ -108,6 +113,16 @@ export const STAGE_REASONS = [ "blockedByPolicy", /** The grader itself failed, so the run says nothing about the server. */ "evaluatorError", + /** + * The MODEL-CALL layer failed: a provider outage, an exhausted credit + * balance, a rate limit, or one of our own spend guardrails. + * + * Broader than the name suggests, and deliberately so — what every case has + * in common is that OUR side of the call broke, so the run says nothing + * about the MCP server under test. Never `failed`: blaming the server for + * our provider's bad day is the mis-attribution this reason exists to stop. + */ + "providerError", /** The harness never got to the test (setup abort). */ "setupAborted", /** @@ -304,6 +319,20 @@ export type StageToolErrorLike = { toolName?: string; }; +/** + * The layer a fatal step error came from, reported by the catch site. + * + * `deriveStageResults` reads only `source`; `code` and `httpStatus` ride along + * as diagnostics for a reader, and are deliberately NOT part of the + * classification — a rule keyed on a provider's status codes would be one + * provider away from mis-attributing a whole class of run. + */ +export type StageStepErrorLike = { + source?: "model" | "setup"; + code?: string; + httpStatus?: number; +}; + export type StageRenderObservationLike = { status?: string; }; @@ -414,6 +443,14 @@ export type StageEvidence = { prompts?: readonly StagePromptSummaryLike[]; predicateResults?: readonly StagePredicateResultLike[]; toolErrors?: readonly StageToolErrorLike[]; + /** + * The fatal step error's LAYER, when one was raised and the runner knew it. + * + * Distinct from `toolErrors`, which are the server's answers. This is our + * own side breaking, and it is the difference between "the server gave us + * bad data" and "we never got to ask". + */ + stepError?: StageStepErrorLike; renderObservations?: readonly StageRenderObservationLike[]; /** `tools_total_before` / `tools_exposed` — the one direct discovery signal. */ toolSignals?: { toolsTotalBefore?: number; toolsExposed?: number }; @@ -1100,7 +1137,14 @@ function categoryFor( evidence: StageEvidence ): FailureCategory | undefined { if (!firstFailed) { - return evidence.evaluatorErrored ? "evaluator" : undefined; + if (evidence.evaluatorErrored) return "evaluator"; + // A model-call failure leaves nothing failed — there was nothing to fail + // against. Before UVH-IN2 that produced a run with no category at all, + // which reads as "we cannot say what went wrong" when in fact we can say + // precisely: our provider did. `setup` is the existing bucket for our own + // side breaking, which is why this needs no new category. + if (evidence.stepError?.source === "model") return "setup"; + return undefined; } const failedRow = rows.find((r) => r.stage === firstFailed); switch (firstFailed) { @@ -1323,11 +1367,43 @@ function mergeMetadataAttributionEvidence( ); } +/** + * Re-label the stages a MODEL-CALL failure left blank. + * + * Applied last, and only to rows that measured nothing: a stage with its own + * evidence keeps its own row, because the provider dying at turn 4 does not + * un-observe what turns 1-3 established. What it replaces is the bare + * `noEvidenceCaptured` / `traceAbsent` gap, which reads as "we looked and the + * server told us nothing" — an accusation, when the truth is that our own + * provider never let us ask. + * + * `notMeasured` throughout, never `failed`. A run that could not be attempted + * has measured nothing about the server, and inflating a server failure rate + * with our own outage is exactly what this reason exists to prevent. + */ +function applyProviderError( + rows: StageResultRow[], + evidence: StageEvidence +): StageResultRow[] { + if (evidence.stepError?.source !== "model") return rows; + const BLANK: ReadonlyArray = [ + "noEvidenceCaptured", + "traceAbsent", + "executorEmitsNoSpans", + ]; + return rows.map((r) => + r.state === "notMeasured" && BLANK.includes(r.reason) + ? { ...r, reason: "providerError" as const } + : r + ); +} + function finalize( rows: StageResultRow[], evidence: StageEvidence, forcedCategory?: FailureCategory ): StageDerivation { + rows = applyProviderError(rows, evidence); const firstFailedStage = rows.find((r) => r.state === "failed")?.stage; const failureCategory = forcedCategory ?? categoryFor(firstFailedStage, rows, evidence); diff --git a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json index 110060d901..f29f8600f5 100644 --- a/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json +++ b/sdk/tests/fixtures/eval-run-decision-summary-fixtures.json @@ -2469,7 +2469,7 @@ "analyzerVersion": 99, "analyzerVersionAhead": { "reported": 99, - "known": 7 + "known": 8 } }, "expected": { diff --git a/sdk/tests/fixtures/parity/v1/MANIFEST.json b/sdk/tests/fixtures/parity/v1/MANIFEST.json index 049f146b9f..470fb78f07 100644 --- a/sdk/tests/fixtures/parity/v1/MANIFEST.json +++ b/sdk/tests/fixtures/parity/v1/MANIFEST.json @@ -27,104 +27,104 @@ "maxIterations": 200, "maxBytes": 2097152 }, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "iterations": 19, "bytes": 34215, - "corpusDigest": "878f6d376c5370c13aa5c35e73ee7b9ea96b41bb4467f791892245ca7a819321", + "corpusDigest": "53b9c34d077cbeecce0e1ac29d3f42c285b2c554e88b1c5cb62965dd059fe6a0", "files": [ { "path": "iteration-0001.json", - "sha256": "0dbc450eea310da28f20936a9237ce6680602d1e60c27158f713464dbdb536b7", + "sha256": "79253c9a3716b5ded0c5223088936038a14926a08be78648d813f6b6ce8353dd", "bytes": 1964 }, { "path": "iteration-0002.json", - "sha256": "002a69070e2692549173a0e15203d95d948ee48a4914d583aa507e9fa08b63a4", + "sha256": "3ec5c4e90fa615c46f3984e9e06f3b89421392dc5e242c5f33ec76983bb6161c", "bytes": 2254 }, { "path": "iteration-0003.json", - "sha256": "d5e3dacda886713ae9d7280de56b2b11d712dd3cdaebdf7ddf97be825b6a8b69", + "sha256": "6b0d0b96c651cc51c38b2dbc9a72e90763a0dbfdafed62ae60535fb17d2514ed", "bytes": 1798 }, { "path": "iteration-0004.json", - "sha256": "a0a8f56d21e166dca2bed0b9ab2b70437fa8f775b295c6d24e72a0d0da55d7bc", + "sha256": "6eb831aac09f403a02369e84d615af8c3fd10f79b7b2a561cbcc9765c9aba2e7", "bytes": 1986 }, { "path": "iteration-0005.json", - "sha256": "355cf2c77f69aeabb3a3c204ec69e951a34cdb4dc60dbd7d9f128c5f51d8b297", + "sha256": "ea441a999f66ddb8b043add4299bd325f1bfc2ab630e67e8c5e5ba330dc5da5a", "bytes": 1933 }, { "path": "iteration-0006.json", - "sha256": "59865c00ee14c90d289961d613235c9a6b7b342ba8b82443f13e0259713cda38", + "sha256": "74534c971315c8c2ddea649ad8e458b57f8a12b236aa0ae7baaad0ad1cb98c27", "bytes": 1510 }, { "path": "iteration-0007.json", - "sha256": "85fdd897c33cf41ca5789c46f79c9491d2abe9701b2d7e00631d4ed65cf37f03", + "sha256": "e4f9697ad5d36a0d351d53c8ed24a5264fa9b3c23c4c8743eeb3246728900983", "bytes": 1512 }, { "path": "iteration-0008.json", - "sha256": "936168e81f242bc01f19d5ea6218901821533cf8be7247cbe32eaab6346a0578", + "sha256": "02cb20b659c1b20696a9e9501836aba4070713b21e16ff3cfd7d946357aea81f", "bytes": 1202 }, { "path": "iteration-0009.json", - "sha256": "b34b97aa9c370e656e9bc69b81d54eff38646f0e4741b631d261722cfcad1775", + "sha256": "8c1283b6ffda30bce6dfed9dedb5224c0da672f5c2df85061d84371aed87ac22", "bytes": 1583 }, { "path": "iteration-0010.json", - "sha256": "e77240046df9a0c992cd155a471cff741fb4772faeed1c388d99502d259f1ea6", + "sha256": "1ea1f568ea41f821a05bd96f163b0ab8b2b02fa3926a6f36559b7f81c2a14919", "bytes": 1602 }, { "path": "iteration-0011.json", - "sha256": "ef36ec48540c87d6100429d4641a17c219af841be25c4ce1541c20e6878eac9e", + "sha256": "e29deff5e828e451764531a8d83349a23adb568a31894fb470d4c5cb332590e6", "bytes": 2201 }, { "path": "iteration-0012.json", - "sha256": "2383fbad01107d300decb40fed1211d4a7696b71272d1835b770bff654366c58", + "sha256": "1335e6c6d80ce303e596c3824e4ca71bdb01d8f3696fa9877d997f34bb6f3209", "bytes": 1616 }, { "path": "iteration-0013.json", - "sha256": "2c24f860be94f279b3acd94845898b3419533e2a95511f992bf5aa68a29860fd", + "sha256": "e8191ee2dbe37d0fdae40faa0e62172fb9a596086cee57f0f8acab407b670c72", "bytes": 1878 }, { "path": "iteration-0014.json", - "sha256": "fb0eb96bc62f721bfd35027f12f78fbae3cbb8798a04111c3bc1588ff2fb487c", + "sha256": "a233cfa58443aa1d81f27202193abb36730978c0acfdae3971bf5be67075e3cc", "bytes": 1874 }, { "path": "iteration-0015.json", - "sha256": "12f5c4dc972aabd9bc01c8e095426064ad9c48fdcafd90d6918590ba5f64b6b6", + "sha256": "070ecee8d19ddad4fbefa5e22a9d6fcd06db2e9cfe4a53488efc5a11a1d53892", "bytes": 1846 }, { "path": "iteration-0016.json", - "sha256": "7d9f4f5dc8080c598fe8b2eaf93c83763550ffed7ae62562685a347a2625ae29", + "sha256": "64d3a700ce158e1e58d59d7a01013130fee480b2cdbc9ea29df21fb43b6810ec", "bytes": 1821 }, { "path": "iteration-0017.json", - "sha256": "e30bd72bd1596bc4a21f74aed893a46f9b07db67373514b4fe31f457c7d836f4", + "sha256": "5e67b557a53c68c0ce87e020cc1f1c590010c68045641d66ec83e043fadad607", "bytes": 1971 }, { "path": "iteration-0018.json", - "sha256": "a46e1490142534f2c8484639df05dfea300ca0e2dad2cfd0d8df7a4355f38cf0", + "sha256": "1dc27f05db7a8c496d58630f64f47a48c507ff964501b4b6bd0efa7b75e50609", "bytes": 1833 }, { "path": "iteration-0019.json", - "sha256": "5285863ba6cb805ee0e866ceb26ee811b5052dba7dfb69477ac1b5b78bc5b6a7", + "sha256": "f120eb03496d89b3b787b373e173264713df898cec2e0ab86e3989cc1503f6a8", "bytes": 1831 } ], diff --git a/sdk/tests/fixtures/parity/v1/iteration-0001.json b/sdk/tests/fixtures/parity/v1/iteration-0001.json index 02b72ee663..dbfd26ff64 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0001.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0001.json @@ -97,6 +97,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0002.json b/sdk/tests/fixtures/parity/v1/iteration-0002.json index 671e58dd23..2b2d649261 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0002.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0002.json @@ -108,6 +108,6 @@ ], "firstFailedStage": "userValue", "failureCategory": "userValue", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0003.json b/sdk/tests/fixtures/parity/v1/iteration-0003.json index 0bd186321c..12bd18d709 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0003.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0003.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0004.json b/sdk/tests/fixtures/parity/v1/iteration-0004.json index 643b425a4d..d99fcb15a4 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0004.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0004.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "serverData", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0005.json b/sdk/tests/fixtures/parity/v1/iteration-0005.json index 4a0b054d2b..151e885fe8 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0005.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0005.json @@ -91,6 +91,6 @@ ], "firstFailedStage": "call", "failureCategory": "setup", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0006.json b/sdk/tests/fixtures/parity/v1/iteration-0006.json index 3b76d6e3e8..7d1557d32c 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0006.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0006.json @@ -65,6 +65,6 @@ ], "firstFailedStage": "connection", "failureCategory": "setup", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0007.json b/sdk/tests/fixtures/parity/v1/iteration-0007.json index 12d81432c4..e5bf06e90f 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0007.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0007.json @@ -67,6 +67,6 @@ ], "firstFailedStage": "discovery", "failureCategory": "setup", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0008.json b/sdk/tests/fixtures/parity/v1/iteration-0008.json index 260069371d..603051bc62 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0008.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0008.json @@ -55,6 +55,6 @@ "reason": "traceAbsent" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0009.json b/sdk/tests/fixtures/parity/v1/iteration-0009.json index d6bd8710c9..85666d705a 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0009.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0009.json @@ -75,6 +75,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0010.json b/sdk/tests/fixtures/parity/v1/iteration-0010.json index 1e8d5b7bc3..094f72b5f7 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0010.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0010.json @@ -77,6 +77,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0011.json b/sdk/tests/fixtures/parity/v1/iteration-0011.json index 3b6278a1c8..7df75ff5cb 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0011.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0011.json @@ -106,6 +106,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0012.json b/sdk/tests/fixtures/parity/v1/iteration-0012.json index 078ef955ea..9266dbc53b 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0012.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0012.json @@ -80,6 +80,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0013.json b/sdk/tests/fixtures/parity/v1/iteration-0013.json index 70d227da83..be8d3c0804 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0013.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0013.json @@ -94,6 +94,6 @@ "reason": "observed" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0014.json b/sdk/tests/fixtures/parity/v1/iteration-0014.json index dc327d7faa..01d1209e6f 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0014.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0014.json @@ -91,6 +91,6 @@ ], "firstFailedStage": "response", "failureCategory": "serverData", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0015.json b/sdk/tests/fixtures/parity/v1/iteration-0015.json index d78b7171e1..63c27ad90d 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0015.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0015.json @@ -90,6 +90,6 @@ } ], "failureCategory": "evaluator", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0016.json b/sdk/tests/fixtures/parity/v1/iteration-0016.json index b34d8605e1..8b5c905204 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0016.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0016.json @@ -88,6 +88,6 @@ "reason": "notAuthored" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0017.json b/sdk/tests/fixtures/parity/v1/iteration-0017.json index 3d1923830b..7d903ed1e0 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0017.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0017.json @@ -95,6 +95,6 @@ ], "firstFailedStage": "call", "failureCategory": "arguments", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0018.json b/sdk/tests/fixtures/parity/v1/iteration-0018.json index 2c48c2506d..47a041d68e 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0018.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0018.json @@ -85,6 +85,6 @@ ], "firstFailedStage": "selection", "failureCategory": "selection", - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/parity/v1/iteration-0019.json b/sdk/tests/fixtures/parity/v1/iteration-0019.json index 39b266116a..8f41d9ce66 100644 --- a/sdk/tests/fixtures/parity/v1/iteration-0019.json +++ b/sdk/tests/fixtures/parity/v1/iteration-0019.json @@ -88,6 +88,6 @@ "reason": "noEvidenceCaptured" } ], - "stageAnalyzerVersion": 7 + "stageAnalyzerVersion": 8 } } diff --git a/sdk/tests/fixtures/stage-analytics-golden.json b/sdk/tests/fixtures/stage-analytics-golden.json index 5c4466c109..b94ff5df57 100644 --- a/sdk/tests/fixtures/stage-analytics-golden.json +++ b/sdk/tests/fixtures/stage-analytics-golden.json @@ -11,7 +11,7 @@ "runCompletedAt": 1700000100000, "sourceIterationCount": 7, "sourceMaxUpdatedAt": 1700000090000, - "stageAnalyzerVersion": 7, + "stageAnalyzerVersion": 8, "measurementsSchemaVersion": 1, "materializationState": "provisional", "createdAt": 1700000150000, diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index 8b68edccf7..baeb59510c 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -781,6 +781,79 @@ describe("userValue", () => { }); }); +// ── UVH-IN2: a model-call failure is OURS, not the server's ────────────────── +// +// 20 prod trials failed on "credit balance too low… Anthropic API": the step +// errored, asserts were skipped, and the chain ended `noEvidenceCaptured` with +// NO failure category — an outage filed as an unattributed server failure. + +describe("a model-call failure is attributed, not left blank", () => { + const providerDied = { stepError: { source: "model" as const } }; + + test("blank stages read as providerError, and the run is categorised setup", () => { + const { stageResults, failureCategory, firstFailedStage } = derive({ + evidence: { traceAbsent: true, ...providerDied }, + }); + + // Every applicable stage says the same true thing: we never got to ask. + for (const r of stageResults.filter((x) => x.state === "notMeasured")) { + expect(r.reason).toBe("providerError"); + } + // `setup` is the existing bucket for our own side breaking, so no new + // category was needed — but a category there MUST be. + expect(failureCategory).toBe("setup"); + // Never `failed`: our provider's bad day is not the server's defect. + expect(firstFailedStage).toBeUndefined(); + expect(stageResults.some((r) => r.state === "failed")).toBe(false); + }); + + test("stages that DID measure something keep their own rows", () => { + // A provider dying at turn 4 does not un-observe turns 1-3. Only the + // blank rows are re-labelled. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [cleanTurn], + ...providerDied, + }, + }); + expect(stateOf(stageResults, "call").state).toBe("passed"); + expect(stateOf(stageResults, "selection").state).toBe("passed"); + }); + + test("a SETUP-layer error is not a provider error", () => { + // Pre-turn setup never reached the model, and `setupAborted` already says + // so precisely. Widening `providerError` over it would lose that. + const { stageResults } = derive({ + evidence: { traceAbsent: true, stepError: { source: "setup" } }, + }); + expect(stageResults.every((r) => r.reason !== "providerError")).toBe(true); + }); + + test("an unclassified error changes nothing", () => { + // Callers that cannot say which layer broke leave `stepError` absent, and + // the chain reports exactly what it did before. + const before = derive({ evidence: { traceAbsent: true } }); + expect(before.stageResults.every((r) => r.reason !== "providerError")).toBe( + true + ); + expect(before.failureCategory).toBeUndefined(); + }); + + test("a broken grader still outranks it", () => { + // `evaluator` is never folded into another category — a grader bug is not + // an infrastructure outage, and counting it as one poisons both rates. + const { failureCategory } = derive({ + evidence: { + traceAbsent: true, + evaluatorErrored: true, + ...providerDied, + }, + }); + expect(failureCategory).toBe("evaluator"); + }); +}); + // ── UVH-IN7: an observed tool error makes `response` measurable ────────────── // // The disagreement class this closes: a case authors only transcript From 025334d3bb13062303f1292fa7d7e84d98eb4400 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 21:03:00 +0000 Subject: [PATCH 07/15] ci: run the test suites on this stack's PRs, not just its root `branches` filters on the PR's BASE, so a stacked PR based on the branch below it never matched `main` and never ran these jobs. Every PR above the root of this stack was green on previews and review bots alone. That is not hypothetical here. A test in this stack asserted a stage-reason label's wording verbatim; a later PR in the same stack changed that wording; the break sat unnoticed until the suite was run by hand. The workflow's own comment already anticipated this and carries a pattern for an earlier stack, so this follows that precedent rather than inventing one. Merged forward through the stack so every PR above this one picks it up: for `pull_request`, the workflow that runs is the one in the merge of head into base, so the pattern has to be present on each head branch. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .github/workflows/test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de3a377350..f78c869e35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,16 @@ on: # `branches` filters on the PR's BASE. A stacked PR is based on the branch # below it rather than on main, so with `main` alone every PR above the # root of a stack merges without these tests having run against it once. - # Drop the second pattern once the Connector Bench stack has landed. + # Drop each stack pattern once that stack has landed. + # + # The UVH pattern was added after this bit us: a test in the stack asserted + # a label's wording verbatim, a later PR in the same stack changed that + # wording, and the break sat unnoticed because every PR above the root was + # green on previews and review bots alone. branches: - main - "claude/mcp-benchmarks-v2-b0tw4b-**" + - "claude/uvc-mcp-eval-reporting-gyycwl**" push: branches: [main] From ec733dd0a645606bc6842a7722323db3dba7b90c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 21:21:38 +0000 Subject: [PATCH 08/15] UVH-IN2 review: fix two catch sites that over- and under-attributed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both found by Codex review, both verified against the code, and both are this same defect wearing a different coat — attributing to the model a failure that never reached it, or attributing nothing at all where it should. HARNESS SETUP FAILURES WERE CALLED PROVIDER FAILURES. `failTurn` said "every path through here is the engine's stream failing". True of the chat engine, false of the harness: `runHarnessTurn` wraps its whole turn — preparation included — in one try (the throws for a missing projectId, a missing auth bearer and disabled broker credential delivery are all inside it), and reports every one through the same `onEngineError` a provider outage uses. So our own setup bug was filed as the provider's. The engine now reports the PHASE it failed in, derived from the trace-started flag the emitter already holds to decide whether a turn happened at all — never from reading the message. An engine that reports no phase still means `model`: every emitter that omits it today is a real stream failure, and defaulting the other way would un-attribute the outages this work exists for. THE LOCAL PATH CARRIED NO SOURCE AT ALL. With orgByokRuntime.kind === "local", execution goes to runLocalIteration, and neither of its finish calls supplied a stepError — so local-BYOK trials that died on the model call still finalized with blank stage reasons and no failure category. The original fix never touched that path. The local driver now records the layer at both error sites and threads it through both finish paths: - an empty model stream is unambiguously the model call; - a non-tool error span is the model call ONLY when its category is `llm`. That branch selects every non-tool error span, and connection, discovery and oauth spans all reach it — tagging those `model` would blame the provider for a server we could not reach. Anything else leaves the source unset. An absent source attributes nothing, which is the right floor: no attribution beats a confident wrong one. Both decisions are extracted as pure functions so the choice can be read and tested on its own rather than inferred from a call site's position. Mutation-checked: ignoring the phase, tagging every span, and tagging none each fail exactly their intended tests. Server evals + harness suites: 1079 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-provider-error-catch-sites.md | 22 +++++++ .../server/services/evals-runner.ts | 14 +++++ .../provider-error-attribution.test.ts | 60 +++++++++++++++++++ .../services/evals/drive-hosted-eval-turn.ts | 47 +++++++++++++-- .../services/evals/drive-local-eval-turn.ts | 36 +++++++++++ .../server/utils/harness/run-harness-turn.ts | 6 ++ .../server/utils/mcpjam-stream-handler.ts | 16 +++++ 7 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 .changeset/evals-provider-error-catch-sites.md create mode 100644 mcpjam-inspector/server/services/evals/__tests__/provider-error-attribution.test.ts diff --git a/.changeset/evals-provider-error-catch-sites.md b/.changeset/evals-provider-error-catch-sites.md new file mode 100644 index 0000000000..dc0635de46 --- /dev/null +++ b/.changeset/evals-provider-error-catch-sites.md @@ -0,0 +1,22 @@ +--- +"@mcpjam/inspector": patch +--- + +Provider-error attribution reaches the local path, and stops over-claiming on the harness + +Two catch sites turned out to be broader than the comments above them claimed. Both were found in review, and both are the provider-error defect wearing a different coat — attributing to the model a failure that never reached it, which misleads a report card exactly as much as the original did. + +**The harness reports setup failures through the same callback as stream failures.** `runHarnessTurn` wraps its entire turn — preparation included — in one `try`, so a missing `projectId`, a missing auth bearer, or disabled broker credential delivery arrives at `onEngineError` looking precisely like a provider outage. `failTurn` tagged every one of them `model`, which files our own setup bug as the provider's. + +The engine now reports the **phase** it failed in, derived from the trace-started flag the emitter already holds to decide whether a turn happened at all — never from reading the message. `failTurn` reads that phase. An engine that reports none still means `model`: every emitter that omits it today is a real stream failure, and defaulting the other way would un-attribute the outages this work was built for. + +**The local path carried no source at all.** When `orgByokRuntime.kind === "local"`, execution goes to `runLocalIteration` rather than the hosted driver, and neither of its finish calls supplied a `stepError`. So local-BYOK trials that died on the model call still finalized with blank stage reasons and no failure category — the exact misattribution the original change removed, surviving untouched on the path it never covered. + +The local driver now records the layer at both of its error sites, and threads it through both finish paths: + +- **An empty model stream** is unambiguously the model call — no other layer reaches that branch. +- **A non-tool error span** is the model call **only when its category is `llm`**. That branch selects every non-tool error span, and `connection`, `discovery` and `oauth` spans are all reachable there; tagging those `model` would blame the provider for a server we could not reach. + +Anything else leaves the source unset. An absent source attributes nothing, which is the right floor — no attribution is strictly better than a confident wrong one. + +Both decisions are extracted as small pure functions (`failedLayerForEngineError`, `modelLayerForErrorSpan`) so the choice can be read and tested on its own rather than inferred from a call site's position. Mutation-checked: ignoring the phase, tagging every span, and tagging none each fail exactly their intended tests. diff --git a/mcpjam-inspector/server/services/evals-runner.ts b/mcpjam-inspector/server/services/evals-runner.ts index c9222d9cec..3569211f14 100644 --- a/mcpjam-inspector/server/services/evals-runner.ts +++ b/mcpjam-inspector/server/services/evals-runner.ts @@ -3366,6 +3366,7 @@ const runLocalIteration = async ({ activeTraceCtx: null, iterationError: undefined, iterationErrorDetails: undefined, + stepErrorSource: undefined, pinnedSetupFailure: false, }; // PR 4d review fix (CodeRabbit): hoisted so persistence sites in the @@ -3847,6 +3848,13 @@ const runLocalIteration = async ({ // consumer than the stored transcript). const finishParams = buildIterationFinishParams({ iterationId, + // The layer that failed, when this driver could tell — the local twin of + // the hosted path's `stepError`. Without it a local-BYOK trial that died + // on the model call finalizes uncategorised, which is the very failure + // the provider-error work removed on the hosted path. + ...(acc.stepErrorSource + ? { stepError: { source: acc.stepErrorSource } } + : {}), // Keys shadow-mismatch telemetry only; never read for the verdict. ...(runId !== null ? { runId: String(runId) } : {}), // The run's FROZEN position. Threaded so a per-suite `off` is honoured on @@ -4083,6 +4091,12 @@ const runLocalIteration = async ({ // success path. const failParams = buildIterationFinishParams({ iterationId, + // Same as the success path: carry the layer when the driver could tell. + // This branch is the one a model-call failure most often ends on, so + // omitting it here would leave the fix half-applied. + ...(acc.stepErrorSource + ? { stepError: { source: acc.stepErrorSource } } + : {}), ...(runId !== null ? { runId: String(runId) } : {}), ...(gradingMode ? { gradingMode } : {}), scoreMatchOptions: scoreMatchOptionsFor(test), diff --git a/mcpjam-inspector/server/services/evals/__tests__/provider-error-attribution.test.ts b/mcpjam-inspector/server/services/evals/__tests__/provider-error-attribution.test.ts new file mode 100644 index 0000000000..8bfe2690d1 --- /dev/null +++ b/mcpjam-inspector/server/services/evals/__tests__/provider-error-attribution.test.ts @@ -0,0 +1,60 @@ +/** + * WHICH LAYER an eval trial's failure is attributed to. + * + * The provider-error work exists to stop our own model-call failures being + * filed against the server under test. Both decisions here are the places that + * work can go wrong in the OTHER direction — attributing to the model a + * failure that never reached it — which is the same defect wearing a different + * coat, and just as misleading in a report card whose job is to say whose side + * broke. + * + * Both were found in review after the first version shipped, and both are + * about a catch site being broader than the comment above it claimed. + */ +import { describe, expect, it } from "vitest"; +import { failedLayerForEngineError } from "../drive-hosted-eval-turn.js"; +import { modelLayerForErrorSpan } from "../drive-local-eval-turn.js"; + +describe("the hosted path reads the engine's reported phase", () => { + it("attributes a stream failure to the model", () => { + expect(failedLayerForEngineError({ phase: "stream" })).toBe("model"); + }); + + it("attributes a PRE-STREAM harness failure to setup, not the model", () => { + // `runHarnessTurn` wraps its whole turn in one try, so a missing + // projectId, a missing auth bearer and disabled broker credential delivery + // all arrive through the same callback a provider outage does. None of + // them reached a model, and calling them `providerError` would file our + // own setup bug as the provider's. + expect(failedLayerForEngineError({ phase: "setup" })).toBe("setup"); + }); + + it("still says model when the engine reports no phase at all", () => { + // The compatibility floor. Every emitter that omits a phase today is a + // real stream failure, and defaulting the other way would un-attribute + // the outages this work was built for. + expect(failedLayerForEngineError({})).toBe("model"); + expect(failedLayerForEngineError(undefined)).toBe("model"); + }); +}); + +describe("the local path reads the error span's own category", () => { + it("attributes an llm span to the model", () => { + expect(modelLayerForErrorSpan({ category: "llm" })).toBe("model"); + }); + + it.each([["connection"], ["discovery"], ["oauth"], ["step"], ["execution"]])( + "attributes NOTHING to a %s span", + (category) => { + // The branch that finds these selects every non-tool error span. A + // server we could not connect to is not a provider outage, and saying so + // would blame the wrong side — the exact failure this work removes. + expect(modelLayerForErrorSpan({ category })).toBeUndefined(); + }, + ); + + it("attributes nothing to a span with no category", () => { + expect(modelLayerForErrorSpan({})).toBeUndefined(); + expect(modelLayerForErrorSpan(undefined)).toBeUndefined(); + }); +}); diff --git a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts index 43e48d26ad..bfc8e22d05 100644 --- a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts +++ b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts @@ -299,6 +299,29 @@ const truncateError = (message: string): string => * forever. Consumed by the executor (R3); this module only exports the value. */ export const MAX_WIDGET_FOLLOWUP_TURNS = 3; + +/** + * Which layer failed, from what the engine REPORTED rather than from where + * this was called. + * + * The first version of this decision lived inline and assumed every engine + * error was a stream failure. That holds for the chat engine and not for the + * harness: `runHarnessTurn` wraps its whole turn — preparation included — in + * one try, so a missing `projectId`, a missing auth bearer, or disabled broker + * credential delivery arrives looking exactly like a provider outage. Calling + * those `model` files our own setup bug as the provider's, which is the + * mis-attribution this work exists to remove, moved one layer over. + * + * `model` stays the answer for an engine that reports no phase at all: every + * emitter that omits it today is a real stream failure, and defaulting the + * other way would un-attribute the outages this work was built for. + */ +export function failedLayerForEngineError( + event: { phase?: "setup" | "stream" } | undefined, +): "model" | "setup" { + return event?.phase === "setup" ? "setup" : "model"; +} + export async function driveHostedEvalTurn( params: DriveHostedEvalTurnParams, ): Promise { @@ -684,14 +707,28 @@ export async function driveHostedEvalTurn( : { iterationError: fallbackError }; logger.error(logLine); sinks.onTurnFailure?.(failure); - // Every path through here is the engine's stream failing, so the layer is - // known without inspecting anything. The structured code and status ride - // along when the engine captured them — they are diagnostics, never the - // basis for the classification. + // WHICH LAYER, from the engine's own report rather than from this call + // site's position. + // + // The first version of this said "every path through here is the engine's + // stream failing". That is true of the chat engine and false of the + // HARNESS: `runHarnessTurn` wraps its entire turn — preparation included — + // in one try, so a missing projectId, a missing auth bearer or disabled + // broker credential delivery arrives here exactly like a provider outage. + // Calling those `model` would file our own setup bug as the provider's, + // which is the mis-attribution this whole change exists to remove, just + // moved one layer over. + // + // So the engine's `phase` decides when it is reported, and `model` remains + // the default only for emitters that do not report one — every such + // emitter today is a real stream failure. + const failedLayer = failedLayerForEngineError(lastEngineError); + // The structured code and status ride along when the engine captured them + // — they are diagnostics, never the basis for the classification. return { kind: "failed" as const, ...failure, - errorSource: "model" as const, + errorSource: failedLayer, ...(lastEngineError?.code ? { errorCode: lastEngineError.code } : {}), ...(typeof lastEngineError?.httpStatus === "number" ? { errorHttpStatus: lastEngineError.httpStatus } diff --git a/mcpjam-inspector/server/services/evals/drive-local-eval-turn.ts b/mcpjam-inspector/server/services/evals/drive-local-eval-turn.ts index 1791b85e9b..cd3bc428c1 100644 --- a/mcpjam-inspector/server/services/evals/drive-local-eval-turn.ts +++ b/mcpjam-inspector/server/services/evals/drive-local-eval-turn.ts @@ -45,9 +45,41 @@ export type LocalEvalTurnAcc = { activeTraceCtx: ReturnType | null; iterationError: string | undefined; iterationErrorDetails: string | undefined; + /** + * WHICH LAYER failed, when this driver can tell. + * + * The hosted path carries this from its catch site; the local one had no + * equivalent, so a local-BYOK trial that died on the model call finalized + * with blank stage reasons and no failure category — the exact + * mis-attribution the provider-error work exists to remove, surviving on the + * path the hosted fix never touched. + * + * Left UNSET whenever the layer is not structurally knowable. An absent + * source changes nothing downstream, which is the right floor: no + * attribution beats a wrong one. + */ + stepErrorSource: "model" | undefined; pinnedSetupFailure: boolean; }; +/** + * Whether an error span means the MODEL-CALL layer failed. + * + * The branch that finds these spans selects every non-tool error span, and + * `connection`, `discovery` and `oauth` spans are all reachable there — so + * treating the whole set as the model would blame the provider for a server we + * could not reach, which is the mis-attribution this work removes rather than + * relocates. + * + * `undefined` for everything else, deliberately. An absent source attributes + * nothing, and no attribution is strictly better than a confident wrong one. + */ +export function modelLayerForErrorSpan( + span: { category?: string } | undefined, +): "model" | undefined { + return span?.category === "llm" ? "model" : undefined; +} + export type LocalEvalTurnOutcome = | { kind: "completed" } | { kind: "cancelled" }; @@ -391,6 +423,9 @@ export async function driveLocalEvalTurn( if (promptResponseMessages.length === 0) { acc.iterationError = "Stream returned no content (local-BYOK driver failed)"; + // The model stream itself returned nothing. Unambiguously the model-call + // layer — there is no other layer this branch can be reached from. + acc.stepErrorSource = "model"; logger.error( "[evals] streamText returned no new messages this turn; treating as cycle failure" ); @@ -426,6 +461,7 @@ export async function driveLocalEvalTurn( ); if (stepErrorSpan) { acc.iterationError = `Local-BYOK step failed mid-turn: ${stepErrorSpan.name}`; + acc.stepErrorSource = modelLayerForErrorSpan(stepErrorSpan); logger.error( `[evals] streamText recorded non-tool error span; treating as cycle failure (span=${stepErrorSpan.name} category=${stepErrorSpan.category})` ); diff --git a/mcpjam-inspector/server/utils/harness/run-harness-turn.ts b/mcpjam-inspector/server/utils/harness/run-harness-turn.ts index ac9869f64b..879c14be37 100644 --- a/mcpjam-inspector/server/utils/harness/run-harness-turn.ts +++ b/mcpjam-inspector/server/utils/harness/run-harness-turn.ts @@ -2767,6 +2767,12 @@ export async function runHarnessTurn( message: errorText, rawText: errorText, promptIndex, + // This catch covers the WHOLE turn, preparation included: the throws + // for a missing projectId, a missing auth bearer and disabled broker + // delivery all land here, and none of them ever reached a model. The + // trace-started flag used just above to decide whether a turn happened + // at all is the same signal for whose layer failed. + phase: driver?.traceStarted ? "stream" : "setup", }); } finally { stopScopeStepUpBridge(); diff --git a/mcpjam-inspector/server/utils/mcpjam-stream-handler.ts b/mcpjam-inspector/server/utils/mcpjam-stream-handler.ts index 1920dd67ad..b8e3ca8a06 100644 --- a/mcpjam-inspector/server/utils/mcpjam-stream-handler.ts +++ b/mcpjam-inspector/server/utils/mcpjam-stream-handler.ts @@ -422,6 +422,22 @@ export interface MCPJamEngineErrorEvent { promptIndex: number; /** Step index when fired inside `processOneStep`; omitted for site (3). */ stepIndex?: number; + /** + * WHICH LAYER was running when this fired — not what the message says. + * + * `"setup"` means the turn died before the model stream began: the harness + * catches its own pre-stream preparation (no `projectId`, no auth bearer, + * broker credential delivery disabled, a sandbox it could not reserve) in + * the same block that catches a stream failure, and reports both here. + * A consumer that assumed every engine error was a provider failure would + * file our own setup bug as the provider's outage. + * + * Derived from a flag the emitter already holds — whether the turn's trace + * ever started — never from reading the message. Omitted by emitters that + * cannot distinguish the two, and a consumer must treat that as unknown + * rather than as either answer. + */ + phase?: "setup" | "stream"; /** * Classified form of this failure, including its `origin` — whose fault the * turn dying was. From f9817f1b3b4b921adcb14fdd92270381662fd82c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 22:01:20 +0000 Subject: [PATCH 09/15] UVH-IN2 review 2: withdraw the failures a provider outage made unknowable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two further findings. The first is the one that mattered: the fix was INERT on the shape it exists for. A MISSING TOOL CALL IS NOT A SELECTION DEFECT WHEN THE PROVIDER NEVER LET US MAKE IT. applyProviderError re-labelled only rows that measured nothing — but a case expecting a tool call whose provider died already carries `selection: failed / missingToolCall`, written by the matcher before the chain is derived. So firstFailedStage stayed `selection`, categoryFor returned `selection`, and the outage was filed as a model-selection defect: the exact misattribution this reason was built to remove, on the commonest case in the corpus. The line the fix turns on: - An ABSENCE verdict (no call arrived, an assertion over output did not hold, the judge scored a truncated transcript low) is only sound if the run was allowed to finish. When our own model call died first, "it did not happen" has a second explanation that outranks the accusation and we cannot tell which is true. Those rows become notMeasured/providerError, and their evidence goes with the verdict it supported — a notMeasured row must not still argue for a failure it no longer claims. - A PRESENCE verdict stands: an unexpected call was really made, arguments really mismatched, a tool really errored, a render really failed. connectFailed and toolsListFailed matter most — they happen BEFORE any model call, so a server that would not connect is never excused by a provider error that came later. Laundering a genuine server defect would be the worse bug of the two. THE PHASE FLAG WAS READ TOO LATE. It used driver.traceStarted, and the driver is only built after agent.stream(...) RESOLVES — so an immediate provider rejection (auth, quota, a rate limit) read as our setup failing when the model had in fact been asked. The flag is now set immediately before the call, so it marks the handover rather than a successful one. Mutation-checked in both directions: never withdrawing a failed row reproduces the original bug, withdrawing every failed row launders the server defects, and keeping stale evidence leaves a withdrawn row arguing for its old verdict. SDK 6894 passed; CLI gate suite 1182 passed, 0 failed; harness + evals 1079. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- ...evals-provider-error-withdraws-absences.md | 19 ++++ .../server/utils/harness/run-harness-turn.ts | 26 +++++- sdk/src/contract/stage-derivation.ts | 93 +++++++++++++++---- sdk/tests/stage-derivation.test.ts | 70 ++++++++++++++ 4 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 .changeset/evals-provider-error-withdraws-absences.md diff --git a/.changeset/evals-provider-error-withdraws-absences.md b/.changeset/evals-provider-error-withdraws-absences.md new file mode 100644 index 0000000000..f5d0bb34a9 --- /dev/null +++ b/.changeset/evals-provider-error-withdraws-absences.md @@ -0,0 +1,19 @@ +--- +"@mcpjam/sdk": patch +"@mcpjam/inspector": patch +--- + +A provider outage withdraws the failures it made unknowable, and is dated from the handover + +Two further review findings, and the first is the one that mattered most: the provider-error fix was **inert on the shape it exists for**. + +**A missing tool call is not a selection defect when the provider never let us make it.** `applyProviderError` re-labelled only rows that measured *nothing*. But a case expecting a tool call whose provider died already has `selection: failed / missingToolCall` written by the matcher before the chain is derived — so `firstFailedStage` stayed `selection`, `categoryFor` returned `selection`, and the outage was filed as a model-selection defect. The exact misattribution this reason was built to remove, on the commonest case in the corpus. + +The fix turns on a distinction worth stating plainly: + +- An **absence** verdict — no call arrived, an assertion over the output did not hold, the judge scored a truncated transcript low — is only sound if the run was allowed to finish. When our own model call died first, "it did not happen" has a second explanation that outranks the accusation, and we cannot tell which is true. Those rows become `notMeasured / providerError`, and their evidence goes with the verdict it supported. +- A **presence** verdict stands: an unexpected call was really made, arguments really mismatched, a tool really errored, a render really failed. `connectFailed` and `toolsListFailed` matter most here — they happen *before* any model call, so a server that would not connect is never excused by a provider error that came later. Letting a provider blip launder a genuine server defect would be the worse bug of the two. + +**The phase flag was read too late.** It used `driver.traceStarted`, and the driver is only built after `agent.stream(...)` *resolves* — so an immediate provider rejection (auth, quota, a rate limit) was reported as our setup failing when the model had in fact been asked. The flag is now set immediately before the call, marking the handover itself rather than a successful one. + +Mutation-checked in both directions: never withdrawing a failed row reproduces the original bug; withdrawing every failed row launders the server defects; and keeping stale evidence on a withdrawn row leaves it arguing for a failure it no longer claims. SDK 6,894 passed; CLI gate suite 1,182 passed, 0 failed; server harness + evals 1,079 passed. diff --git a/mcpjam-inspector/server/utils/harness/run-harness-turn.ts b/mcpjam-inspector/server/utils/harness/run-harness-turn.ts index 879c14be37..f2e185f8a2 100644 --- a/mcpjam-inspector/server/utils/harness/run-harness-turn.ts +++ b/mcpjam-inspector/server/utils/harness/run-harness-turn.ts @@ -725,6 +725,19 @@ export async function runHarnessTurn( // PersistedTurnTrace / abort). Constructed at STREAM start once `traceBaseMs` // is finalized; read by `onFinishEngine` (a sibling closure) for the trace. let driver: StreamTurnDriver | undefined; + /** + * Whether the model request was HANDED OVER — the boundary between our own + * preparation and the provider's turn. + * + * Not `driver.traceStarted`, which was the first version of this and is set + * far too late: the driver is only built after `agent.stream(...)` RESOLVES, + * so an immediate provider rejection (auth, quota, a rate limit) would be + * reported as our setup failing when the model had in fact been asked. + * + * Set immediately before the call instead, so the flag marks the handover + * itself rather than a successful one. + */ + let modelInvoked = false; const executeEngine = async ({ writer }: { writer: ChunkWriter }) => { onStreamWriterReady?.(writer); @@ -1862,6 +1875,9 @@ export async function runHarnessTurn( // WS3: a resume carries no new user prompt — feed the approval decision // into the in-flight turn via continueStream (the adapter collapses // `messages` to the last user message, so stream() would re-prompt). + // Everything above this line is ours; everything at or below it is the + // model's turn. See `modelInvoked`. + modelInvoked = true; const res = resumeFromApproval ? await agent.continueStream({ session, @@ -2769,10 +2785,12 @@ export async function runHarnessTurn( promptIndex, // This catch covers the WHOLE turn, preparation included: the throws // for a missing projectId, a missing auth bearer and disabled broker - // delivery all land here, and none of them ever reached a model. The - // trace-started flag used just above to decide whether a turn happened - // at all is the same signal for whose layer failed. - phase: driver?.traceStarted ? "stream" : "setup", + // delivery all land here, and none of them ever reached a model. + // + // The flag is set at the HANDOVER, not once the stream is running, so + // a provider that rejects the request outright still reads as the + // model's failure rather than ours. + phase: modelInvoked ? "stream" : "setup", }); } finally { stopScopeStepUpBridge(); diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index 1eff7674f5..d25f96c0e1 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -1368,14 +1368,59 @@ function mergeMetadataAttributionEvidence( } /** - * Re-label the stages a MODEL-CALL failure left blank. + * Reasons a stage reported NOTHING, which a model-call failure explains. * - * Applied last, and only to rows that measured nothing: a stage with its own - * evidence keeps its own row, because the provider dying at turn 4 does not - * un-observe what turns 1-3 established. What it replaces is the bare - * `noEvidenceCaptured` / `traceAbsent` gap, which reads as "we looked and the - * server told us nothing" — an accusation, when the truth is that our own - * provider never let us ask. + * Each reads as "we looked and the server told us nothing" — an accusation, + * when the truth is that our own provider never let us ask. + */ +const PROVIDER_BLANKED_REASONS: ReadonlyArray = [ + "noEvidenceCaptured", + "traceAbsent", + "executorEmitsNoSpans", +]; + +/** + * Failure reasons that conclude from something NOT HAPPENING. + * + * This is the distinction that decides whether a `failed` row survives a + * provider outage, and it is the whole of the second half of this function. + * + * An ABSENCE verdict — no tool call arrived, an assertion over the output did + * not hold, the judge scored a transcript low — is only sound if the run was + * allowed to finish. When our own model call died first, "it did not happen" + * has a second explanation that outranks the accusation, and we cannot tell + * which is true. The honest answer is that we did not measure it. + * + * PRESENCE verdicts are deliberately absent from this list and keep their + * rows: an unexpected call was really made, arguments really mismatched, a + * tool really errored, a render really failed. Those observations stand + * whatever killed the turn afterwards. `connectFailed` and `toolsListFailed` + * matter most here — they happen BEFORE any model call, so a server that would + * not connect must never be excused by a provider error that came later. + */ +const PROVIDER_UNKNOWABLE_FAILURES: ReadonlyArray = [ + "missingToolCall", + "predicateFailed", + "judgePartial", + "judgeFailed", +]; + +/** + * Re-label what a MODEL-CALL failure made unknowable. + * + * Applied last, and in two parts. + * + * BLANK ROWS. A stage that measured nothing is re-labelled, while a stage with + * its own evidence keeps its own row: the provider dying at turn 4 does not + * un-observe what turns 1-3 established. + * + * FAILED ROWS THAT REST ON AN ABSENCE. The first version of this stopped at + * blank rows, and that left the fix inert on the shape it matters most for. A + * case expecting a tool call whose provider died first still had + * `selection: failed / missingToolCall` written by the matcher before this + * ever ran — so `firstFailedStage` stayed `selection`, `categoryFor` returned + * `selection`, and the outage was filed as a model-selection defect. The one + * thing this reason exists to prevent, on the most common case in the corpus. * * `notMeasured` throughout, never `failed`. A run that could not be attempted * has measured nothing about the server, and inflating a server failure rate @@ -1386,16 +1431,30 @@ function applyProviderError( evidence: StageEvidence ): StageResultRow[] { if (evidence.stepError?.source !== "model") return rows; - const BLANK: ReadonlyArray = [ - "noEvidenceCaptured", - "traceAbsent", - "executorEmitsNoSpans", - ]; - return rows.map((r) => - r.state === "notMeasured" && BLANK.includes(r.reason) - ? { ...r, reason: "providerError" as const } - : r - ); + return rows.map((r) => { + if ( + r.state === "notMeasured" && + PROVIDER_BLANKED_REASONS.includes(r.reason) + ) { + return { ...r, reason: "providerError" as const }; + } + if ( + r.state === "failed" && + PROVIDER_UNKNOWABLE_FAILURES.includes(r.reason) + ) { + // The EVIDENCE goes with the verdict it supported. Those lines say why + // the absence was judged a failure, and that judgement is exactly what + // is being withdrawn — keeping them would leave a `notMeasured` row + // arguing for a failure it no longer claims. + const { evidence: _dropped, ...rest } = r; + return { + ...rest, + state: "notMeasured" as const, + reason: "providerError" as const, + }; + } + return r; + }); } function finalize( diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index baeb59510c..ec914cb1ed 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -821,6 +821,76 @@ describe("a model-call failure is attributed, not left blank", () => { expect(stateOf(stageResults, "selection").state).toBe("passed"); }); + test("a missing call the provider never let us make is not a selection defect", () => { + // THE CASE THIS REASON EXISTS FOR, and the one the first version missed. + // + // A case expecting a tool call whose provider died has + // `selection: failed / missingToolCall` written by the matcher before the + // chain is derived. Re-labelling only BLANK rows left that standing, so + // `firstFailedStage` stayed `selection` and the outage was filed as a + // model-selection defect — the exact misattribution this whole reason was + // built to remove, on the commonest shape in the corpus. + const { stageResults, failureCategory, firstFailedStage } = derive({ + evidence: { + prompts: [{ promptIndex: 0, missing: [{ toolName: "search" }] }], + ...providerDied, + }, + }); + + const selection = stateOf(stageResults, "selection"); + expect(selection.state).toBe("notMeasured"); + expect(selection.reason).toBe("providerError"); + // The evidence went with the verdict it supported: a notMeasured row must + // not still be arguing for a failure it no longer claims. + expect(selection.evidence).toBeUndefined(); + expect(firstFailedStage).toBeUndefined(); + expect(failureCategory).toBe("setup"); + }); + + test("a call that really was made wrongly still counts against the server", () => { + // The other side of that line, and the one that keeps this honest. An + // UNEXPECTED call was actually observed — a presence, not an absence — so + // a provider dying afterwards does not un-observe it. Withdrawing this too + // would let any provider blip launder a genuine server defect. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [ + { + promptIndex: 0, + unexpected: [{ toolName: "delete_all" }], + passed: false, + }, + ], + ...providerDied, + }, + }); + const selection = stateOf(stageResults, "selection"); + expect(selection.state).toBe("failed"); + expect(selection.reason).toBe("unexpectedToolCall"); + }); + + test("a server that would not connect is never excused by a later outage", () => { + // `connection` fails BEFORE any model call, so a provider error that came + // afterwards cannot explain it. This is the failure mode that would be + // most damaging to launder away. + const { stageResults, firstFailedStage } = derive({ + evidence: { + setupSignals: { + connection: { + outcome: "failed", + attribution: "theirs", + egressVerified: true, + spanIds: ["run-connect-s1"], + }, + }, + ...providerDied, + }, + }); + expect(stateOf(stageResults, "connection").state).toBe("failed"); + expect(firstFailedStage).toBe("connection"); + }); + test("a SETUP-layer error is not a provider error", () => { // Pre-turn setup never reached the model, and `setupAborted` already says // so precisely. Widening `providerError` over it would lose that. From 0440ac3e9b439cc6f8bc225245490d1bd2a1f4e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 22:28:24 +0000 Subject: [PATCH 10/15] UVH-IN2 review 3: withdraw before the cascade, and scope the reason's label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups on the withdrawal added last round. Both are defects I introduced with that fix. THE CASCADE RAN BEFORE THE WITHDRAWAL. The positional pass reads `failed` rows to decide which later stages "never ran". Withdrawing a provider-blocked failure AFTER it had run left call, response and userValue still saying `earlierStageFailed` while no stage failed and no firstFailedStage existed — three rows citing a failure the chain no longer records. A chain that argues with itself is worse than either reading alone. The withdrawal now happens first, so the cascade sees the rows as they will actually be reported: a provider outage marks the later stages `providerError`, which is WHY they were not measured, rather than blaming a stage that is no longer failed. A failure the provider did NOT explain still cascades exactly as before — `unexpectedToolCall` survives the withdrawal, stays the first failed row, and the stages after it still read `notReached`. Repaired, not disabled. `applyProviderError` stays in `finalize` as well, because the early-return paths never reach the cascade; it is idempotent, so the second pass over already-converted rows finds nothing to change. THE REASON'S LABEL SPOKE FOR THE RUN. It read "the model provider failed the call, so the run never reached the server" — but the reason is applied PER ROW, so a multi-turn iteration whose provider died at turn 4 keeps its earlier measured rows, and that claim would sit directly beside a `call: passed` that disproves it. It now says "so this stage was never measured", which is true of the row it labels. Mutation-checked: restoring the old ordering reproduces the self-contradicting chain, and restoring the run-level wording fails the label's scope test. SDK 6896 passed; CLI gate suite 1182 passed, 0 failed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-provider-error-chain-consistency.md | 15 ++++ sdk/src/contract/decision-labels.ts | 7 +- sdk/src/contract/stage-derivation.ts | 26 ++++++- sdk/tests/stage-derivation.test.ts | 72 +++++++++++++++++++ 4 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 .changeset/evals-provider-error-chain-consistency.md diff --git a/.changeset/evals-provider-error-chain-consistency.md b/.changeset/evals-provider-error-chain-consistency.md new file mode 100644 index 0000000000..ff18aad909 --- /dev/null +++ b/.changeset/evals-provider-error-chain-consistency.md @@ -0,0 +1,15 @@ +--- +"@mcpjam/sdk": patch +--- + +Withdrawing a provider-blocked failure no longer leaves the chain arguing with itself + +Two follow-ups on the withdrawal added in the previous round. + +**The cascade ran before the withdrawal.** The positional pass reads `failed` rows to decide which later stages "never ran". Withdrawing a provider-blocked failure *after* it had run left `call`, `response` and `userValue` still saying `earlierStageFailed` while no stage failed and no `firstFailedStage` existed — three rows citing a failure the chain no longer records. + +The withdrawal now happens first, so the cascade sees the rows as they will actually be reported: a provider outage marks the later stages `providerError`, which is *why* they were not measured, rather than blaming a stage that is no longer failed. A failure the provider did **not** explain still cascades exactly as before — `unexpectedToolCall` survives the withdrawal, stays the first failed row, and the stages after it still read `notReached`. The cascade is repaired, not disabled. + +**The reason's label spoke for the run.** It read *"the model provider failed the call, so the run never reached the server"* — but `providerError` is applied per row, so a multi-turn iteration whose provider died at turn 4 keeps its earlier measured rows. The run-level claim would sit directly beside a `call: passed` that disproves it, and send a reader after the wrong timeline. It now says *"…so this stage was never measured"*, which is true of the row it labels. + +Mutation-checked: restoring the old ordering reproduces the self-contradicting chain, and restoring the run-level wording fails the label's scope test. diff --git a/sdk/src/contract/decision-labels.ts b/sdk/src/contract/decision-labels.ts index 56ef074076..4684e49f05 100644 --- a/sdk/src/contract/decision-labels.ts +++ b/sdk/src/contract/decision-labels.ts @@ -102,8 +102,13 @@ export const STAGE_REASON_LABELS = Object.freeze({ blockedByPolicy: "a policy blocked the run before it could be measured", evaluatorError: "the evaluator itself failed, so the run says nothing about the server", + // Scoped to THIS STAGE, not to the run. `providerError` is applied per row, + // and a multi-turn iteration whose provider died at turn 4 keeps its earlier + // measured rows — so a run-level "never reached the server" would sit + // directly beside a `call: passed` that disproves it, and send a reader + // after the wrong timeline. providerError: - "the model provider failed the call, so the run never reached the server", + "the model provider failed the call, so this stage was never measured", setupAborted: "the environment was never prepared, so the test never began", connectFailed: "the configured server was reached and initialize failed there", diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index d25f96c0e1..0a5bc4b5b4 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -1322,11 +1322,27 @@ export function deriveStageResults( // those measured rows with "never ran" destroys the evidence an operator // needs and states something the run disproves. `firstFailedStage` already // carries "where the chain broke" — the rows do not have to lie to say it. - const firstFailedIndex = derived.findIndex((r) => r.state === "failed"); + // WITHDRAWN FIRST, then cascaded — the order matters and getting it wrong + // produces a chain that argues with itself. + // + // The cascade reads `failed` rows to decide which later stages "never ran". + // Withdrawing a provider-blocked failure AFTER it has run leaves the + // downstream rows still saying `earlierStageFailed` while no stage failed + // and no `firstFailedStage` exists — three rows citing a failure the chain + // no longer records. Running the withdrawal first means the cascade sees the + // rows as they will actually be reported, so a provider outage marks the + // later stages `providerError` (which is why they were not measured) rather + // than blaming a stage that is no longer failed. + // + // A failure the provider did NOT explain still cascades exactly as before: + // `unexpectedToolCall` at `selection` survives the withdrawal, stays the + // first failed row, and the stages after it still read `notReached`. + const withdrawn = applyProviderError(derived, evidence); + const firstFailedIndex = withdrawn.findIndex((r) => r.state === "failed"); const rows = firstFailedIndex < 0 - ? derived - : derived.map((r, i) => + ? withdrawn + : withdrawn.map((r, i) => i > firstFailedIndex && r.state === "notMeasured" ? row(r.stage, "notReached", "earlierStageFailed") : r @@ -1462,6 +1478,10 @@ function finalize( evidence: StageEvidence, forcedCategory?: FailureCategory ): StageDerivation { + // IDEMPOTENT, and applied here as well as before the positional cascade: the + // early-return paths above never reach that cascade, so this is the only + // place they get it. A second pass over rows the first already converted + // finds nothing left to change — `providerError` is in neither list. rows = applyProviderError(rows, evidence); const firstFailedStage = rows.find((r) => r.state === "failed")?.stage; const failureCategory = diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index ec914cb1ed..3fd7d78cfb 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -12,6 +12,7 @@ */ import { describe, expect, test } from "vitest"; +import { STAGE_REASON_LABELS } from "../src/contract/decision-labels.js"; import { MAX_EVIDENCE_REASONS, MAX_EVIDENCE_REASON_CHARS, @@ -847,6 +848,77 @@ describe("a model-call failure is attributed, not left blank", () => { expect(failureCategory).toBe("setup"); }); + test("the reason speaks for its own stage, not for the run", () => { + // Review finding on the label. `providerError` is applied PER ROW, so a + // multi-turn iteration whose provider died late keeps its earlier measured + // rows — and a run-level "the run never reached the server" would sit + // directly beside a `call: passed` that disproves it. + const { stageResults } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [cleanTurn], + ...providerDied, + }, + }); + // The precondition that makes the label's scope matter: the server WAS + // reached on this run. + expect(stateOf(stageResults, "call").state).toBe("passed"); + expect(STAGE_REASON_LABELS.providerError).not.toContain( + "never reached the server" + ); + expect(STAGE_REASON_LABELS.providerError).toContain("this stage"); + }); + + test("the chain does not argue with itself after a withdrawal", () => { + // Review finding on the withdrawal itself. The positional cascade reads + // `failed` rows to decide which later stages "never ran", so withdrawing + // the failure AFTER it ran left `call`, `response` and `userValue` saying + // `earlierStageFailed` while no stage failed and no firstFailedStage + // existed — three rows citing a failure the chain no longer records. + const { stageResults, firstFailedStage } = derive({ + evidence: { + prompts: [{ promptIndex: 0, missing: [{ toolName: "search" }] }], + ...providerDied, + }, + }); + + expect(firstFailedStage).toBeUndefined(); + expect(stageResults.some((r) => r.state === "failed")).toBe(false); + // Nothing may still be blaming a stage that is no longer failed. + expect(stageResults.some((r) => r.reason === "earlierStageFailed")).toBe( + false + ); + // And the later stages say the true thing about why they are blank. + for (const stage of ["call", "response", "userValue"] as const) { + const r = stateOf(stageResults, stage); + if (r.state === "notMeasured") expect(r.reason).toBe("providerError"); + } + }); + + test("a failure the provider did NOT explain still cascades", () => { + // The other side. An unexpected call survives the withdrawal, so it stays + // the first failed row and the stages after it still read `notReached` — + // the cascade is repaired, not disabled. + const { stageResults, firstFailedStage } = derive({ + evidence: { + spans: [toolSpan()], + prompts: [ + { + promptIndex: 0, + unexpected: [{ toolName: "delete_all" }], + passed: false, + }, + ], + ...providerDied, + }, + }); + expect(firstFailedStage).toBe("selection"); + const after = stageResults.slice( + stageResults.findIndex((r) => r.stage === "selection") + 1 + ); + expect(after.some((r) => r.state === "notReached")).toBe(true); + }); + test("a call that really was made wrongly still counts against the server", () => { // The other side of that line, and the one that keeps this honest. An // UNEXPECTED call was actually observed — a presence, not an absence — so From 228d09f4943f86ec0aa2ea2112e8e30412142ce6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 18:26:43 +0000 Subject: [PATCH 11/15] =?UTF-8?q?UVH-IN2=20review=204:=20the=20attribution?= =?UTF-8?q?=20never=20reached=20the=20runner=20=E2=80=94=20three=20cut=20w?= =?UTF-8?q?ires?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Late-delivered review findings against earlier heads. Two P1s that together meant providerError NEVER FIRED on the hosted path — the path the audited Anthropic-credit trials ran on. The feature worked in every test and not at all in production. THE HOSTED BRIDGE COPIED ONLY THE MESSAGE. buildHostedStepHandlers converts a HostedEvalTurnOutcome into a StepEngineOutcome, and both sites copied iterationError/iterationErrorDetails and nothing else. drive-hosted-eval-turn classified the failure and step-executor was ready to propagate it, but the classification died one hop from where it was made. THE WIDGET FOLLOW-UP LOOP REDUCED A FULL OUTCOME TO A BARE STRING. A turn dying on the provider is the same event whether it was the prompt or a ui/message follow-up; reporting one and not the other made attribution depend on which turn the model happened to fail on. THE JUDGE SECOND PASS RE-DERIVED WITHOUT IT. stepError is transient runner state, never persisted, so the moment a judge verdict landed providerError and the setup category were dropped and the run went back to being filed against the server. Recovered from the stored chain rather than a new persisted field: providerError is written if and only if the model layer was classified as the failure, so its presence is a faithful witness of that input. Only code and httpStatus are lost, and those were never part of the classification. WHY ALL THREE SHIPPED GREEN, and the most important part of this change: every existing test builds a StageEvidence with stepError already on it and asserts the analyzer's behaviour. None exercised the plumbing that puts it there, so no test could fail when the wire was cut. This adds tests that drive the real step-handlers -> step-executor path and the real judge payload builder. Each new test was verified to reproduce the exact bug it covers: restoring any of the three breaks fails its test and nothing else. The judge test initially passed under a cut wire because it tested the decision and not its use — the same mistake, caught and fixed before commit. Server evals + harness suites: 1089 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- ...evals-provider-error-reaches-the-runner.md | 21 ++ .../evals/__tests__/judge-second-pass.test.ts | 88 ++++++++ .../__tests__/provider-error-plumbing.test.ts | 212 ++++++++++++++++++ .../services/evals/judge-second-pass.ts | 43 +++- .../server/services/evals/step-executor.ts | 31 ++- .../server/services/evals/step-handlers.ts | 30 +++ 6 files changed, 417 insertions(+), 8 deletions(-) create mode 100644 .changeset/evals-provider-error-reaches-the-runner.md create mode 100644 mcpjam-inspector/server/services/evals/__tests__/provider-error-plumbing.test.ts diff --git a/.changeset/evals-provider-error-reaches-the-runner.md b/.changeset/evals-provider-error-reaches-the-runner.md new file mode 100644 index 0000000000..cecd0ed9cb --- /dev/null +++ b/.changeset/evals-provider-error-reaches-the-runner.md @@ -0,0 +1,21 @@ +--- +"@mcpjam/inspector": patch +--- + +Provider attribution actually reaches the runner — it was cut in three places + +Three breaks in the wire between a classified provider failure and an attributed chain. Together the first two meant `providerError` **never fired on the hosted path at all** — the path the audited Anthropic-credit trials ran on. + +**The hosted bridge copied only the message.** `buildHostedStepHandlers` converts a `HostedEvalTurnOutcome` into a `StepEngineOutcome`, and both of its sites copied `iterationError` / `iterationErrorDetails` and nothing else. `drive-hosted-eval-turn` classified the failure and `step-executor` was ready to propagate it, but the classification died one hop from where it was made, so `iterationStepError` was never built and no hosted run was ever attributed. + +**The widget follow-up loop reduced a full outcome to a bare string.** A turn that dies on the provider is the same event whether it was the prompt or a `ui/message` follow-up; reporting one and not the other made attribution depend on which turn the model happened to fail on. + +**The judge second pass re-derived without it.** `stepError` is transient runner state — an input to the first derivation, never persisted — so the moment a judge verdict landed, `providerError` and the `setup` category were silently dropped and the run went back to being filed against the server. + +That last one is recovered from the stored chain rather than a new persisted field: `providerError` is written **if and only if** the model layer was classified as the failure, so its presence in `stageResults` is a faithful witness of that input. Nothing is invented. Only `code` and `httpStatus` are lost, and those were explicitly diagnostics rather than part of the classification. + +## Why all three shipped green + +Every existing provider-error test builds a `StageEvidence` with `stepError` already on it and asserts what the analyzer does with it. **None exercised the plumbing that puts it there** — so a unit test of the analyzer could not fail when the wire was cut. + +This adds tests that drive the real `step-handlers` → `step-executor` path and the real judge payload builder. Each was verified to reproduce the exact bug it covers: restoring any of the three breaks fails its test and nothing else. diff --git a/mcpjam-inspector/server/services/evals/__tests__/judge-second-pass.test.ts b/mcpjam-inspector/server/services/evals/__tests__/judge-second-pass.test.ts index 4ffb982632..9dd64efac9 100644 --- a/mcpjam-inspector/server/services/evals/__tests__/judge-second-pass.test.ts +++ b/mcpjam-inspector/server/services/evals/__tests__/judge-second-pass.test.ts @@ -12,7 +12,9 @@ import { type MetadataAttributionStageDerivationBody, } from "../judge-stage-backend.js"; import { + deriveIterationPayload, judgeEvidenceFromVerdict, + stepErrorFromStoredChain, metadataAttributionEvidenceFromVerdict, runJudgeSecondPass, type JudgeSecondPassPorts, @@ -941,3 +943,89 @@ describe("the second pass keeps its contract with the run and the first pass", ( ).toBe(true); }); }); + +describe("stepErrorFromStoredChain", () => { + // `stepError` is an INPUT to the first derivation and is never persisted, so + // the judge pass re-derived without it and dropped `providerError` and the + // `setup` category the moment a verdict landed — a run our own provider + // killed went back to being filed against the server. + + it("recovers the model layer from a chain that recorded providerError", () => { + expect( + stepErrorFromStoredChain([ + { stage: "connection", state: "passed", reason: "observed" }, + { stage: "selection", state: "notMeasured", reason: "providerError" }, + ]), + ).toEqual({ source: "model" }); + }); + + it("recovers nothing from a chain that recorded no provider failure", () => { + // The witness has to be the reason itself. Inferring a provider error from + // any other blank row would re-introduce the guess this reason removed. + expect( + stepErrorFromStoredChain([ + { stage: "selection", state: "notMeasured", reason: "noEvidenceCaptured" }, + { stage: "userValue", state: "failed", reason: "predicateFailed" }, + ]), + ).toBeUndefined(); + }); + + it("recovers nothing when there is no chain at all", () => { + expect(stepErrorFromStoredChain(undefined)).toBeUndefined(); + expect(stepErrorFromStoredChain([])).toBeUndefined(); + expect(stepErrorFromStoredChain("not an array")).toBeUndefined(); + }); +}); + +describe("the recovery is WIRED into the re-derivation", () => { + // Testing `stepErrorFromStoredChain` alone cannot fail when the wire is cut, + // and a cut wire is exactly how this attribution kept getting lost. So this + // drives the real payload builder and asserts the re-derived chain still + // says the model layer failed. + // + // The fixture is the SHAPE a provider failure actually leaves: the server was + // reached (spans exist, so connection and discovery are implied) and then our + // model call died, leaving the stages after it blank. An iteration with no + // evidence at all derives to `setupAborted`, which is a more specific reason + // and correctly outranks `providerError` — so it would prove nothing here. + const reachedThenDied = { + status: "failed", + traceComplete: true, + stageCase: { + mode: "model_driven", + expectsToolCall: false, + expectsWidgetRender: false, + assertionCount: 0, + }, + spans: [{ id: "s1", name: "tools/call", category: "tool", status: "ok" }], + }; + + const derive = (stageResults: unknown) => + deriveIterationPayload({ + iteration: { ...reachedThenDied, metadata: { stageResults } }, + mode: "advisory", + judgeVerdict: { status: "scored", verdict: "fail", score: 0.1 }, + attributionVerdict: undefined, + } as never).stage; + + const reasons = (stage: Record) => + (stage.stageResults as { reason: string }[]).map((r) => r.reason); + + it("keeps providerError and the setup category through a judge pass", () => { + const stage = derive([ + { stage: "selection", state: "notMeasured", reason: "providerError" }, + ]); + expect(reasons(stage)).toContain("providerError"); + expect(stage.failureCategory).toBe("setup"); + }); + + it("does not invent a provider failure on a run that had none", () => { + // The same evidence, a stored chain that never blamed the provider. The + // blank stage stays blank rather than acquiring an attribution the first + // derivation did not make. + const stage = derive([ + { stage: "selection", state: "notMeasured", reason: "noEvidenceCaptured" }, + ]); + expect(reasons(stage)).not.toContain("providerError"); + }); +}); diff --git a/mcpjam-inspector/server/services/evals/__tests__/provider-error-plumbing.test.ts b/mcpjam-inspector/server/services/evals/__tests__/provider-error-plumbing.test.ts new file mode 100644 index 0000000000..88ef3c3cfe --- /dev/null +++ b/mcpjam-inspector/server/services/evals/__tests__/provider-error-plumbing.test.ts @@ -0,0 +1,212 @@ +/** + * The WIRE from a classified provider failure to an attributed chain. + * + * Every other provider-error test builds a `StageEvidence` with `stepError` + * already on it and asserts what the analyzer does with it. None of them + * exercises the plumbing that is supposed to PUT it there — and that gap is why + * four independent breaks in that plumbing all shipped green: + * + * - the hosted bridge in `step-handlers` copied only the error message; + * - the widget follow-up loop reduced a full outcome to a bare string; + * - the local driver carried no source at all; + * - the judge second pass re-derived without it. + * + * So these tests run the REAL `executeSteps` over real handler outcomes and + * assert the attribution survives each hop. A unit test of the analyzer cannot + * fail when the wire is cut; these can. + */ +import { describe, it, expect, vi } from "vitest"; +import type { TestStep } from "@/shared/steps"; + +// Mocked so the HOSTED BRIDGE itself is under test: `buildHostedStepHandlers` +// converts a `HostedEvalTurnOutcome` into a `StepEngineOutcome`, and that +// conversion is where the attribution was being dropped. +const { driveHostedEvalTurnMock } = vi.hoisted(() => ({ + driveHostedEvalTurnMock: vi.fn(), +})); +vi.mock("../drive-hosted-eval-turn", async () => { + const actual = await vi.importActual< + typeof import("../drive-hosted-eval-turn") + >("../drive-hosted-eval-turn"); + return { ...actual, driveHostedEvalTurn: driveHostedEvalTurnMock }; +}); + +import { + createStepExecutionState, + executeSteps, + type StepEngineOutcome, + type StepExecutorHandlers, +} from "../step-executor"; +import { buildHostedStepHandlers } from "../step-handlers"; +import type { BrowserSessionContext } from "../../browser-session-context"; + +type BrowserMock = Pick< + BrowserSessionContext, + | "replayInteractStep" + | "evaluateWidgetAssertion" + | "setKeepWidgetsMountedForSteps" + | "setActivePromptIndex" + | "setActiveAuthoredStepId" + | "widgetRenderObservations" + | "drainFollowUps" +>; + +function makeBrowser(overrides: Partial = {}): BrowserMock { + return { + setActivePromptIndex: vi.fn(), + setActiveAuthoredStepId: vi.fn(), + setKeepWidgetsMountedForSteps: vi.fn(), + replayInteractStep: vi.fn(async () => ({ ok: true })), + evaluateWidgetAssertion: vi.fn(async () => ({ ok: true })), + widgetRenderObservations: [], + drainFollowUps: vi.fn(() => []), + ...overrides, + }; +} + +/** What a hosted turn returns when our own model call died. */ +const PROVIDER_DIED: StepEngineOutcome = { + iterationError: "credit balance too low", + iterationErrorDetails: "Anthropic API", + errorSource: "model", + errorCode: "billing_limit_reached", + errorHttpStatus: 429, +}; + +const ONE_PROMPT: TestStep[] = [ + { id: "p", kind: "prompt", prompt: "Find my order" }, +]; + +function run( + steps: TestStep[], + handlers: Partial, + browser = makeBrowser(), +) { + return executeSteps({ + steps, + state: createStepExecutionState(), + browser: browser as unknown as BrowserSessionContext, + handlers: { + onPrompt: vi.fn(async () => ({}) as StepEngineOutcome), + onToolCall: vi.fn(async () => ({}) as StepEngineOutcome), + ...handlers, + } as StepExecutorHandlers, + }); +} + +describe("a classified provider failure reaches the runner", () => { + it("carries source, code and status off a failed prompt turn", async () => { + // The main hosted path, and the one the audited Anthropic-credit trials + // ran on. Before this the executor received only `iterationError`, so + // `iterationStepError` was never built and NO hosted run was attributed. + const result = await run(ONE_PROMPT, { + onPrompt: vi.fn(async () => PROVIDER_DIED), + }); + + expect(result.iterationError).toBe("credit balance too low"); + expect(result.errorSource).toBe("model"); + // Diagnostics ride along; they are never the basis for the classification. + expect(result.errorCode).toBe("billing_limit_reached"); + expect(result.errorHttpStatus).toBe(429); + }); + + it("carries the same attribution off a widget FOLLOW-UP turn", async () => { + // A turn that dies on the provider is the same event whether it was the + // prompt or a `ui/message` follow-up. Reporting one and not the other made + // the attribution depend on which turn the model happened to fail on. + const browser = makeBrowser({ + drainFollowUps: vi + .fn() + .mockReturnValueOnce(["add the red one"]) + .mockReturnValue([]), + }); + + const result = await run( + [ + { id: "p", kind: "prompt", prompt: "Show me a redbull" }, + { + id: "i", + kind: "interact", + toolName: "search-products", + action: { kind: "click", target: { text: "🛒" } }, + }, + ], + { + onPrompt: vi.fn(async () => ({}) as StepEngineOutcome), + onFollowUp: vi.fn(async () => PROVIDER_DIED), + }, + browser, + ); + + expect(result.iterationError).toBe("credit balance too low"); + expect(result.errorSource).toBe("model"); + expect(result.errorHttpStatus).toBe(429); + }); + + it("says nothing when the engine classified nothing", async () => { + // The compatibility floor, end to end. A handler that cannot name the + // layer must leave the source absent rather than have the wire invent one. + const result = await run(ONE_PROMPT, { + onPrompt: vi.fn(async () => ({ + iterationError: "something went wrong", + })), + }); + + expect(result.iterationError).toBe("something went wrong"); + expect(result.errorSource).toBeUndefined(); + expect(result.errorCode).toBeUndefined(); + }); + + it("says nothing about a turn that did not fail", async () => { + const result = await run(ONE_PROMPT, { + onPrompt: vi.fn(async () => ({}) as StepEngineOutcome), + }); + expect(result.iterationError).toBeUndefined(); + expect(result.errorSource).toBeUndefined(); + }); +}); + +describe("the hosted bridge does not drop what the engine classified", () => { + it("converts a classified turn outcome into an attributed step outcome", async () => { + // THE BREAK THIS FILE EXISTS FOR. `driveHostedEvalTurn` classifies the + // failure; `buildHostedStepHandlers` converts its outcome for the executor. + // That conversion copied only the message, so the classification died one + // hop from where it was made and every hosted run went unattributed. + driveHostedEvalTurnMock.mockResolvedValue({ + kind: "failed", + iterationError: "credit balance too low", + iterationErrorDetails: "Anthropic API", + errorSource: "model", + errorCode: "billing_limit_reached", + errorHttpStatus: 429, + }); + + const handlers = buildHostedStepHandlers({ + acc: { + messageHistory: [], + capturedSpans: [], + accumulatedUsage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + toolsCalledByPrompt: [], + assistantMessageByPrompt: [], + toolErrorsByPrompt: [], + pinnedToolErrors: [], + }, + browser: { + setActivePromptIndex: vi.fn(), + setActiveWidgetChecks: vi.fn(), + dismissCarriedWidget: vi.fn(async () => {}), + }, + } as never); + + const outcome = await handlers.onPrompt({ + step: { id: "p", kind: "prompt", prompt: "Find my order" }, + stepIndex: 0, + turnOrdinal: 0, + } as never); + + expect(outcome.iterationError).toBe("credit balance too low"); + expect(outcome.errorSource).toBe("model"); + expect(outcome.errorCode).toBe("billing_limit_reached"); + expect(outcome.errorHttpStatus).toBe(429); + }); +}); diff --git a/mcpjam-inspector/server/services/evals/judge-second-pass.ts b/mcpjam-inspector/server/services/evals/judge-second-pass.ts index 74492721d8..75dfe883b9 100644 --- a/mcpjam-inspector/server/services/evals/judge-second-pass.ts +++ b/mcpjam-inspector/server/services/evals/judge-second-pass.ts @@ -269,8 +269,44 @@ function isPredicateRow(value: unknown): value is StoredPredicateRow { ); } -/** Everything the derivation needs from one stored iteration. */ -function deriveIterationPayload(args: { +/** + * Recover the FIRST derivation's verdict about the model layer from the chain + * it wrote. + * + * `stepError` is transient runner state — an INPUT to that derivation, never + * persisted — so the judge second pass re-derived without it and silently + * dropped `providerError` and the `setup` category the moment a verdict + * arrived. A run our own provider killed went back to being filed against the + * server, which is exactly what that reason exists to prevent. + * + * Read from the stored chain rather than from a new persisted field, because + * the chain already says it: `providerError` is written if and only if the + * model layer was classified as the failure, so its presence is a faithful + * witness of that input. Nothing is invented here. + * + * `code` and `httpStatus` are not recoverable and are not recovered. They are + * diagnostics for a reader and were explicitly never part of the + * classification, so their absence changes no verdict. + */ +export function stepErrorFromStoredChain( + stageResults: unknown, +): { source: "model" } | undefined { + const rows = asArray(stageResults); + if (!rows) return undefined; + return rows.some((row) => asRecord(row)?.reason === "providerError") + ? { source: "model" } + : undefined; +} + +/** + * Everything the derivation needs from one stored iteration. + * + * Exported for tests: the recovery above is only useful if it is actually + * WIRED into this payload, and a test of the recovery alone cannot fail when + * the wire is cut — which is the precise gap that let four separate breaks in + * this attribution ship green. + */ +export function deriveIterationPayload(args: { iteration: JudgeSecondPassIterationRow; mode: GradingEngineMode; judgeVerdict: JudgeVerdictMetadata | undefined; @@ -331,6 +367,8 @@ function deriveIterationPayload(args: { // failed. const traceUsable = iteration.traceComplete !== false; + const recoveredStepError = stepErrorFromStoredChain(metadata.stageResults); + const stage = traceUsable ? buildStageMetadata({ ...(stageCase ? { stageCase } : {}), @@ -338,6 +376,7 @@ function deriveIterationPayload(args: { ...(iteration.prompts?.length ? { prompts: iteration.prompts } : {}), ...(iteration.messages?.length ? { messages: iteration.messages } : {}), ...(predicateRows.length ? { predicateResults: predicateRows } : {}), + ...(recoveredStepError ? { stepError: recoveredStepError } : {}), ...(iteration.toolSignals ? { toolSignals: iteration.toolSignals } : {}), ...(iteration.setupSignals ? { setupSignals: iteration.setupSignals } diff --git a/mcpjam-inspector/server/services/evals/step-executor.ts b/mcpjam-inspector/server/services/evals/step-executor.ts index d8f1efd9c3..dcdac0bca7 100644 --- a/mcpjam-inspector/server/services/evals/step-executor.ts +++ b/mcpjam-inspector/server/services/evals/step-executor.ts @@ -353,7 +353,10 @@ async function drainAndDriveFollowUps( browser: Pick, handlers: StepExecutorHandlers, state: StepExecutionState, -): Promise { + // The failing OUTCOME, not just its message. Reducing it to a string here + // discarded the layer attribution, so a provider failure on a widget + // follow-up turn stayed uncategorised even once the hosted bridge carried it. +): Promise { if (!handlers.onFollowUp) return undefined; let remaining = MAX_WIDGET_FOLLOWUP_TURNS; while (remaining > 0) { @@ -379,7 +382,7 @@ async function drainAndDriveFollowUps( turnOrdinal: turn, }); applyOutcome(state, outcome, turn); - if (outcome.iterationError) return outcome.iterationError; + if (outcome.iterationError) return outcome; } } return undefined; @@ -523,7 +526,7 @@ export async function executeSteps(args: { sIdx: number, turn: number, ): Promise => { - const err = await drainAndDriveFollowUps( + const failed = await drainAndDriveFollowUps( label, sIdx, turn, @@ -531,16 +534,32 @@ export async function executeSteps(args: { handlers, state, ); - if (!err) return undefined; + if (!failed) return undefined; emitStatus(sIdx, "fail"); recordSkippedSteps( state, steps, sIdx + 1, - `widget follow-up turn errored (step ${sIdx}): ${err}`, + `widget follow-up turn errored (step ${sIdx}): ${failed.iterationError}`, ); emitSkipped(sIdx + 1); - return { state, iterationError: err, setupFailure: false }; + // The SAME shape the prompt-step failure path returns. A follow-up turn + // dying on the provider is the same event as a prompt turn dying on it, and + // reporting one and not the other made the attribution depend on which + // turn the model happened to fail. + return { + state, + iterationError: failed.iterationError, + ...(failed.iterationErrorDetails + ? { iterationErrorDetails: failed.iterationErrorDetails } + : {}), + ...(failed.errorSource ? { errorSource: failed.errorSource } : {}), + ...(failed.errorCode ? { errorCode: failed.errorCode } : {}), + ...(typeof failed.errorHttpStatus === "number" + ? { errorHttpStatus: failed.errorHttpStatus } + : {}), + setupFailure: false, + }; }; for (let stepIndex = 0; stepIndex < steps.length; stepIndex++) { diff --git a/mcpjam-inspector/server/services/evals/step-handlers.ts b/mcpjam-inspector/server/services/evals/step-handlers.ts index cfe88325cd..bed9aded70 100644 --- a/mcpjam-inspector/server/services/evals/step-handlers.ts +++ b/mcpjam-inspector/server/services/evals/step-handlers.ts @@ -260,6 +260,21 @@ export function buildHostedStepHandlers( ...(outcome.iterationErrorDetails ? { iterationErrorDetails: outcome.iterationErrorDetails } : {}), + // WHICH LAYER failed, carried across the bridge. + // + // Dropping these was enough to make provider attribution dead on + // the hosted path entirely: `drive-hosted-eval-turn` classifies the + // failure, `step-executor` propagates whatever it is handed, and + // `evals-runner` builds `stepError` from it — but this conversion + // sat in the middle copying only the message, so `errorSource` was + // always undefined and no hosted run was ever attributed. + ...(outcome.errorSource + ? { errorSource: outcome.errorSource } + : {}), + ...(outcome.errorCode ? { errorCode: outcome.errorCode } : {}), + ...(typeof outcome.errorHttpStatus === "number" + ? { errorHttpStatus: outcome.errorHttpStatus } + : {}), } : {}), }; @@ -310,6 +325,21 @@ export function buildHostedStepHandlers( ...(outcome.iterationErrorDetails ? { iterationErrorDetails: outcome.iterationErrorDetails } : {}), + // WHICH LAYER failed, carried across the bridge. + // + // Dropping these was enough to make provider attribution dead on + // the hosted path entirely: `drive-hosted-eval-turn` classifies the + // failure, `step-executor` propagates whatever it is handed, and + // `evals-runner` builds `stepError` from it — but this conversion + // sat in the middle copying only the message, so `errorSource` was + // always undefined and no hosted run was ever attributed. + ...(outcome.errorSource + ? { errorSource: outcome.errorSource } + : {}), + ...(outcome.errorCode ? { errorCode: outcome.errorCode } : {}), + ...(typeof outcome.errorHttpStatus === "number" + ? { errorHttpStatus: outcome.errorHttpStatus } + : {}), } : {}), }; From e72f8637e80283eff46a60995c13659c36fb895b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 19:09:57 +0000 Subject: [PATCH 12/15] UVH-IN5 review: stop the dark-ship probe filing itself as a bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe is mounted on every run-detail visit, and until the rollup query is deployed `useQuery` throws every time. `ErrorBoundary.componentDidCatch` reported unconditionally, so the documented, intended dark-ship state produced one Sentry issue and one PostHog event per run VIEWED — noise indistinguishable from a real regression at exactly the moment one would matter most. `ErrorBoundary` gains an opt-in `isExpectedError` predicate. Deliberately not a boolean: reporting lives in the boundary rather than at the ~21 mount sites precisely so a `fallback={null}` cannot silently eat an error, and a boundary that suppressed everything would give that back. A predicate keeps the exception as narrow as the call site can describe. `isConvexQueryUnavailable` names the only two shapes Convex throws for a query that cannot run — an undeployed function, and no `ConvexProvider` above the tree — matched on the message because neither carries a code or a class. Anything else is a real failure and still reports. `onError` still fires and the fallback still renders, so the rail still closes; only the telemetry is suppressed. A predicate that itself throws reports as normal — fail loud, not silent, since losing errors is the one outcome worse than the noise this removes. Also extracts `runHasInsightContent`, which was an inline Boolean in `run-detail-view.tsx` with no test. It is the OUTER half of the same decision `RunInsightRail` makes: the band wraps the rail, so a rail that correctly decides to open is never seen if this says no — which the rail's own tests cannot catch, because they run inside a band that already rendered. It now sits beside the rail's gate, where the agreement between the two is written down. Mutation-checked: removing `isExpectedError` from the probe fails exactly the dark-ship test; widening the predicate to always-true fails five; dropping `hasStageFunnel` from the band gate fails the chain-only case. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-run-detail-funnel-visibility.md | 2 + .../evals/__tests__/run-insight-rail.test.tsx | 47 ++++++++++++ .../src/components/evals/run-detail-view.tsx | 15 ++-- .../src/components/evals/run-insight-rail.tsx | 41 +++++++++++ .../user-value-chain/StageFunnelPanels.tsx | 30 ++++++++ .../__tests__/StageFunnelPanels.test.tsx | 68 ++++++++++++++++++ .../ui/__tests__/error-boundary.test.tsx | 71 +++++++++++++++++++ .../src/components/ui/error-boundary.tsx | 50 +++++++++++-- 8 files changed, 312 insertions(+), 12 deletions(-) diff --git a/.changeset/evals-run-detail-funnel-visibility.md b/.changeset/evals-run-detail-funnel-visibility.md index 303df22cb6..e58ab0ff80 100644 --- a/.changeset/evals-run-detail-funnel-visibility.md +++ b/.changeset/evals-run-detail-funnel-visibility.md @@ -11,3 +11,5 @@ Adding the chain card to those checks would have traded one bug for another, whi So the gates now read a fact about the DATA instead. A probe mounted above every layout branch asks the same rollup query the funnel itself uses — `undefined` while loading, `null` for a run with no rollup, which is exactly the panel's own render condition — and reports one boolean that both gates consume. Convex de-duplicates identical subscriptions, so asking twice costs one query, and the probe carries the same `ErrorBoundary` the panels do: `useQuery` throws when the query is not deployed or when there is no `ConvexProvider`, and a probe that took the page down with it would be worse than the empty rail it exists to prevent. Undeployed reads as "no funnel", which is correct. The state starts `false`, so a run without a funnel never flashes an empty rail on the way to finding out. + +The probe's boundary does not file the dark-ship window with Sentry or PostHog. Until the query is deployed it throws on every run-detail visit, so an unconditional report would turn a documented, intended state into one issue and one event per run viewed — noise indistinguishable from a real regression at the moment one would matter most. `ErrorBoundary` gains an opt-in `isExpectedError` predicate for that, deliberately not a boolean: a boundary that suppressed everything would swallow the real bug it exists to surface. Only the two shapes Convex actually throws for an unavailable query are matched, `onError` still fires so the rail still closes, and a predicate that throws reports as normal. diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx b/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx index 263ce25f69..7e4a26d757 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx +++ b/mcpjam-inspector/client/src/components/evals/__tests__/run-insight-rail.test.tsx @@ -5,6 +5,7 @@ import { RunAccuracyHeroBand, RunDetailMetricsCharts, RunInsightRail, + runHasInsightContent, } from "../run-insight-rail"; import type { EvalIteration, EvalSuiteRun } from "../types"; @@ -258,3 +259,49 @@ describe("RunDetailMetricsCharts", () => { ).toBeInTheDocument(); }); }); + +describe("runHasInsightContent", () => { + // The OUTER half of the same decision `RunInsightRail` makes, and the one + // that runs first: it decides whether the band that wraps the rail exists at + // all. A rail that correctly opens is still never seen if this says no, so + // the rail's own tests cannot catch a regression here — they run inside a + // band that already rendered. + const none = { + serverQualityTriage: null, + goalCompletionPanel: null, + groundednessPanel: null, + actionableFindingsPanel: null, + hasStageFunnel: false, + }; + + it("is false when a run has nothing to show", () => { + expect(runHasInsightContent(none)).toBe(false); + }); + + it("is true for a run whose ONLY insight is its user-value chain", () => { + // The bug UVH-IN5 exists to fix. Every other member here is absent, so + // this is the case that was invisible: the chain is the report card of + // what the eval measured, and it was hidden on exactly the runs where it + // was the whole story. + expect(runHasInsightContent({ ...none, hasStageFunnel: true })).toBe(true); + }); + + it.each([ + ["serverQualityTriage"], + ["goalCompletionPanel"], + ["groundednessPanel"], + ["actionableFindingsPanel"], + ])("is true for a run whose only insight is %s", (key) => { + expect(runHasInsightContent({ ...none, [key]:
})).toBe(true); + }); + + it("reads the DATA, never the node", () => { + // The chain card is a fragment whose two halves each self-suppress while + // the fragment itself stays truthy. Passing the node would keep an + // otherwise-empty band alive as a full-height column of dead space, which + // is why the signature takes a boolean about the data instead. + expect(runHasInsightContent({ ...none, hasStageFunnel: false })).toBe( + false, + ); + }); +}); diff --git a/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx b/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx index d0ba859d58..b6318b6cfa 100644 --- a/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx +++ b/mcpjam-inspector/client/src/components/evals/run-detail-view.tsx @@ -66,6 +66,7 @@ import { HostChip } from "@/components/hosts/host-chip"; import { RunAccuracyHeroBand, RunInsightRail, + runHasInsightContent, shouldShowRunAccuracyHero, type RunTrendPoint, } from "./run-insight-rail"; @@ -852,13 +853,13 @@ export function RunDetailView({ * is why it was excluded from these checks in the first place, and why the * fix has to be data-driven rather than a matter of adding the node. */ - const hasInsightContent = Boolean( - serverQualityTriage || - goalCompletionPanel || - groundednessPanel || - actionableFindingsPanel || - hasStageFunnel, - ); + const hasInsightContent = runHasInsightContent({ + serverQualityTriage, + goalCompletionPanel, + groundednessPanel, + actionableFindingsPanel, + hasStageFunnel, + }); const triageFixCount = useMemo( () => diff --git a/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx b/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx index 969ef91bc6..e150580f0a 100644 --- a/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx +++ b/mcpjam-inspector/client/src/components/evals/run-insight-rail.tsx @@ -409,6 +409,47 @@ export function RunDetailMetricsCharts({ ); } +/** + * Whether the run-detail INSIGHT BAND should exist at all. + * + * The outer half of the same decision `RunInsightRail` makes below, and the + * one that runs first: the band wraps the rail, so a rail that correctly + * decides to open is never seen if this says no. They are separate conditions + * because they count different things — the band also counts the actionable- + * findings panel, which the rail does not render — but they must agree about + * the chain, and this is where that agreement is written down. + * + * `hasStageFunnel` is the member worth naming. Like `userValueChainHasContent` + * below it is a fact about the DATA, not about a node: the chain card is a + * fragment whose two halves each self-suppress while the fragment itself stays + * truthy, so counting the node would keep an otherwise-empty band alive as + * dead space. Dropping it, conversely, hides the band on exactly the runs + * whose chain is the only thing there is to show — the bug this lane exists to + * fix, and one the rail's own tests cannot catch from inside a band that never + * rendered. + */ +export function runHasInsightContent({ + serverQualityTriage, + goalCompletionPanel, + groundednessPanel, + actionableFindingsPanel, + hasStageFunnel, +}: { + serverQualityTriage?: ReactNode; + goalCompletionPanel?: ReactNode; + groundednessPanel?: ReactNode; + actionableFindingsPanel?: ReactNode; + hasStageFunnel?: boolean; +}): boolean { + return Boolean( + serverQualityTriage || + goalCompletionPanel || + groundednessPanel || + actionableFindingsPanel || + hasStageFunnel, + ); +} + /** Right column: AI insights only. */ export function RunInsightRail({ triageCard, diff --git a/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx b/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx index 0eb335b40d..03603862d5 100644 --- a/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx +++ b/mcpjam-inspector/client/src/components/shared/user-value-chain/StageFunnelPanels.tsx @@ -168,6 +168,29 @@ export function SuiteRunStageFunnelPanel({ * whole run-detail page down with it would be worse than the empty rail it * exists to prevent. Undeployed reads as "no funnel", which is correct. */ +/** + * The two failure shapes this probe genuinely EXPECTS, and nothing else. + * + * `useQuery` throws a plain `Error` when the deployed backend has no such + * function — the dark-ship window, which is an intended state, not a + * malfunction — and again when there is no `ConvexProvider` above it, which is + * every test tree that renders this component without one. + * + * Matched on the message because that is all Convex gives us: neither throw + * carries a code or a distinguishing class. Narrow by design — a query that + * fails for any OTHER reason is a real failure and still reports. Anyone + * widening this list is turning off an alarm, and should have to say so here. + */ +export function isConvexQueryUnavailable(error: Error): boolean { + const message = typeof error?.message === "string" ? error.message : ""; + return ( + // The function is not deployed (dark ship, or a browser outliving a rollback). + message.includes("Could not find public function") || + // No ConvexProvider above this tree. + message.includes("Could not find Convex client") + ); +} + export function SuiteRunStageFunnelAvailability({ suiteRunId, onChange, @@ -196,6 +219,13 @@ export function SuiteRunStageFunnelAvailability({ onChange(suiteRunId, false)} > diff --git a/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx b/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx index b0b416d283..c910b0e17a 100644 --- a/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx +++ b/mcpjam-inspector/client/src/components/shared/user-value-chain/__tests__/StageFunnelPanels.test.tsx @@ -23,7 +23,19 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const convex = vi.hoisted(() => ({ useQuery: vi.fn() })); vi.mock("convex/react", () => convex); +// The probe's boundary must not FILE the dark-ship window. Mocked rather than +// spied so the assertion is about what the boundary decided, not about whether +// Sentry happened to be configured in this test process. +const { reportBoundaryError } = vi.hoisted(() => ({ + reportBoundaryError: vi.fn(), +})); +vi.mock("@/lib/error-reporting", () => ({ + reportBoundaryError, + reportCaught: vi.fn(), +})); + const { + isConvexQueryUnavailable, ScenarioStageFunnelPanel, SuiteRunStageFunnelAvailability, SwarmRunStageFunnelPanels, @@ -233,6 +245,37 @@ describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () expect(container.textContent).toBe("the rest of the page"); }); + it("files NOTHING with Sentry/PostHog for the dark-ship window", () => { + // The probe is mounted on every run-detail visit and the query is + // deliberately undeployed, so an unconditional boundary report turns the + // intended state into one issue and one event per run VIEWED — noise + // indistinguishable from a real regression, at the moment one would + // matter most. + queryThrows(); + render( + , + ); + expect(reportBoundaryError).not.toHaveBeenCalled(); + }); + + it("DOES file a failure that is not one of the two expected shapes", () => { + // The suppression is a predicate, not a mute button: a probe that stopped + // reporting everything would swallow the real bug it exists to surface. + convex.useQuery.mockImplementation(() => { + throw new Error("TypeError: cannot read properties of undefined"); + }); + const onChange = vi.fn(); + render( + , + ); + expect(reportBoundaryError).toHaveBeenCalledTimes(1); + // And the rail still closes either way. + expect(onChange).toHaveBeenCalledWith("run-1", false); + }); + it("clears a previous answer when the SAME run's probe then fails", () => { // The boundary key re-arms the probe across runs, but a query that throws // after answering for the run still on screen renders the fallback @@ -307,3 +350,28 @@ describe("SuiteRunStageFunnelAvailability — the probe that opens the rail", () expect(onChange).toHaveBeenCalledWith("run-2", true); }); }); + +describe("isConvexQueryUnavailable", () => { + it.each([ + ["an undeployed function", "Could not find public function for 'x:y'"], + ["no ConvexProvider", "Could not find Convex client!"], + ])("recognises %s", (_label, message) => { + expect(isConvexQueryUnavailable(new Error(message))).toBe(true); + }); + + it.each([ + ["a real bug", "Cannot read properties of undefined (reading 'stages')"], + ["an auth refusal", "Authenticated user required"], + ["an empty message", ""], + ])("does NOT recognise %s", (_label, message) => { + expect(isConvexQueryUnavailable(new Error(message))).toBe(false); + }); + + it("survives an error with no message at all", () => { + // The predicate runs inside a boundary that is already handling a failure; + // throwing from here would be the second failure, at the worst moment. + expect( + isConvexQueryUnavailable({ message: undefined } as unknown as Error), + ).toBe(false); + }); +}); diff --git a/mcpjam-inspector/client/src/components/ui/__tests__/error-boundary.test.tsx b/mcpjam-inspector/client/src/components/ui/__tests__/error-boundary.test.tsx index fe21f7b931..fc38a31a32 100644 --- a/mcpjam-inspector/client/src/components/ui/__tests__/error-boundary.test.tsx +++ b/mcpjam-inspector/client/src/components/ui/__tests__/error-boundary.test.tsx @@ -106,6 +106,77 @@ describe("ErrorBoundary", () => { expect(screen.getByText("recovered")).toBeInTheDocument(); }); + it("does NOT report an error its boundary declared expected", () => { + // UVH-IN5. A dark-shipped query throws on a page users open repeatedly, + // so an unconditional report turns a documented, intended state into one + // Sentry issue and one PostHog event per visit. + render( + error.message === "kaboom"} + > + + , + ); + + expect(reportBoundaryError).not.toHaveBeenCalled(); + }); + + it("still reports an error the SAME boundary did not expect", () => { + // The predicate is the whole point: a boundary that suppressed everything + // would swallow the real bug it exists to surface. + render( + error.message === "something else"} + > + + , + ); + + expect(reportBoundaryError).toHaveBeenCalledTimes(1); + }); + + it("still calls onError for an expected error — telemetry only is suppressed", () => { + // The probe that motivated this uses `onError` to close its rail. Losing + // that alongside the reporting would trade a noisy alarm for a stuck UI. + const onError = vi.fn(); + render( + true} + onError={onError} + > + + , + ); + + expect(reportBoundaryError).not.toHaveBeenCalled(); + expect(onError).toHaveBeenCalledTimes(1); + }); + + it("reports normally when the predicate itself throws", () => { + // Fail loud, not silent: a broken predicate must not become a way to lose + // errors, which is the one outcome worse than the noise this suppresses. + render( + { + throw new Error("predicate is broken"); + }} + > + + , + ); + + expect(reportBoundaryError).toHaveBeenCalledTimes(1); + expect((reportBoundaryError.mock.calls[0][0] as Error).message).toBe( + "kaboom", + ); + }); + it("renders the default UI and does not report when nothing throws", () => { render( diff --git a/mcpjam-inspector/client/src/components/ui/error-boundary.tsx b/mcpjam-inspector/client/src/components/ui/error-boundary.tsx index 386e53b794..c5f4915100 100644 --- a/mcpjam-inspector/client/src/components/ui/error-boundary.tsx +++ b/mcpjam-inspector/client/src/components/ui/error-boundary.tsx @@ -15,6 +15,26 @@ interface ErrorBoundaryProps { * failures you expect to triage separately. */ name?: string; + /** + * Opt one boundary out of reporting for errors it EXPECTS, by predicate. + * + * Deliberately not a boolean. A boundary that suppressed everything would + * also swallow the real bug it was meant to surface, and the whole point of + * reporting here rather than at the mount sites is that a `fallback={null}` + * cannot silently eat one. A predicate keeps the exception as narrow as the + * call site can describe it: match the shape you know is expected, and + * anything else still reports exactly as before. + * + * The case this exists for is a dark-shipped query. `useQuery` throws while + * the function is not deployed yet, and a probe mounted on a page a user + * visits repeatedly turns a DOCUMENTED, intended state into one Sentry issue + * and one PostHog event per visit — noise that is indistinguishable from a + * real regression precisely when a real regression would matter most. + * + * `onError` still fires, and the fallback still renders: this suppresses the + * telemetry, never the handling. + */ + isExpectedError?: (error: Error) => boolean; } interface ErrorBoundaryState { @@ -27,7 +47,8 @@ interface ErrorBoundaryState { * * Every caught error is reported to Sentry + PostHog regardless of which * fallback renders — "silent to the user" is a UI choice, never a telemetry - * one. + * one. The single exception is `isExpectedError`, an opt-in predicate for a + * boundary that can name a failure shape it genuinely expects; see below. * * Fallback semantics: * - `fallback={null}` → render nothing on error (intentional silence; e.g. @@ -53,13 +74,32 @@ export class ErrorBoundary extends React.Component< } componentDidCatch(error: Error, errorInfo: React.ErrorInfo) { - console.error("ErrorBoundary caught an error:", error, errorInfo); - // Reported here rather than at the ~21 mount sites: a boundary rendering - // `fallback={null}` used to swallow its error entirely. - reportBoundaryError(error, errorInfo, this.props.name); + // An expected error is still worth a breadcrumb — it just is not a fault, + // so it goes to `debug` rather than shouting on the console once per visit. + const expected = this.isExpected(error); + if (expected) { + console.debug("ErrorBoundary caught an expected error:", error); + } else { + console.error("ErrorBoundary caught an error:", error, errorInfo); + // Reported here rather than at the ~21 mount sites: a boundary rendering + // `fallback={null}` used to swallow its error entirely. + reportBoundaryError(error, errorInfo, this.props.name); + } + // Outside the branch on purpose: suppressing telemetry must not also + // suppress the caller's handling of the failure. this.props.onError?.(error, errorInfo); } + /** A throwing predicate must not stop the error from being reported. */ + private isExpected(error: Error): boolean { + if (!this.props.isExpectedError) return false; + try { + return this.props.isExpectedError(error) === true; + } catch { + return false; + } + } + handleReset = () => { this.setState({ hasError: false, error: null }); }; From 54a9b4021f1dfc1927fd3138fcd5f6ca372970f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 19:11:36 +0000 Subject: [PATCH 13/15] UVH-IN7 review: pin that the chain reports the error policy forgives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every existing case here drives the scenario where `failOnToolError` fails the trial. The inverse was untested and is the one that shows why the chain is not a mirror of the verdict: under `failOnToolError: false` the legacy verdict PASSES on the very run whose tool errored, and the `response` row must still be red. Without that, a suite run entirely under that policy would report a clean funnel over servers that were failing calls. Both halves are exercised against the same span rather than asserted separately, because the property IS the relationship between them — the verdict answers "did policy fail this trial", the chain answers "what happened". A control assertion confirms the same span does fail the trial under the default policy, so the test is about the policy and not about a span that was never failure-worthy. Mutation-checked: forcing `hasObservedToolFailure` to false fails this test along with the four that already covered the stage. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- ...ls-observed-tool-error-reaches-response.md | 2 + sdk/tests/stage-derivation.test.ts | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.changeset/evals-observed-tool-error-reaches-response.md b/.changeset/evals-observed-tool-error-reaches-response.md index b9e4eddc17..51f939b2d9 100644 --- a/.changeset/evals-observed-tool-error-reaches-response.md +++ b/.changeset/evals-observed-tool-error-reaches-response.md @@ -15,4 +15,6 @@ Two boundaries are deliberate. A span carrying an `mcpErrorCode` never reached t This is the one evidence-driven entry in the applicability table, and it is a _positive observation_ rather than a gap, which is what keeps the surrounding rule intact: a stage turned on by observed evidence cannot then be reported as an evidence gap, because the deriver holds the very span that turned it on. +The chain says this whether or not the policy agrees. Under `failOnToolError: false` the trial passes on the very run whose tool errored, and the `response` row is still red — the verdict answers "did policy fail this trial", the chain answers "what happened", and those are allowed to differ. Without that, a suite run entirely under that policy would report a clean funnel over servers that were failing calls. + Analyzer 6 → 7. `STAGE_REASONS` does not move (`toolError` already existed), so the backend mirror needs no re-pin. diff --git a/sdk/tests/stage-derivation.test.ts b/sdk/tests/stage-derivation.test.ts index 8b68edccf7..789cbd293f 100644 --- a/sdk/tests/stage-derivation.test.ts +++ b/sdk/tests/stage-derivation.test.ts @@ -12,6 +12,7 @@ */ import { describe, expect, test } from "vitest"; +import { finalizePassedForEval } from "../src/eval-tool-execution"; import { MAX_EVIDENCE_REASONS, MAX_EVIDENCE_REASON_CHARS, @@ -832,6 +833,51 @@ describe("an observed tool error reaches response, even unauthored", () => { expect(stageResults.some((r) => r.state === "failed")).toBe(true); }); + test("the chain reports the error even when POLICY passes the trial", () => { + // The other half of the disagreement, and the one that shows why the + // chain is not just a mirror of the verdict. With `failOnToolError: false` + // the legacy verdict PASSES on the very run whose tool errored — so if the + // chain also went all-green, a suite run entirely under that policy would + // report a clean funnel over servers that were failing calls. + // + // Both halves are exercised against the SAME scenario rather than asserted + // separately, because the property is the relationship between them: the + // verdict answers "did policy fail this trial", the chain answers "what + // happened", and those are allowed to differ. + const erroredSpan = erroredToolSpan(); + + const passed = finalizePassedForEval({ + matchPassed: true, + trace: { spans: [erroredSpan] }, + failOnToolError: false, + predicateResults: [{ passed: true }], + }); + expect(passed).toBe(true); + + const { stageResults } = derive({ + authored: predicateOnlyCase, + evidence: { + spans: [erroredSpan], + predicateResults: [{ passed: true, reason: "ok" }], + }, + }); + expect(stateOf(stageResults, "response")).toMatchObject({ + state: "failed", + reason: "toolError", + }); + + // And the control: the same span DOES fail the trial under the default + // policy, so the test above is about the policy and not about a span that + // was never failure-worthy. + expect( + finalizePassedForEval({ + matchPassed: true, + trace: { spans: [erroredSpan] }, + predicateResults: [{ passed: true }], + }) + ).toBe(false); + }); + test("a transport-local error does NOT turn the stage on", () => { // A span carrying an MCP error code never reached the server's handler, // so it is a setup fact rather than the server's answer — and turning From 0f18e76950f620ce22730ce482f1744c3a6f5db6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 19:18:57 +0000 Subject: [PATCH 14/15] UVH-IN2 review: strip the prettier churn, correct two stale claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running prettier 3 over files last formatted under a `trailingComma: "es5"` config buried ~12 functional lines in ~350 lines of reformatting. In `evals-runner.ts` that meant 266 changed lines for 30 lines of actual change — a diff a reviewer cannot read, and one that would conflict with anything else touching those files. Reverted mechanically, in two passes: every line identical to its base modulo a trailing comma, then every hunk whose old and new sides are token-identical once whitespace and pre-bracket commas are normalized (prettier's re-wraps, which the first pass cannot see because they change line counts). 134 + 43 lines restored. One stray comment re-indent went with them — correct, but not this PR's business. evals-runner.ts 266 → 30 changed lines finalize-iteration.ts 86 → 26 drive-hosted-eval-turn.ts 129 → 89 step-executor.ts 83 → 59 The server diff's deletions drop from 240 to 58, so it now reads as what it is: an almost purely additive change. Verified by diffing the whole-repo server typecheck output before and after — 199 errors both times, the same 199, all pre-existing. Server evals suites: 720 passed. Two stale claims, both invalidated by later rounds in this same PR: - `applyProviderError`'s docblock said "applied last". True until the withdrawal had to move AHEAD of the positional cascade, which reads `failed` rows to decide which later stages never ran; withdrawing after it ran left three rows citing a failure the chain no longer recorded. - The first changeset said "only blank rows are re-labelled", which the withdraw-absences round then contradicted two changesets further down the same release note. It now states the rule that actually ships: a stage with its own observation keeps its row, a blank one is re-labelled, and a `failed` row resting on an ABSENCE the outage could equally well explain is withdrawn. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .../evals-provider-error-attribution.md | 2 +- .../server/services/evals-runner.ts | 236 +++++++++--------- .../services/evals/drive-hosted-eval-turn.ts | 40 +-- .../services/evals/finalize-iteration.ts | 60 +++-- .../server/services/evals/step-executor.ts | 24 +- sdk/src/contract/stage-derivation.ts | 10 +- 6 files changed, 192 insertions(+), 180 deletions(-) diff --git a/.changeset/evals-provider-error-attribution.md b/.changeset/evals-provider-error-attribution.md index 6093f9c9b0..fd455f172d 100644 --- a/.changeset/evals-provider-error-attribution.md +++ b/.changeset/evals-provider-error-attribution.md @@ -13,7 +13,7 @@ The layer that failed is now tagged at the catch site and carried into the chain Three boundaries are deliberate: -- **Only blank rows are re-labelled.** A provider dying at turn 4 does not un-observe turns 1–3, so a stage with its own evidence keeps its own row. +- **A stage with its own observation keeps its own row.** A provider dying at turn 4 does not un-observe turns 1–3. What is re-labelled is a stage that measured nothing, plus — see "withdraws the failures it made unknowable" below — a `failed` row whose verdict rests on an *absence* the outage could equally well explain. A verdict resting on something actually observed always stands. - **Never `failed`.** A run that could not be attempted has measured nothing about the server, and inflating a server failure rate with our own outage is the mis-attribution this reason exists to prevent. - **A broken grader still outranks it.** `evaluator` is never folded into another category. diff --git a/mcpjam-inspector/server/services/evals-runner.ts b/mcpjam-inspector/server/services/evals-runner.ts index 3569211f14..d27491db16 100644 --- a/mcpjam-inspector/server/services/evals-runner.ts +++ b/mcpjam-inspector/server/services/evals-runner.ts @@ -429,7 +429,7 @@ export function runFrozenSkillOptions(run: { run.pinnedHarnessSkills === null ? "null" : typeof run.pinnedHarnessSkills - })`, + })` ); } return { @@ -496,7 +496,7 @@ export function resolveIterationSkillsSource(args: { * never wrote. Change both, or neither. */ function scoreMatchOptionsFor( - test: Pick, + test: Pick ): Record { return resolveMatchOptions(undefined, test.matchOptions) as unknown as Record< string, @@ -650,13 +650,13 @@ export type EvalIterationOutcome = { export function narrowToolsToAdvertised( allTools: PrepareChatV2Result["allTools"], progressivePlan: ProgressiveToolPlan, - discoveryState: ToolDiscoveryState, + discoveryState: ToolDiscoveryState ): PrepareChatV2Result["allTools"] { if (!progressivePlan.enabled) { return allTools; } const advertisedNames = new Set( - resolveActiveToolNames(progressivePlan, discoveryState), + resolveActiveToolNames(progressivePlan, discoveryState) ); const narrowed: PrepareChatV2Result["allTools"] = {}; for (const [name, tool] of Object.entries(allTools)) { @@ -769,12 +769,12 @@ type TraceSnapshotKind = "step_finish" | "turn_finish" | "failure"; function getServerLabelForEvalError( serverId: string, - environment: RunEvalSuiteOptions["config"]["environment"] | undefined, + environment: RunEvalSuiteOptions["config"]["environment"] | undefined ): string { const binding = environment?.serverBindings?.find( (entry) => entry.projectServerId === serverId || - entry.projectServerId?.toLowerCase() === serverId.toLowerCase(), + entry.projectServerId?.toLowerCase() === serverId.toLowerCase() ); return binding?.serverName || serverId; } @@ -819,7 +819,7 @@ function throwSetupPhaseError(args: { }): never { const serverLabel = getServerLabelForEvalError( args.serverId, - args.environment, + args.environment ); if (isMissingRuntimeServerError(args.error) || args.phase === "connection") { throw new EvalSetupPhaseError({ @@ -893,7 +893,7 @@ async function getEvalToolsForAiSdkOrThrow(args: { const tools = toolOptions ? await args.mcpClientManager.getToolsForAiSdk( [serverId], - toolOptions, + toolOptions ) : await args.mcpClientManager.getToolsForAiSdk([serverId]); const endedAt = now(); @@ -939,7 +939,7 @@ async function getEvalToolsForAiSdkOrThrow(args: { firstError.push({ error, serverId, phase: "discovery" }); return null; } - }), + }) ); if (observer) { @@ -987,7 +987,7 @@ export function resolveConfiguredServerIds(args: { const availableServerIdsSet = new Set(availableServerIds); const availableServerIdByLowercase = new Map( - availableServerIds.map((serverId) => [serverId.toLowerCase(), serverId]), + availableServerIds.map((serverId) => [serverId.toLowerCase(), serverId]) ); const projectServerIdByName = new Map(); const serverNameByProjectServerId = new Map(); @@ -1025,7 +1025,7 @@ export function resolveConfiguredServerIds(args: { : availableServerIdByLowercase.get(trimmedServerRef.toLowerCase()) ?? (() => { const projectServerId = projectServerIdByName.get( - trimmedServerRef.toLowerCase(), + trimmedServerRef.toLowerCase() ); if (projectServerId) { return ( @@ -1037,7 +1037,7 @@ export function resolveConfiguredServerIds(args: { } const serverName = serverNameByProjectServerId.get( - trimmedServerRef.toLowerCase(), + trimmedServerRef.toLowerCase() ); if (serverName) { return ( @@ -1132,11 +1132,11 @@ function resolvePinnedServerKey( pinned: PinnedToolCall, environment: RunEvalSuiteOptions["config"]["environment"] | undefined, selectedServers: string[], - mcpClientManager: MCPClientManager, + mcpClientManager: MCPClientManager ): string | undefined { const connected = new Set(selectedServers); const candidates = [pinned.serverId, pinned.serverName].filter( - (ref): ref is string => !!ref, + (ref): ref is string => !!ref ); for (const candidate of candidates) { const [resolved] = resolveConfiguredServerIds({ @@ -1155,12 +1155,12 @@ function resolvePinnedServerKey( function buildPromptTraceSummaries( evaluation: MultiTurnEvaluationResult, - turnCheckResults: PredicateResult[] = [], + turnCheckResults: PredicateResult[] = [] ): PromptTraceSummary[] { return evaluation.promptSummaries.map((summary) => { const perTurn = turnCheckResults.filter( (r) => - r.scope?.kind === "turn" && r.scope.promptIndex === summary.promptIndex, + r.scope?.kind === "turn" && r.scope.promptIndex === summary.promptIndex ); return { promptIndex: summary.promptIndex, @@ -1190,7 +1190,7 @@ function buildPromptTraceSummaries( mismatchedArguments: Array.from(mismatchedArguments).filter( (key) => JSON.stringify(mismatch.expectedArgs?.[key]) !== - JSON.stringify(mismatch.actualArgs?.[key]), + JSON.stringify(mismatch.actualArgs?.[key]) ), }; }), @@ -1233,7 +1233,7 @@ function extractToolCallsFromConversation(params: { (toolCall) => toolCall.toolName === name && JSON.stringify(toolCall.arguments) === - JSON.stringify(argumentsValue), + JSON.stringify(argumentsValue) ); if (!alreadyAdded) { toolsCalled.push({ @@ -1258,7 +1258,7 @@ function extractToolCallsFromConversation(params: { (toolCall) => toolCall.toolName === toolName && JSON.stringify(toolCall.arguments) === - JSON.stringify(argumentsValue), + JSON.stringify(argumentsValue) ); if (!alreadyAdded) { toolsCalled.push({ @@ -1282,12 +1282,12 @@ function extractToolCallsExcludingPolicyBlocks( steps?: ReadonlyArray; messages: ModelMessage[]; }, - blockedToolCallIds: ReadonlySet, + blockedToolCallIds: ReadonlySet ): ToolCall[] { return extractToolCallsFromConversation(params).filter( (toolCall) => toolCall.toolCallId === undefined || - !blockedToolCallIds.has(toolCall.toolCallId), + !blockedToolCallIds.has(toolCall.toolCallId) ); } @@ -1297,7 +1297,7 @@ function toolCallIdentity(toolCall: ToolCall): string { function mergeToolCalls( existingToolCalls: ToolCall[], - incomingToolCalls: ToolCall[], + incomingToolCalls: ToolCall[] ): ToolCall[] { const seen = new Set(existingToolCalls.map(toolCallIdentity)); const merged = [...existingToolCalls]; @@ -1331,14 +1331,14 @@ function appendPartialToolCallsToPrompt(params: { } const existingToolCalls = Array.isArray( - params.toolsCalledByPrompt[params.promptIndex], + params.toolsCalledByPrompt[params.promptIndex] ) ? params.toolsCalledByPrompt[params.promptIndex]! : []; params.toolsCalledByPrompt[params.promptIndex] = mergeToolCalls( existingToolCalls, - partialToolCalls, + partialToolCalls ); } @@ -1377,7 +1377,7 @@ function buildTraceSnapshotEvent(params: { snapshotKind: params.snapshotKind, trace: sanitizeForConvexTransport(trace), actualToolCalls: sanitizeForConvexTransport( - toStreamToolCalls(params.actualToolCalls), + toStreamToolCalls(params.actualToolCalls) ), usage: { inputTokens: params.usage.inputTokens ?? 0, @@ -1397,7 +1397,7 @@ function buildTraceSnapshotEvent(params: { * (already steps-shaped) — and prefers an existing `steps` array if present. */ function snapshotWithStepsForConvex( - snapshot: Record, + snapshot: Record ): Record { if ( !snapshot || @@ -1441,7 +1441,7 @@ async function createIterationDirectly( }; iterationNumber: number; startedAt: number; - }, + } ): Promise { try { const result = await convexClient.mutation( @@ -1449,11 +1449,11 @@ async function createIterationDirectly( { testCaseId: params.testCaseId, testCaseSnapshot: sanitizeForConvexTransport( - snapshotWithStepsForConvex(params.testCaseSnapshot), + snapshotWithStepsForConvex(params.testCaseSnapshot) ), iterationNumber: params.iterationNumber, startedAt: params.startedAt, - }, + } ); return result?.iterationId as string | undefined; @@ -1555,10 +1555,10 @@ async function persistRunSetupFailure(args: { try { const details = (await args.convexClient.query( "testSuites:getTestSuiteRunDetails" as any, - { runId: args.runId }, + { runId: args.runId } )) as { iterations?: Array> } | null; return (details?.iterations ?? []).filter( - (row) => row.status === "pending", + (row) => row.status === "pending" ); } catch (readError) { logger.warn("[evals] Failed to read pending setup iterations", { @@ -1581,7 +1581,7 @@ async function persistRunSetupFailure(args: { : undefined; const test = args.tests.find( (candidate) => - candidate.testCaseId && candidate.testCaseId === row.testCaseId, + candidate.testCaseId && candidate.testCaseId === row.testCaseId ); const snapshot = row.testCaseSnapshot as | { query?: string; expectedToolCalls?: unknown[] } @@ -1616,7 +1616,7 @@ async function persistRunSetupFailure(args: { recorder: args.recorder, convexClient: args.convexClient, }); - }), + }) ); }; @@ -1633,7 +1633,7 @@ async function persistRunSetupFailure(args: { try { await args.convexClient.mutation( "testSuites:markSetupPendingIterationsFailed" as any, - { runId: args.runId, error: args.errorMessage }, + { runId: args.runId, error: args.errorMessage } ); } catch (cleanupError) { logger.warn("[evals] Failed to mark residual setup iterations failed", { @@ -1823,7 +1823,7 @@ const buildModelDefinition = (test: EvalTestCase): ModelDefinition => { function lookupProviderApiKey( modelApiKeys: Record | undefined, - provider: string, + provider: string ): string | undefined { return modelApiKeys?.[provider] ?? modelApiKeys?.[provider.toLowerCase()]; } @@ -1853,7 +1853,7 @@ function resolveEvalModelRuntime(args: { const provider = args.modelDefinition.provider; if (!apiKey && provider !== "ollama" && provider !== "custom") { throw new Error( - `Missing API key for provider ${args.test.provider} (test: ${args.test.title})`, + `Missing API key for provider ${args.test.provider} (test: ${args.test.title})` ); } @@ -1869,14 +1869,14 @@ function resolveEvalModelRuntime(args: { } function hasExplicitModelApiKeys( - modelApiKeys: Record | undefined, + modelApiKeys: Record | undefined ): boolean { return Boolean(modelApiKeys && Object.keys(modelApiKeys).length > 0); } function resolveOrgTargetForEval( test: EvalTestCase, - explicitTarget?: ResolveOrgModelConfigTarget, + explicitTarget?: ResolveOrgModelConfigTarget ): ResolveOrgModelConfigTarget | undefined { if (explicitTarget) return explicitTarget; const maybeProjectId = (test as { projectId?: unknown }).projectId; @@ -1926,7 +1926,7 @@ async function resolveOrgByokEvalRuntime(args: { target, providerKey, String(args.modelDefinition.id), - { bearerToken: args.convexAuthToken }, + { bearerToken: args.convexAuthToken } ); if (runtime.runtimeLocation === "cloud") { return { kind: "cloud", providerKey: runtime.providerKey, target }; @@ -1941,15 +1941,15 @@ async function resolveOrgByokEvalRuntime(args: { // PR6: single hosted wrapper for both modes (emit optional). Owns the browser // harness lifecycle (try/finally guarantees Chromium teardown on every exit). const runHostedIteration = async ( - params: RunIterationBackendParams & { emit?: StreamEmit }, + params: RunIterationBackendParams & { emit?: StreamEmit } ): Promise => { // First pinned turn's per-call render-budget override (mirrors the local // runner's harness creation). const pinnedRenderTimeoutMs = resolveEvalTestCase( - params.test, + params.test ).promptTurns.find( (t) => - isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number", + isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number" )?.pinnedToolCall?.renderTimeoutMs; const browser = await createBrowserSessionContext({ model: params.test.model, @@ -1997,7 +1997,7 @@ async function findIterationIdForTimeout(args: { try { const response = await args.convexClient.query( "testSuites:getTestSuiteRunDetails" as any, - { runId: args.runId }, + { runId: args.runId } ); const iterations = response?.iterations ?? []; const matching = iterations.find((iteration: any) => { @@ -2207,7 +2207,7 @@ const executeTestCase = async (params: { runner: () => Promise, precreatedIterationId: string | undefined, runIndex: number, - timeoutTest: EvalTestCase = normalizedTest, + timeoutTest: EvalTestCase = normalizedTest ): Promise => { if (abortSignal?.aborted) { const reason = abortSignal.reason; @@ -2292,8 +2292,8 @@ const executeTestCase = async (params: { }), undefined, runIndex, - normalizedTest, - ), + normalizedTest + ) ); } return outcomes; @@ -2302,11 +2302,11 @@ const executeTestCase = async (params: { const modelDefinition = buildModelDefinition(test); const resolvedModelId = getCanonicalModelId( String(modelDefinition.id), - modelDefinition.provider, + modelDefinition.provider ); const isJamModel = isHostedCatalogModel( resolvedModelId, - modelDefinition.provider, + modelDefinition.provider ); const orgByokRuntime = isJamModel ? undefined @@ -2366,7 +2366,7 @@ const executeTestCase = async (params: { { runIndex, error: error instanceof Error ? error.message : String(error), - }, + } ); precreatedIterationIds.push(undefined); } @@ -2429,7 +2429,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test, + test ); outcomes.push(iterationOutcome); continue; @@ -2491,7 +2491,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test, + test ); outcomes.push(iterationOutcome); continue; @@ -2545,7 +2545,7 @@ const executeTestCase = async (params: { }), precreatedIterationId, runIndex, - test, + test ); outcomes.push(iterationOutcome); } @@ -2556,7 +2556,7 @@ const executeTestCase = async (params: { // Thin batch wrapper (no `emit`) — preserves the call site in // `runEvalSuiteWithAiSdk` and tests with zero churn. const runTestCase = ( - params: Omit[0], "emit">, + params: Omit[0], "emit"> ) => executeTestCase(params); export const runEvalSuiteWithAiSdk = async ({ @@ -2602,7 +2602,7 @@ export const runEvalSuiteWithAiSdk = async ({ ) { logger.warn( "[evals] readOnly tool policy does not restrict the sandbox bash tool", - { suiteId }, + { suiteId } ); } @@ -2640,7 +2640,7 @@ export const runEvalSuiteWithAiSdk = async ({ const evalTasksSeam = resolveToolTaskSeam({ tasksPolicy: readTasksPolicy( - (suiteHostConfig ?? undefined) as Parameters[0], + (suiteHostConfig ?? undefined) as Parameters[0] ), surface: "eval", // Driver `timeoutMs` stays at its default — the task drive nests under @@ -2678,19 +2678,19 @@ export const runEvalSuiteWithAiSdk = async ({ const toolAnnotations: ToolAnnotationsLookup = new Map(); if (toolPolicy) { const uncachedServerIds = serverIds.filter( - (serverId) => !mcpClientManager.hasCachedToolAnnotations(serverId), + (serverId) => !mcpClientManager.hasCachedToolAnnotations(serverId) ); if (uncachedServerIds.length > 0) { throw new WebRouteError( 400, ErrorCode.VALIDATION_ERROR, `TOOL_POLICY_ANNOTATIONS_UNAVAILABLE: tool policy requires a populated annotation cache for every selected server; missing ${uncachedServerIds.join( - ", ", + ", " )}.`, { reason: "TOOL_POLICY_ANNOTATIONS_UNAVAILABLE", serverIds: uncachedServerIds, - }, + } ); } try { @@ -2711,18 +2711,18 @@ export const runEvalSuiteWithAiSdk = async ({ 400, ErrorCode.VALIDATION_ERROR, error.message, - { reason: "TOOL_POLICY_INVALID" }, + { reason: "TOOL_POLICY_INVALID" } ); } throw error; } for (const serverId of serverIds) { for (const [toolName, annotations] of Object.entries( - mcpClientManager.getAllToolAnnotations(serverId), + mcpClientManager.getAllToolAnnotations(serverId) )) { toolAnnotations.set( toolAnnotationsKey(serverId, toolName), - annotations, + annotations ); } } @@ -2735,7 +2735,7 @@ export const runEvalSuiteWithAiSdk = async ({ ? applyVisibilityPolicyAndCountSignals( tools as Record, mcpClientManager, - hostExecutionPolicy, + hostExecutionPolicy ) : undefined; const resolvedSetupSignals = setupObserver.buildSignals(); @@ -2752,7 +2752,7 @@ export const runEvalSuiteWithAiSdk = async ({ "testSuites:getTestSuiteRun" as any, { runId, - }, + } ); if (currentRun?.status === "cancelled") { @@ -2775,7 +2775,7 @@ export const runEvalSuiteWithAiSdk = async ({ // exhaust the worker. LLM-only cases are network-bound and stay unbounded. // The limiter releases a slot when each case settles, so it can't leak. const renderCheckLimit = createConcurrencyLimiter( - MAX_CONCURRENT_RENDER_CHECKS, + MAX_CONCURRENT_RENDER_CHECKS ); const runOne = (test: (typeof tests)[number]) => runTestCase({ @@ -2837,7 +2837,7 @@ export const runEvalSuiteWithAiSdk = async ({ promptTurns: resolveEvalTestCase(test).promptTurns, }) ? renderCheckLimit(() => runOne(test)) - : runOne(test), + : runOne(test) ); // Poll the run status: user cancellation, or a `timed_out` status set @@ -2851,7 +2851,7 @@ export const runEvalSuiteWithAiSdk = async ({ try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId }, + { runId } ); if (currentRun?.status === "cancelled") { abortRun(RUN_CANCELLED_ERROR); @@ -2886,7 +2886,7 @@ export const runEvalSuiteWithAiSdk = async ({ try { await convexClient.mutation( "testSuites:heartbeatTestSuiteRun" as any, - { runId }, + { runId } ); } catch (error) { logger.warn("[evals] Failed to heartbeat eval run", { @@ -2920,8 +2920,8 @@ export const runEvalSuiteWithAiSdk = async ({ throw error; } return never(); - }), - ), + }) + ) ); const allTestsSettled = Promise.allSettled(testPromises); @@ -2989,7 +2989,7 @@ export const runEvalSuiteWithAiSdk = async ({ } } summary.policyBlockedIterations += outcomes.filter( - (outcome) => (outcome.policyBlockCount ?? 0) > 0, + (outcome) => (outcome.policyBlockCount ?? 0) > 0 ).length; if (runId === null) { quickRunOutcomes.push(...outcomes); @@ -3101,7 +3101,7 @@ async function seedAndAnnotateEvalAttachments(args: { }); if (!seeded.note) return; const firstModelTurnIndex = args.promptTurns.findIndex( - (t) => !isPinnedTurn(t), + (t) => !isPinnedTurn(t) ); if (firstModelTurnIndex < 0) return; args.promptTurns[firstModelTurnIndex] = { @@ -3170,7 +3170,7 @@ const runLocalIteration = async ({ try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId }, + { runId } ); if (currentRun?.status === "cancelled") { return { @@ -3182,7 +3182,7 @@ const runLocalIteration = async ({ resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -3205,7 +3205,7 @@ const runLocalIteration = async ({ resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -3259,7 +3259,7 @@ const runLocalIteration = async ({ }); const system = withHostContextSystemPrompt( resolvedExecution.systemPrompt, - test.hostConfigOverride?.hostContext as Record | undefined, + test.hostConfigOverride?.hostContext as Record | undefined ); const temperature = resolvedExecution.temperature; const toolChoice = normalizeToolChoice(advancedConfig?.toolChoice); @@ -3276,7 +3276,7 @@ const runLocalIteration = async ({ // First pinned turn's render-budget override; applied to the shared harness. const pinnedRenderTimeoutMs = promptTurns.find( (t) => - isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number", + isPinnedTurn(t) && typeof t.pinnedToolCall?.renderTimeoutMs === "number" )?.pinnedToolCall?.renderTimeoutMs; const modelRuntime = caseNeedsModel @@ -3446,7 +3446,7 @@ const runLocalIteration = async ({ if (caseNeedsModel) { resolveHostTools( { builtInToolIds: resolvedExecution.builtInToolIds }, - null, + null ); prepared = await prepareChatV2({ mcpClientManager, @@ -3495,7 +3495,7 @@ const runLocalIteration = async ({ modelDefinition, modelRuntime!.apiKey, modelRuntime!.baseUrls, - modelRuntime!.customProviders, + modelRuntime!.customProviders ); // Reproducible evals: boot a fresh ephemeral sandbox from the suite's @@ -3520,7 +3520,7 @@ const runLocalIteration = async ({ // paid box only the backend TTL GC could reap. Fail loudly instead. if (!isComputersDataPlaneConfigured()) { throw new Error( - "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it.", + "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." ); } evalSandbox = await provisionEvalSandbox({ @@ -3531,7 +3531,7 @@ const runLocalIteration = async ({ }); if (!evalSandbox.ok) { throw new Error( - `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}`, + `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}` ); } // COMP-17: seed the case's pinned attachments into the fresh box before @@ -3561,7 +3561,7 @@ const runLocalIteration = async ({ !Object.hasOwn(browser.computerWidgetTools, toolChoice.toolName) ) { throw new Error( - `Configured tool choice '${toolChoice.toolName}' is not available for this eval run.`, + `Configured tool choice '${toolChoice.toolName}' is not available for this eval run.` ); } } @@ -3578,7 +3578,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -3614,7 +3614,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }), + }) ); }, onTurnFailure: ({ @@ -3633,7 +3633,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }), + }) ); emit({ type: "step_status", @@ -3653,7 +3653,7 @@ const runLocalIteration = async ({ spans, actualToolCalls: extractToolCallsFromConversation({ messages }), usage, - }), + }) ); emit({ type: "turn_finish", turnIndex }); emit({ @@ -3666,7 +3666,7 @@ const runLocalIteration = async ({ onPinnedTurn: (ctx) => emitPinnedTurnSse( { emit, withSystemPrefix, buildTraceSnapshotEvent }, - { turnIndex, ...ctx }, + { turnIndex, ...ctx } ), }) : undefined; @@ -3694,7 +3694,7 @@ const runLocalIteration = async ({ pinned, environment, selectedServers, - mcpClientManager, + mcpClientManager ), prepared, llmModel, @@ -3711,7 +3711,7 @@ const runLocalIteration = async ({ extractToolCalls: (params) => extractToolCallsExcludingPolicyBlocks( params, - toolPolicyGate?.blockedToolCallIds() ?? new Set(), + toolPolicyGate?.blockedToolCallIds() ?? new Set() ), // Per-turn streaming play-by-play (headless in batch). buildSinks: makeSinks, @@ -3755,13 +3755,13 @@ const runLocalIteration = async ({ // accounting (a click that fires a tool the case forbade SHOULD fail it). const toolsCalledByPromptWithWidgets = mergeToolCallsByPromptIndex( acc.toolsCalledByPrompt, - widgetToolCallsByPromptIndex(browser.browserInteractionSteps), + widgetToolCallsByPromptIndex(browser.browserInteractionSteps) ); // Per-turn predicate results from step assert execution facts (not a // re-evaluation of promptTurns.checks — avoids duplicates vs executeSteps). const turnCheckResults = resolveTurnCheckResultsFromStepExecution( stepState, - steps, + steps ); const failOnToolError = (advancedConfig as { failOnToolError?: boolean } | undefined) @@ -3809,7 +3809,7 @@ const runLocalIteration = async ({ ? acc.accumulatedUsage : undefined, renderObservations: summarizeRenderObservations( - browser.widgetRenderObservations, + browser.widgetRenderObservations ), toolErrors: acc.pinnedToolErrors, iterationError: acc.iterationError, @@ -3823,7 +3823,7 @@ const runLocalIteration = async ({ }); const promptTraceSummaries = buildPromptTraceSummaries( evaluation, - turnCheckResults, + turnCheckResults ); // Reflect the gated verdict (match AND tool-error gate AND predicates) in // the returned evaluation so totals built from `evaluation.passed` agree @@ -3937,13 +3937,13 @@ const runLocalIteration = async ({ ? narrowToolsToAdvertised( selectionToolsForFinish, selectionDiscoveryForFinish.progressivePlan, - selectionDiscoveryForFinish.discoveryState, + selectionDiscoveryForFinish.discoveryState ) : selectionToolsForFinish, } : {}), }); - // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. + // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. // // At `enforce` the iteration's result is the conjunction of the boolean // pipeline and the gating score rows, computed inside @@ -3985,7 +3985,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -4040,7 +4040,7 @@ const runLocalIteration = async ({ promptTurns, acc.toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions, + test.matchOptions ); // Suite summary aggregates `evaluation.passed` (see runEvalSuiteWithAiSdk). // The persisted iteration is hard-coded `passed: false` below, but the @@ -4077,7 +4077,7 @@ const runLocalIteration = async ({ totalTokens: acc.accumulatedUsage.totalTokens, }, prompts: promptTraceSummaries, - }), + }) ); emit({ type: "error", @@ -4166,7 +4166,7 @@ const runLocalIteration = async ({ ? narrowToolsToAdvertised( selectionToolsForFinish, selectionDiscoveryForFinish.progressivePlan, - selectionDiscoveryForFinish.discoveryState, + selectionDiscoveryForFinish.discoveryState ) : selectionToolsForFinish, } @@ -4256,7 +4256,7 @@ const runHostedIterationWithBrowser = async ( }: RunIterationBackendParams & { emit?: StreamEmit; }, - browser: BrowserSessionContext, + browser: BrowserSessionContext ): Promise => { const resolvedTest = resolveEvalTestCase(test); const toolPolicyGate = resolveEnforcementGate({ @@ -4275,7 +4275,7 @@ const runHostedIterationWithBrowser = async ( try { const currentRun = await convexClient.query( "testSuites:getTestSuiteRun" as any, - { runId }, + { runId } ); if (currentRun?.status === "cancelled") { return { @@ -4287,7 +4287,7 @@ const runHostedIterationWithBrowser = async ( resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -4310,7 +4310,7 @@ const runHostedIterationWithBrowser = async ( resolvedTest.promptTurns, [], test.isNegativeTest, - test.matchOptions, + test.matchOptions ), passed: false, }, @@ -4351,7 +4351,7 @@ const runHostedIterationWithBrowser = async ( }); const systemPrompt = withHostContextSystemPrompt( resolvedExecution.systemPrompt, - test.hostConfigOverride?.hostContext as Record | undefined, + test.hostConfigOverride?.hostContext as Record | undefined ); const temperature = resolvedExecution.temperature; const toolChoice = normalizeToolChoice(advancedConfig?.toolChoice); @@ -4443,7 +4443,7 @@ const runHostedIterationWithBrowser = async ( { builtInToolIds: resolvedExecution.builtInToolIds }, builtInTarget && "projectId" in builtInTarget ? { authHeader: convexAuthToken, projectId: builtInTarget.projectId } - : null, + : null ); // ── Harness execution inputs, resolved once per iteration. // @@ -4555,7 +4555,7 @@ const runHostedIterationWithBrowser = async ( throw new Error( pinnedEnvironmentId ? "This eval pins a reproducible computer environment, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." - : "This eval runs on a harness, which boots a disposable computer per iteration, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it.", + : "This eval runs on a harness, which boots a disposable computer per iteration, but this server isn't a computers data plane (deployed servers bootstrap credentials from INSPECTOR_SERVICE_TOKEN; see docs/project-computers.md) — it could provision a sandbox but not exec or release it." ); } evalSandbox = await provisionEvalSandbox({ @@ -4566,7 +4566,7 @@ const runHostedIterationWithBrowser = async ( }); if (!evalSandbox.ok) { throw new Error( - `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}`, + `Could not provision the eval's reproducible sandbox: ${evalSandbox.error}` ); } // COMP-17: seed the case's pinned attachments before exposing `bash` @@ -4627,7 +4627,7 @@ const runHostedIterationWithBrowser = async ( promptTurns, [], test.isNegativeTest, - test.matchOptions, + test.matchOptions ); failedEvaluation.passed = false; return { @@ -4668,7 +4668,7 @@ const runHostedIterationWithBrowser = async ( promptTurns, toolsCalledByPrompt, test.isNegativeTest, - test.matchOptions, + test.matchOptions ), iterationId: undefined, }); @@ -4699,7 +4699,7 @@ const runHostedIterationWithBrowser = async ( extractToolCalls: (messages) => extractToolCallsExcludingPolicyBlocks( { messages }, - toolPolicyGate?.blockedToolCallIds() ?? new Set(), + toolPolicyGate?.blockedToolCallIds() ?? new Set() ), buildTraceSnapshotEvent, }) @@ -4818,7 +4818,7 @@ const runHostedIterationWithBrowser = async ( extractToolCalls: (messages) => extractToolCallsExcludingPolicyBlocks( { messages }, - toolPolicyGate?.blockedToolCallIds() ?? new Set(), + toolPolicyGate?.blockedToolCallIds() ?? new Set() ), acc: { messageHistory, @@ -4833,7 +4833,7 @@ const runHostedIterationWithBrowser = async ( pinned, environment, selectedServers, - mcpClientManager, + mcpClientManager ), pinnedToolErrors, ...(emit @@ -4841,7 +4841,7 @@ const runHostedIterationWithBrowser = async ( emitPinnedTurn: (payload: PinnedTurnSsePayload) => emitPinnedTurnSse( { emit, withSystemPrefix, buildTraceSnapshotEvent }, - payload, + payload ), } : {}), @@ -4900,7 +4900,7 @@ const runHostedIterationWithBrowser = async ( // calls whether authored as an expected tool call or a predicate. const toolsCalledByPromptWithWidgets = mergeToolCallsByPromptIndex( toolsCalledByPrompt, - widgetToolCallsByPromptIndex(browser.browserInteractionSteps), + widgetToolCallsByPromptIndex(browser.browserInteractionSteps) ); const failOnToolError = (advancedConfig as { failOnToolError?: boolean } | undefined) @@ -4918,7 +4918,7 @@ const runHostedIterationWithBrowser = async ( // Per-turn predicate results from step assert execution (hosted parity). const turnCheckResults = resolveTurnCheckResultsFromStepExecution( stepState, - steps, + steps ); const effectivePredicates = test.successPredicates?.length ? test.successPredicates @@ -4941,7 +4941,7 @@ const runHostedIterationWithBrowser = async ( trace: traceForGate, usage: hasReportedUsage(accumulatedUsage) ? accumulatedUsage : undefined, renderObservations: summarizeRenderObservations( - browser.widgetRenderObservations, + browser.widgetRenderObservations ), toolErrors: pinnedToolErrors, iterationError, @@ -4955,7 +4955,7 @@ const runHostedIterationWithBrowser = async ( }); const promptTraceSummaries = buildPromptTraceSummaries( evaluation, - turnCheckResults, + turnCheckResults ); // Reflect the gated verdict (match AND tool-error gate AND predicates) in the // returned evaluation so totals built from `evaluation.passed` agree with the @@ -5043,7 +5043,7 @@ const runHostedIterationWithBrowser = async ( selectionTools: narrowToolsToAdvertised( prepared.allTools, prepared.progressivePlan, - prepared.discoveryState, + prepared.discoveryState ), }); // RE-READ THE DERIVED VERDICT so run totals agree with the persisted row. @@ -5084,5 +5084,5 @@ const runHostedIterationWithBrowser = async ( export const streamTestCase = ( params: Omit[0], "emit"> & { emit: StreamEmit; - }, + } ) => executeTestCase(params); diff --git a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts index bfc8e22d05..21dbc6e391 100644 --- a/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts +++ b/mcpjam-inspector/server/services/evals/drive-hosted-eval-turn.ts @@ -323,10 +323,16 @@ export function failedLayerForEngineError( } export async function driveHostedEvalTurn( - params: DriveHostedEvalTurnParams, + params: DriveHostedEvalTurnParams ): Promise { - const { promptIndex, browser, prepared, acc, isAborted, abortSignal } = - params; + const { + promptIndex, + browser, + prepared, + acc, + isAborted, + abortSignal, + } = params; const logSuffix = params.logSuffix ?? ""; // Browser-rendered MCP App eval (PR 14): stamp collected artifacts with @@ -353,7 +359,7 @@ export async function driveHostedEvalTurn( ? params.toolPolicyGate.wrap(mergedTools) : mergedTools, traceCtx, - promptIndex, + promptIndex ); // Push the user prompt into `messageHistory` BEFORE the engine call so a @@ -378,9 +384,8 @@ export async function driveHostedEvalTurn( // parent's already-committed calls end so the post-turn reconcile below // replaces only THIS turn's live entries (the stream runner's `onToolCall` // populates the array live) without wiping the parent's. - const promptToolsCalled: ToolCall[] = (acc.toolsCalledByPrompt[ - promptIndex - ] ??= []); + const promptToolsCalled: ToolCall[] = (acc.toolsCalledByPrompt[promptIndex] ??= + []); const promptToolsBaseline = promptToolsCalled.length; // Built inside the pre-turn try below; `{}` until then so the failure @@ -401,14 +406,14 @@ export async function driveHostedEvalTurn( // failure branches below (CodeRabbit, PR 2610). const mapThrownTurnError = ( error: unknown, - failedStage: string, + failedStage: string ): HostedEvalTurnOutcome => { if ( isAborted() || (error instanceof Error && error.name === "AbortError") ) { logger.debug( - `[evals] backend iteration${logSuffix} aborted due to cancellation`, + `[evals] backend iteration${logSuffix} aborted due to cancellation` ); return { kind: "cancelled" }; } @@ -503,7 +508,7 @@ export async function driveHostedEvalTurn( systemPrompt: EVAL_WIDGET_MODEL_CONTEXT ? withWidgetContextSystemPrompt( prepared.enhancedSystemPrompt, - browser.browserInteractionSteps, + browser.browserInteractionSteps ) : prepared.enhancedSystemPrompt, ...(prepared.resolvedTemperature != null @@ -577,7 +582,8 @@ export async function driveHostedEvalTurn( // opt-out and a truthy check would erase it. ...(params.modelVisibleMcpToolResults !== undefined ? { - modelVisibleMcpToolResults: params.modelVisibleMcpToolResults, + modelVisibleMcpToolResults: + params.modelVisibleMcpToolResults, } : {}), ...(params.respectToolVisibility !== undefined @@ -630,7 +636,7 @@ export async function driveHostedEvalTurn( // aborted run as a verdict failure. if (isAborted()) { logger.debug( - `[evals] backend iteration${logSuffix} aborted mid-turn; skipping record`, + `[evals] backend iteration${logSuffix} aborted mid-turn; skipping record` ); return { kind: "cancelled" }; } @@ -697,7 +703,7 @@ export async function driveHostedEvalTurn( // generic fallbacks. const failTurn = ( fallbackError: string, - logLine: string, + logLine: string ): HostedEvalTurnOutcome => { const failure = lastEngineError ? { @@ -743,7 +749,7 @@ export async function driveHostedEvalTurn( newMessages.length > 0 }, engineError=${ lastEngineError ? lastEngineError.code ?? "uncoded" : "none" - })`, + })` ); } if (newMessages.length === 0) { @@ -751,7 +757,7 @@ export async function driveHostedEvalTurn( "Backend step returned no content (stream error or empty response)", `[evals] runAssistantTurn${logSuffix} produced no new messages this turn; treating as cycle failure (engineError=${ lastEngineError ? lastEngineError.code ?? "uncoded" : "none" - })`, + })` ); } // Cursor / Codex review fix: filter to backend step / LLM failure spans @@ -764,7 +770,7 @@ export async function driveHostedEvalTurn( (span) => span.status === "error" && span.category !== "tool" && - !(span as { toolCallId?: string }).toolCallId, + !(span as { toolCallId?: string }).toolCallId ); if (stepErrorSpan) { return failTurn( @@ -773,7 +779,7 @@ export async function driveHostedEvalTurn( stepErrorSpan.name } category=${stepErrorSpan.category} engineError=${ lastEngineError ? lastEngineError.code ?? "uncoded" : "none" - })`, + })` ); } diff --git a/mcpjam-inspector/server/services/evals/finalize-iteration.ts b/mcpjam-inspector/server/services/evals/finalize-iteration.ts index e667ec09d9..bba515fc95 100644 --- a/mcpjam-inspector/server/services/evals/finalize-iteration.ts +++ b/mcpjam-inspector/server/services/evals/finalize-iteration.ts @@ -92,7 +92,7 @@ type PolicyBlockRecord = { reason?: unknown }; * summary reason when multiple policy blocks occur. */ function getIterationPolicyReason( - policyBlocks: ReadonlyArray, + policyBlocks: ReadonlyArray ): string | undefined { const reason = policyBlocks[0]?.reason; return typeof reason === "string" ? reason : undefined; @@ -260,7 +260,7 @@ export function buildStageMetadata(args: { /** A predicate row the score projection can key a criterion off. */ function isHostedPredicateResult( - value: unknown, + value: unknown ): value is HostedPredicateResultLike { if (typeof value !== "object" || value === null) return false; const row = value as { predicate?: unknown; passed?: unknown }; @@ -273,7 +273,7 @@ function isHostedPredicateResult( /** Read only the matcher fields the projection needs, typed rather than cast. */ function narrowEvaluation( - evaluation: Record, + evaluation: Record ): HostedEvaluationLike { const list = (key: string): readonly unknown[] | undefined => { const value = evaluation[key]; @@ -363,7 +363,7 @@ function buildScoreMetadata(args: { } { if (args.mode === "off") return { keys: {} }; const predicateResults = (args.predicateResults ?? []).filter( - isHostedPredicateResult, + isHostedPredicateResult ); const { scores, evaluationConfig } = buildHostedScoreContract({ ...(predicateResults.length ? { predicateResults } : {}), @@ -429,7 +429,7 @@ function buildScoreMetadata(args: { ...(typeof args.stageMetadata.stageAnalyzerVersion === "number" ? { stageAnalyzerVersion: args.stageMetadata.stageAnalyzerVersion } : {}), - }, + } ); // The emitter is only REACHED on disagreement, so a spy on it counts // mismatches rather than comparisons — that is what makes @@ -460,11 +460,7 @@ function readUserValueRow( for (const row of rows) { if (typeof row !== "object" || row === null) continue; const candidate = row as Partial; - if ( - candidate.stage === "userValue" && - candidate.state && - candidate.reason - ) { + if (candidate.stage === "userValue" && candidate.state && candidate.reason) { return { state: candidate.state, reason: candidate.reason }; } } @@ -501,14 +497,12 @@ function buildSelectionToolCatalogMetadata(args: { // and folding them in could fill the catalog's cap before the turn that // actually caused the failure is ever considered. const failingPrompts = prompts.filter( - (p) => (p.missing?.length ?? 0) > 0 || (p.unexpected?.length ?? 0) > 0, + (p) => (p.missing?.length ?? 0) > 0 || (p.unexpected?.length ?? 0) > 0 ); const expectedToolNames = failingPrompts .flatMap((p) => p.missing ?? []) .map((t) => t.toolName) - .filter( - (name): name is string => typeof name === "string" && name.length > 0, - ); + .filter((name): name is string => typeof name === "string" && name.length > 0); // `unexpected` names FIRST, then the rest of the turn's actual calls: // `buildSelectionToolCatalog`'s cap is shared across both roles, and for // an `unexpectedToolCall` failure (e.g. `maxExtraToolCalls: 0`, six @@ -523,15 +517,11 @@ function buildSelectionToolCatalogMetadata(args: { const unexpectedToolNames = failingPrompts .flatMap((p) => p.unexpected ?? []) .map((t) => t.toolName) - .filter( - (name): name is string => typeof name === "string" && name.length > 0, - ); + .filter((name): name is string => typeof name === "string" && name.length > 0); const otherActualToolNames = failingPrompts .flatMap((p) => p.actualToolCalls ?? []) .map((t) => t.toolName) - .filter( - (name): name is string => typeof name === "string" && name.length > 0, - ); + .filter((name): name is string => typeof name === "string" && name.length > 0); const actualToolNames = [...unexpectedToolNames, ...otherActualToolNames]; if (expectedToolNames.length === 0 && actualToolNames.length === 0) { return {}; @@ -722,7 +712,10 @@ export function buildIterationFinishParams(args: { selectionTools, } = args; const gradingMode = args.gradingMode ?? resolveGradingEngineMode(); - const persistedSpans = [...(setupSpans ?? []), ...(spans ?? [])]; + const persistedSpans = [ + ...(setupSpans ?? []), + ...(spans ?? []), + ]; const stageMetadata = buildStageMetadata({ ...(stageCase ? { stageCase } : {}), spans, @@ -768,13 +761,14 @@ export function buildIterationFinishParams(args: { // would silently switch D7's catalog capture back OFF for the cohort that // has progressed furthest. The predicate is what keeps "dual_write and // above" in one place. - const selectionToolCatalogMetadata = isDualWrite(gradingMode) - ? buildSelectionToolCatalogMetadata({ - stageMetadata, - prompts, - selectionTools, - }) - : {}; + const selectionToolCatalogMetadata = + isDualWrite(gradingMode) + ? buildSelectionToolCatalogMetadata({ + stageMetadata, + prompts, + selectionTools, + }) + : {}; // THE FLIP, and the ONE DIRECTION IT MAY MOVE. // @@ -1039,8 +1033,8 @@ export async function finalizeEvalIteration( iterationStatus === "cancelled" ? "eval_cancelled" : isCycleFailure - ? "eval_failed" - : "eval_completed"; + ? "eval_failed" + : "eval_completed"; // PR 13: emit per-iteration browser-eval observability from the runner-local // arrays (covers both the stream + non-stream paths via this shared choke @@ -1095,7 +1089,8 @@ export async function finalizeEvalIteration( // before any turn landed. With turns already written, re-sending // would overwrite turn 0 (W1 always writes at promptIndex: 0) and // orphan turns 1..N. See persist-eval-trace.ts for the contract. - const useW1Fallback = fanout.persisted === false && fanout.turnsWritten === 0; + const useW1Fallback = + fanout.persisted === false && fanout.turnsWritten === 0; if (fanout.persisted === false) { logger.warn( useW1Fallback @@ -1141,7 +1136,8 @@ export async function finalizeEvalIteration( : {}), ...(widgetSnapshots?.length ? { - widgetSnapshots: sanitizeForConvexTransport(widgetSnapshots), + widgetSnapshots: + sanitizeForConvexTransport(widgetSnapshots), } : {}), // PR 6b: browser artifacts already uploaded + sanitized above; diff --git a/mcpjam-inspector/server/services/evals/step-executor.ts b/mcpjam-inspector/server/services/evals/step-executor.ts index dcdac0bca7..b82d042a49 100644 --- a/mcpjam-inspector/server/services/evals/step-executor.ts +++ b/mcpjam-inspector/server/services/evals/step-executor.ts @@ -24,7 +24,10 @@ */ import type { ModelMessage } from "ai"; -import type { PredicateResult, ToolErrorRecord } from "@/shared/eval-matching"; +import type { + PredicateResult, + ToolErrorRecord, +} from "@/shared/eval-matching"; import { buildIterationTranscript, evaluatePredicates, @@ -237,7 +240,8 @@ export interface StepExecutorResult { export function hasWidgetDrivingStep(steps: TestStep[]): boolean { return steps.some( (s) => - isInteractStep(s) || (isAssertStep(s) && isWidgetAssertion(s.assertion)), + isInteractStep(s) || + (isAssertStep(s) && isWidgetAssertion(s.assertion)), ); } @@ -262,8 +266,7 @@ function applyOutcome( turn: number, ): void { if (outcome.messages?.length) state.messages.push(...outcome.messages); - if (outcome.toolCalls?.length) - recordToolCalls(state, turn, outcome.toolCalls); + if (outcome.toolCalls?.length) recordToolCalls(state, turn, outcome.toolCalls); if (outcome.toolErrors?.length) state.toolErrors.push(...outcome.toolErrors); if (outcome.usage) { state.usage.inputTokens += outcome.usage.inputTokens ?? 0; @@ -277,7 +280,9 @@ function snapshotTranscript(state: StepExecutionState) { const finalAssistantMessage = extractFinalAssistantMessage(state.messages); return buildIterationTranscript({ toolCalls: state.toolCalls, - ...(finalAssistantMessage !== undefined ? { finalAssistantMessage } : {}), + ...(finalAssistantMessage !== undefined + ? { finalAssistantMessage } + : {}), usage: state.usage.inputTokens || state.usage.outputTokens || @@ -376,11 +381,7 @@ async function drainAndDriveFollowUps( return undefined; } remaining -= 1; - const outcome = await handlers.onFollowUp!({ - text, - stepIndex, - turnOrdinal: turn, - }); + const outcome = await handlers.onFollowUp!({ text, stepIndex, turnOrdinal: turn }); applyOutcome(state, outcome, turn); if (outcome.iterationError) return outcome; } @@ -407,7 +408,8 @@ async function runAssertStep( passed: outcome.ok, reason: outcome.ok ? `widget assertion "${step.assertion.kind}" passed` - : outcome.reason ?? `widget assertion "${step.assertion.kind}" failed`, + : outcome.reason ?? + `widget assertion "${step.assertion.kind}" failed`, }); return; } diff --git a/sdk/src/contract/stage-derivation.ts b/sdk/src/contract/stage-derivation.ts index 0a5bc4b5b4..cc847edc0b 100644 --- a/sdk/src/contract/stage-derivation.ts +++ b/sdk/src/contract/stage-derivation.ts @@ -1424,7 +1424,15 @@ const PROVIDER_UNKNOWABLE_FAILURES: ReadonlyArray = [ /** * Re-label what a MODEL-CALL failure made unknowable. * - * Applied last, and in two parts. + * Applied BEFORE the positional cascade, and again in `finalize` for the + * early-return paths that never reach it — idempotent, so the second pass over + * already-converted rows finds nothing to do. An earlier revision of this + * docblock said "applied last"; that was true until the withdrawal had to move + * ahead of the cascade, which reads `failed` rows to decide which later stages + * never ran. Withdrawing after it ran left three rows citing a failure the + * chain no longer recorded. See the comment at the call site. + * + * Two parts. * * BLANK ROWS. A stage that measured nothing is re-labelled, while a stage with * its own evidence keeps its own row: the provider dying at turn 4 does not From 961f72a80ac8a11d383d5c22fe6baa8f84fe794f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 31 Aug 2026 19:41:38 +0000 Subject: [PATCH 15/15] UVH-IN5: run the BUILD gate on this stack's PRs too, not just the tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test.yml` got this stack's branch pattern when a missing base filter let a broken test ship green. `lint.yml` — "Build and Test", which is the typecheck and build gate — has the same `branches` filter on the PR's BASE and did not get the pattern, so not one of the seven stacked inspector PRs has ever been typechecked or built in CI. Everything green on them so far is previews, review bots and local runs. Same one-line fix as the tests, on the same branch, for the same reason. Both patterns come out when the stack lands. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y4jTtZJsDeaterEzKwpF2p --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c1d1b18285..1a486ed503 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,6 +7,7 @@ on: branches: - main - "claude/mcp-benchmarks-v2-b0tw4b-**" + - "claude/uvc-mcp-eval-reporting-gyycwl**" push: branches: [main]