Skip to content

Commit 94a16ac

Browse files
author
jariy17
committed
test(eval): ondemand simulate uses matchGolden committed goldens (runtime-handler pattern)
Switch the golden assertions from bun toMatchSnapshot to matchGolden + committed __fixtures__/*.golden.json, matching the runtime handlers' golden style. Driven by TestCoreClient: fixtureFactories record/replay can't key this command because its inputs carry random per-session UUIDs + now-based CloudWatch windows (fixturePath hashes the input), so replay never matches — the same reason batch simulate uses TestCoreClient.
1 parent 8ff2657 commit 94a16ac

4 files changed

Lines changed: 37 additions & 32 deletions

File tree

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[
2+
{
3+
"context": {
4+
"spanContext": {
5+
"sessionId": "s1"
6+
}
7+
},
8+
"assertions": [
9+
{
10+
"text": "polite"
11+
}
12+
],
13+
"expectedTrajectory": {
14+
"toolNames": [
15+
"lookup"
16+
]
17+
}
18+
}
19+
]
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"sessionsRequested": 2,
3+
"sessionsEvaluated": 2,
4+
"results": [],
5+
"examplesInvoked": 2,
6+
"examplesFailed": 0
7+
}

src/handlers/eval/ondemand/simulate/__snapshots__/simulate.test.tsx.snap

Lines changed: 0 additions & 23 deletions
This file was deleted.

src/handlers/eval/ondemand/simulate/simulate.test.tsx

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
import { test, expect, describe } from "bun:test";
2+
import { join } from "node:path";
23
import { createRootHandler } from "../../../index";
34
import {
45
createSilentLogger,
6+
matchGolden,
57
TestCoreClient,
68
testIO,
79
TestGlobalConfigAccessor,
810
} from "../../../../testing";
911
import type { EvaluateResult, InvokeDatasetResult, SessionTrace } from "../../types";
1012

13+
const FIXTURES = join(import.meta.dir, "__fixtures__");
14+
1115
// Two invoked sessions: e1 carries session-level ground truth (assertions + trajectory),
1216
// e2 carries none. The handler adapts these to EvaluationReferenceInput[] for evaluate.
1317
const INVOKE_RESULT: InvokeDatasetResult = {
@@ -163,18 +167,16 @@ describe("eval ondemand simulate", () => {
163167
const evaluate = core.eval.calls.find((c) => c.method === "evaluate");
164168
expect(evaluate).toBeDefined();
165169
const input = evaluate!.args[0] as { groundTruth: unknown };
166-
expect(input.groundTruth).toMatchSnapshot();
170+
matchGolden(
171+
FIXTURES,
172+
"ondemand-simulate-groundtruth.golden.json",
173+
JSON.stringify(input.groundTruth, null, 2),
174+
);
167175
});
168176

169-
test("renders scores inline with invoked/failed counts", async () => {
177+
test("renders scores inline with invoked/failed counts [golden]", async () => {
170178
const { stdout } = await run(BASE);
171-
expect(JSON.parse(stdout)).toEqual({
172-
sessionsRequested: 2,
173-
sessionsEvaluated: 2,
174-
results: [],
175-
examplesInvoked: 2,
176-
examplesFailed: 0,
177-
});
179+
matchGolden(FIXTURES, "ondemand-simulate-output.golden.json", stdout);
178180
});
179181

180182
test("refuses to evaluate when nothing was invoked", async () => {

0 commit comments

Comments
 (0)