Skip to content

Commit 32e7a35

Browse files
andreiborzaclaude
andauthored
test(e2e): Cover Vercel AI decisions in node-openai-decisions-send-to-sentry (#25458)
## What The OpenAI decisions e2e app now also makes a decision through the Vercel AI SDK with `openai.decisionModel()`, and checks the `gen_ai.decide` span in Sentry, including the confidence of the answers. ## Why The Vercel AI SDK calls the Decisions API without the `openai` client, so only the Vercel AI integration records it. This app already has the OpenAI decisions key, so no CI change is needed. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 7989c8b commit 32e7a35

3 files changed

Lines changed: 97 additions & 3 deletions

File tree

‎dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "node-openai-decisions-send-to-sentry",
3-
"description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project",
3+
"description": "The OpenAI Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions requests against OpenAI through the openai client and the Vercel AI SDK, and the gen_ai.decide spans read back from a real Sentry project",
44
"version": "1.0.0",
55
"private": true,
66
"type": "module",
@@ -10,13 +10,16 @@
1010
"test": "playwright test",
1111
"clean": "npx rimraf node_modules dist pnpm-lock.yaml",
1212
"test:build": "pnpm install && pnpm build",
13-
"test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build",
13+
"test:build-latest": "pnpm install && pnpm add openai@latest ai@latest @ai-sdk/openai@latest && pnpm build",
1414
"test:assert": "pnpm test"
1515
},
1616
"dependencies": {
17+
"@ai-sdk/openai": "4.0.95",
1718
"@sentry/node": "file:../../packed/sentry-node-packed.tgz",
19+
"ai": "7.0.142",
1820
"express": "^4.21.2",
19-
"openai": "7.32.0"
21+
"openai": "7.32.0",
22+
"zod": "^4.1.8"
2023
},
2124
"devDependencies": {
2225
"@playwright/test": "1.64.1",

‎dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here.
2+
import { createOpenAI } from '@ai-sdk/openai';
23
import * as Sentry from '@sentry/node';
4+
import { experimental_decide } from 'ai';
35
import express from 'express';
46
import OpenAI from 'openai';
57

@@ -11,6 +13,7 @@ if (!apiKey) {
1113
}
1214

1315
const client = new OpenAI({ apiKey });
16+
const aiSdkOpenAI = createOpenAI({ apiKey });
1417

1518
const MODEL = 'gpt-6-luna';
1619

@@ -74,6 +77,29 @@ app.get('/decide-image', async (_req, res, next) => {
7477
}
7578
});
7679

80+
// The same decision through the Vercel AI SDK, which calls the Decisions API itself rather than through
81+
// the `openai` client, so the Vercel AI integration records it.
82+
app.get('/vercel-ai-decide', async (_req, res, next) => {
83+
try {
84+
const { answers } = await experimental_decide({
85+
model: aiSdkOpenAI.decisionModel(MODEL),
86+
state: 'I was charged twice this month and want a refund.',
87+
questions: {
88+
wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
89+
department: {
90+
type: 'choice',
91+
instructions: 'Which team should handle this?',
92+
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
93+
},
94+
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
95+
},
96+
});
97+
res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers });
98+
} catch (error) {
99+
next(error);
100+
}
101+
});
102+
77103
app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
78104
res.status(500).send({ message: error.message });
79105
});

‎dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,3 +157,68 @@ test('records an image decision with its image data', async ({ baseURL }) => {
157157
{ type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } },
158158
]);
159159
});
160+
161+
test('records a Vercel AI decision with the confidence of its answers', async ({ baseURL }) => {
162+
const response = await fetch(`${baseURL}/vercel-ai-decide`);
163+
const body = await response.json();
164+
expect(response.status, JSON.stringify(body)).toBe(200);
165+
const { traceId }: { traceId: string } = body;
166+
167+
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
168+
169+
let spanId: string | undefined;
170+
await expect
171+
.poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS)
172+
.toBeDefined();
173+
174+
let attributes: Record<string, unknown> | undefined;
175+
await expect
176+
.poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS)
177+
.toMatchObject({
178+
'span.name': `decide ${MODEL}`,
179+
[SEARCH_SPAN__OP]: GEN_AI_DECIDE,
180+
[SEARCH_SPAN__STATUS]: 'ok',
181+
[SEARCH_ORIGIN]: 'auto.vercelai.channel',
182+
[SEARCH_GEN_AI__OPERATION__NAME]: 'decide',
183+
[SEARCH_GEN_AI__PROVIDER__NAME]: 'openai.decision',
184+
[SEARCH_GEN_AI__REQUEST__MODEL]: MODEL,
185+
});
186+
187+
expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([
188+
{
189+
type: 'evaluation',
190+
state: 'I was charged twice this month and want a refund.',
191+
questions: {
192+
wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
193+
department: {
194+
type: 'choice',
195+
instructions: 'Which team should handle this?',
196+
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
197+
},
198+
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
199+
},
200+
},
201+
]);
202+
203+
// The OpenAI provider moves `confidence` into its provider metadata; the span must keep it on the answers.
204+
expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([
205+
{
206+
type: 'evaluation',
207+
answers: {
208+
wantsRefund: { type: 'boolean', probability: expect.any(Number) },
209+
department: {
210+
type: 'choice',
211+
choice: expect.stringMatching(/^(billing|other)$/),
212+
probabilities: { billing: expect.any(Number), other: expect.any(Number) },
213+
confidence: expect.any(Number),
214+
},
215+
urgency: {
216+
type: 'score',
217+
score: expect.any(Number),
218+
probabilities: { 0: expect.any(Number), 1: expect.any(Number) },
219+
confidence: expect.any(Number),
220+
},
221+
},
222+
},
223+
]);
224+
});

0 commit comments

Comments
 (0)