Skip to content

Commit 05bd143

Browse files
andreiborzaandreiborza
authored
Merge branch 'develop' into ab/vercel-ai-decide-spans
2 parents c1b1d30 + 36fdc3f commit 05bd143

21 files changed

Lines changed: 489 additions & 124 deletions

File tree

‎dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
SENTRY_OP,
1111
SENTRY_ORIGIN,
1212
} from '@sentry/conventions/attributes';
13-
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
13+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
1414
import { expect, it } from 'vitest';
1515
import { createRunner } from '../../../runner';
1616
import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils';
@@ -29,28 +29,28 @@ it('traces a TypeSafeClassifier call through the Vite plugin instrumentation', a
2929

3030
expect(genAiSpans).toHaveLength(1);
3131

32-
const evaluateSpan = genAiSpans[0]!;
33-
expect(evaluateSpan.name).toBe('evaluate jev-latest');
34-
expect(evaluateSpan.parent_span_id).toBe(segmentSpan?.span_id);
35-
expect(evaluateSpan.status).toBe('ok');
36-
expect(evaluateSpan.attributes).toMatchObject({
37-
[SENTRY_OP]: { value: GEN_AI_EVALUATE, type: 'string' },
32+
const decideSpan = genAiSpans[0]!;
33+
expect(decideSpan.name).toBe('decide jev-latest');
34+
expect(decideSpan.parent_span_id).toBe(segmentSpan?.span_id);
35+
expect(decideSpan.status).toBe('ok');
36+
expect(decideSpan.attributes).toMatchObject({
37+
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
3838
[SENTRY_ORIGIN]: { value: 'auto.ai.langchain', type: 'string' },
39-
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
39+
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
4040
[GEN_AI_PROVIDER_NAME]: { value: 'typesafe', type: 'string' },
4141
[GEN_AI_REQUEST_MODEL]: { value: 'jev-latest', type: 'string' },
4242
[GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13', type: 'string' },
4343
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 30, type: 'integer' },
4444
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 2, type: 'integer' },
4545
});
46-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
46+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
4747
{
4848
type: 'evaluation',
4949
state: ['user: My payouts have been failing.'],
5050
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
5151
},
5252
]);
53-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
53+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
5454
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
5555
]);
5656
})

‎dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
SENTRY_OP,
2222
SENTRY_ORIGIN,
2323
} from '@sentry/conventions/attributes';
24+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
2425
import { SDK_VERSION } from '@sentry/core';
2526
import { expect, it } from 'vitest';
2627
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
@@ -149,14 +150,14 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
149150

150151
expect(genAiSpans[0]).toEqual(
151152
expect.objectContaining({
152-
name: 'evaluate typesafe/jev',
153+
name: 'decide typesafe/jev',
153154
status: 'ok',
154155
is_segment: false,
155156
attributes: {
156157
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
157-
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
158+
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
158159
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
159-
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
160+
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
160161
[GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' },
161162
[GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13.0', type: 'string' },
162163
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 426, type: 'integer' },
@@ -195,14 +196,14 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal })
195196

196197
expect(genAiSpans[0]).toEqual(
197198
expect.objectContaining({
198-
name: 'evaluate @cf/cloudflare/clef',
199+
name: 'decide @cf/cloudflare/clef',
199200
status: 'ok',
200201
is_segment: false,
201202
attributes: {
202203
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
203-
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
204+
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
204205
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
205-
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
206+
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
206207
[GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' },
207208
[GEN_AI_RESPONSE_MODEL]: { value: 'clef', type: 'string' },
208209
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 412, type: 'integer' },

‎dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,41 +94,41 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => {
9494
});
9595
});
9696

97-
test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => {
97+
test('Sends a Workers AI gen_ai.decide span for a TypeSafe Jev call to Sentry', async () => {
9898
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200));
9999

100-
console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
100+
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
101101

102102
let spanId: string | undefined;
103103
await expect
104-
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
104+
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
105105
.toBeDefined();
106106

107107
await expect
108108
.poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
109109
.toMatchObject({
110-
'gen_ai.operation.name': 'evaluate',
110+
'gen_ai.operation.name': 'decide',
111111
'gen_ai.request.model': 'typesafe/jev',
112112
'gen_ai.response.model': expect.stringMatching(/^jev-/),
113113
'gen_ai.input.messages': expect.stringContaining('My payouts have been failing'),
114114
'gen_ai.output.messages': expect.stringContaining('"is_urgent"'),
115115
});
116116
});
117117

118-
test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => {
118+
test('Sends a Workers AI gen_ai.decide span for a Clef call to Sentry', async () => {
119119
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200));
120120

121-
console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
121+
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
122122

123123
let spanId: string | undefined;
124124
await expect
125-
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
125+
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
126126
.toBeDefined();
127127

128128
await expect
129129
.poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
130130
.toMatchObject({
131-
'gen_ai.operation.name': 'evaluate',
131+
'gen_ai.operation.name': 'decide',
132132
'gen_ai.request.model': '@cf/cloudflare/clef',
133133
'gen_ai.response.model': expect.stringContaining('clef'),
134134
'gen_ai.input.messages': expect.stringContaining('Checkout has been failing'),

‎dev-packages/e2e-tests/test-applications/nuxt-4-sourcemaps/assert-build.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
getSourcemapSources,
1616
getSourcemaps,
1717
loadMockServerResults,
18+
startNodeServer,
1819
} from '@sentry-internal/test-utils';
1920

2021
/** This variant omits `sourcemaps.filesToDeleteAfterUpload`, so Sentry must upload but not delete. */
@@ -170,6 +171,11 @@ const debugIdsWithoutSourceMap = debugIdsOfChunksWithoutSourceMap(CLIENT_BUILD_O
170171
for (const outputDir of isStaticBuild ? [CLIENT_OUTPUT] : [CLIENT_OUTPUT, SERVER_OUTPUT]) {
171172
const injectedDebugIds = findInjectedDebugIds({ outputDir });
172173
assert.ok(injectedDebugIds.length > 0, `Expected debug IDs to be injected into ${outputDir}`);
174+
// The exemption for chunks without a source map must not let a build without any uploaded map pass.
175+
assert.ok(
176+
injectedDebugIds.some(debugId => uploadedDebugIds.has(debugId)),
177+
`Expected at least one debug ID in ${outputDir} to have an uploaded sourcemap`,
178+
);
173179

174180
const unuploaded = injectedDebugIds.filter(
175181
debugId => !uploadedDebugIds.has(debugId) && !debugIdsWithoutSourceMap.has(debugId),
@@ -232,4 +238,48 @@ if (keepClientSourceMaps) {
232238
console.log(` ${CLIENT_OUTPUT} is free of source maps, sourceMappingURL comments and original source\n`);
233239
}
234240

241+
// --- What the built server answers for the client source maps ---
242+
243+
// `nuxt generate` emits no server, and a static host serves only the files that exist.
244+
if (!isStaticBuild) {
245+
const clientChunkUrls = fs
246+
.readdirSync(CLIENT_OUTPUT, { recursive: true, withFileTypes: true })
247+
.filter(entry => entry.isFile() && entry.name.endsWith('.js'))
248+
.map(
249+
entry => `/${path.relative(CLIENT_OUTPUT, path.join(entry.parentPath, entry.name)).split(path.sep).join('/')}`,
250+
);
251+
assert.ok(clientChunkUrls.length > 0, `Expected client chunks in ${CLIENT_OUTPUT}`);
252+
253+
const server = await startNodeServer({ entry: path.join(SERVER_OUTPUT, 'index.mjs'), port: 3030 });
254+
255+
try {
256+
// Without this, a server that serves no public files at all would pass the 404 checks.
257+
const chunkStatus = (await fetch(`${server.baseUrl}${clientChunkUrls[0]}`)).status;
258+
assert.equal(chunkStatus, 200, `Expected the server to serve the client chunk ${clientChunkUrls[0]}`);
259+
260+
// Nitro serves public files from a list it writes into the server bundle. A map deleted after that
261+
// stays on the list, and Nitro answers its request with a 500 instead of a 404.
262+
const responses = await Promise.all(
263+
clientChunkUrls.map(async chunkUrl => {
264+
const mapUrl = `${chunkUrl}.map`;
265+
const expectedStatus = fs.existsSync(path.join(CLIENT_OUTPUT, mapUrl)) ? 200 : 404;
266+
const { status } = await fetch(`${server.baseUrl}${mapUrl}`);
267+
return { mapUrl, status, expectedStatus };
268+
}),
269+
);
270+
const unexpectedStatuses = responses
271+
.filter(({ status, expectedStatus }) => status !== expectedStatus)
272+
.map(({ mapUrl, status, expectedStatus }) => `${mapUrl} → ${status}, expected ${expectedStatus}`);
273+
assert.deepEqual(
274+
unexpectedStatuses,
275+
[],
276+
'Expected the server to answer each client source map URL with 200 if the map is in the output, else 404',
277+
);
278+
279+
console.log(` The server answers all ${clientChunkUrls.length} client source map URL(s) as expected\n`);
280+
} finally {
281+
await server.close();
282+
}
283+
}
284+
235285
console.log('All sourcemap assertions passed!');

‎dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import {
2020
SENTRY_OP,
2121
SENTRY_ORIGIN,
2222
} from '@sentry/conventions/attributes';
23-
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
23+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
2424
import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
2525
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
2626
import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests';
@@ -241,7 +241,7 @@ describe('LangChain integration (v1)', () => {
241241
'scenario-typesafe-classifier.mjs',
242242
'instrument-with-pii.mjs',
243243
(createRunner, test) => {
244-
test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => {
244+
test('records a TypeSafeClassifier run as a gen_ai.decide span', async () => {
245245
const runner = createRunner().ignore('event');
246246
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
247247

@@ -250,34 +250,34 @@ describe('LangChain integration (v1)', () => {
250250
const spans = (await spansPromise).filter(
251251
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain',
252252
);
253-
expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']);
254-
255-
const [evaluateSpan, messageEvaluateSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp);
256-
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
257-
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
258-
expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate');
259-
expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe');
260-
expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
261-
expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13');
262-
expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
263-
expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
264-
expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
265-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
253+
expect(spans.map(span => span.name)).toEqual(['decide jev-latest', 'decide jev-latest']);
254+
255+
const [decideSpan, messageDecideSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp);
256+
expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE);
257+
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
258+
expect(decideSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide');
259+
expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe');
260+
expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
261+
expect(decideSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13');
262+
expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
263+
expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
264+
expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
265+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
266266
{
267267
type: 'evaluation',
268268
state: 'My payouts have been failing.',
269269
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
270270
},
271271
]);
272272
// A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form.
273-
expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
273+
expect(JSON.parse(messageDecideSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
274274
{
275275
type: 'evaluation',
276276
state: 'user: My card was charged twice.',
277277
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
278278
},
279279
]);
280-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
280+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
281281
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
282282
]);
283283
});
@@ -305,7 +305,7 @@ describe('LangChain integration (v1)', () => {
305305
const spans = await spansPromise;
306306
const segment = spans.find(span => span.is_segment && span.name === 'main');
307307
expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child');
308-
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1);
308+
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)).toHaveLength(1);
309309
});
310310
},
311311
{
@@ -330,15 +330,15 @@ describe('LangChain integration (v1)', () => {
330330
await runner.start().completed();
331331

332332
const spans = await spansPromise;
333-
const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!;
333+
const decideSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)!;
334334
const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!;
335335

336336
// The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped.
337-
expect(evaluateSpan.name).toBe('evaluate jev-latest');
338-
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
339-
expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent');
340-
expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id);
341-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
337+
expect(decideSpan.name).toBe('decide jev-latest');
338+
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
339+
expect(decideSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent');
340+
expect(decideSpan.parent_span_id).toBe(chatSpan.parent_span_id);
341+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
342342
{
343343
type: 'evaluation',
344344
// The router passes the latest human message, which the classifier sends as a transcript line.
@@ -352,7 +352,7 @@ describe('LangChain integration (v1)', () => {
352352
},
353353
},
354354
]);
355-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
355+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
356356
{
357357
type: 'evaluation',
358358
answers: {

0 commit comments

Comments
 (0)