Skip to content

Commit e934b44

Browse files
mydeaclaude
mydea
and
claude
authored
feat(server-utils): Add orchestrion support for Vercel AI v4 (#22139)
Extends the orchestrion diagnostics-channel instrumentation to cover the `ai` SDK **v4** (`>=4.0.0 <5.0.0`), so the channel path now covers v4/v5/v6/v7 and emits spans identical to the OTel processor. This closes the last version gap before the OTel `SentryVercelAiInstrumentation` can eventually be retired. v4 differs from v5/v6 in three ways, all handled by reusing the existing shared span-building core: - **No `resolveLanguageModel` chokepoint** — v4 calls the passed `LanguageModelV1` (`specificationVersion: 'v1'`) directly, so its `doGenerate`/`doStream` are patched at the operation start instead, gated on `'v1'` so v5/v6 models are never touched (strictly additive). Tool calls reuse the existing per-tool `execute` patch, and the model-call tool list is read from v4's `mode.tools`. - **Token names** — the shared core now reads `promptTokens`/`completionTokens` (v4) alongside `inputTokens`/`outputTokens` (v5+); the fallbacks are `undefined`-inert on v5+. - **Streamed text delta** — `accumulateChunk` reads `textDelta` (v4) alongside `delta` (v5+). The top-level `system` prompt is also lifted into `instructions` for v4/v5/v6, matching v7's native channel. The v4 integration-test suites (`vercelai` and `span-streaming-v4`) now run under orchestrion as well as the OTel processor, and a new `streamText` scenario covers the v4 `doStream` path. Fixes #20914 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 04bbb79 commit e934b44

11 files changed

Lines changed: 400 additions & 144 deletions

File tree

‎dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import * as Sentry from '@sentry/node';
2+
import { streamText } from 'ai';
3+
import { convertArrayToReadableStream, MockLanguageModelV1 } from 'ai/test';
4+
5+
async function run() {
6+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
7+
const result = streamText({
8+
experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true },
9+
model: new MockLanguageModelV1({
10+
doStream: async () => ({
11+
stream: convertArrayToReadableStream([
12+
{ type: 'response-metadata', id: 'resp-1', modelId: 'mock-model-id' },
13+
{ type: 'text-delta', textDelta: 'Stream ' },
14+
{ type: 'text-delta', textDelta: 'response!' },
15+
{
16+
type: 'finish',
17+
finishReason: 'stop',
18+
usage: { promptTokens: 10, completionTokens: 20 },
19+
},
20+
]),
21+
rawCall: { rawPrompt: null, rawSettings: {} },
22+
}),
23+
}),
24+
prompt: 'Stream me a response',
25+
});
26+
27+
// streamText returns synchronously; drive the lazy stream to completion so the spans finish.
28+
for await (const _part of result.textStream) {
29+
void _part;
30+
}
31+
});
32+
33+
await Sentry.flush(2000);
34+
}
35+
36+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts‎

Lines changed: 25 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,15 @@ function attr(value: unknown) {
3030
return expect.objectContaining({ value });
3131
}
3232

33-
describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)', () => {
33+
const expectedOrigin = isOrchestrionEnabled() ? 'auto.vercelai.channel' : 'auto.vercelai.otel';
34+
35+
// v4's OTel path serializes tool-call arguments from the provider's raw JSON string (whitespace
36+
// preserved); the channel path serializes the SDK-parsed args object (compact). Same data, different spacing.
37+
const toolCallArgs = isOrchestrionEnabled()
38+
? '{\\"location\\":\\"San Francisco\\"}'
39+
: '{ \\"location\\": \\"San Francisco\\" }';
40+
41+
describe('Vercel AI integration (streaming v4)', () => {
3442
afterAll(() => {
3543
cleanupChildProcesses();
3644
});
@@ -49,9 +57,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
4957
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
5058
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
5159
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
52-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
53-
'vercel.ai.pipeline.name': attr('generateText'),
54-
'vercel.ai.streaming': attr(false),
60+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
5561
}),
5662
}),
5763
// Second span - generate_content for simple generateText
@@ -66,9 +72,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
6672
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
6773
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
6874
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
69-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
70-
'vercel.ai.pipeline.name': attr('generateText.doGenerate'),
71-
'vercel.ai.streaming': attr(false),
75+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
7276
}),
7377
}),
7478
// Third span - invoke_agent for explicit telemetry generateText
@@ -82,7 +86,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
8286
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
8387
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
8488
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
85-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
89+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
8690
}),
8791
}),
8892
// Fourth span - tool call invoke_agent
@@ -96,7 +100,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
96100
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
97101
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
98102
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
99-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
103+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
100104
}),
101105
}),
102106
// Fifth span - tool call generate_content
@@ -110,7 +114,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
110114
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
111115
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
112116
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
113-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
117+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
114118
}),
115119
}),
116120
// Sixth span - execute_tool
@@ -124,7 +128,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
124128
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
125129
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
126130
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
127-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
131+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
128132
}),
129133
}),
130134
]),
@@ -149,9 +153,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
149153
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
150154
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
151155
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
152-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
153-
'vercel.ai.pipeline.name': attr('generateText'),
154-
'vercel.ai.streaming': attr(false),
156+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
155157
}),
156158
}),
157159
// Second span - generate_content with input/output messages
@@ -170,7 +172,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
170172
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
171173
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
172174
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
173-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
175+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
174176
}),
175177
}),
176178
// Third span - explicit telemetry invoke_agent with messages
@@ -188,7 +190,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
188190
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30),
189191
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
190192
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
191-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
193+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
192194
}),
193195
}),
194196
// Fourth span - tool call invoke_agent with messages
@@ -200,15 +202,15 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
200202
'[{"role":"user","content":"What is the weather in San Francisco?"}]',
201203
),
202204
[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr(
203-
'[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]',
205+
`[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`,
204206
),
205207
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'),
206208
[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15),
207209
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25),
208210
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
209211
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
210212
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
211-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
213+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
212214
}),
213215
}),
214216
// Fifth span - tool call generate_content with available_tools
@@ -225,7 +227,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
225227
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
226228
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
227229
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
228-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
230+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
229231
}),
230232
}),
231233
// Sixth span - execute_tool with description and input/output
@@ -241,7 +243,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
241243
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
242244
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
243245
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
244-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
246+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
245247
}),
246248
}),
247249
]),
@@ -255,7 +257,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
255257
attributes: expect.objectContaining({
256258
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'),
257259
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'),
258-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
260+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
259261
}),
260262
}),
261263
expect.objectContaining({
@@ -268,7 +270,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
268270
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40),
269271
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'),
270272
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'),
271-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
273+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
272274
}),
273275
}),
274276
expect.objectContaining({
@@ -280,7 +282,7 @@ describe.skipIf(isOrchestrionEnabled())('Vercel AI integration (streaming v4)',
280282
[GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'),
281283
[GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'),
282284
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'),
283-
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'),
285+
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr(expectedOrigin),
284286
}),
285287
}),
286288
]),

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test-generate-object.ts‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

0 commit comments

Comments
 (0)