feat(server-utils): Record Vercel AI experimental_decide calls as gen_ai.decide spans

dev-packages/e2e-tests/test-applications/node-eve/agent/tools/classify_ticket.ts

@@ -1,14 +1,14 @@
-import { experimental_evaluate } from 'ai';
-import { Experimental_EvaluationMockModelV4 } from 'ai/test';
+import { experimental_decide } from 'ai';
+import { Experimental_DecisionMockModelV4 } from 'ai/test';
 import { defineTool } from 'eve/tools';
 import { z } from 'zod';
 
-// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.evaluate` span. The model is
+// Runs a TypeSafe Jev evaluation so the e2e test can assert its `gen_ai.decide` span. The model is
 // mocked because the e2e environment has no TypeSafe credentials.
-const jev = new Experimental_EvaluationMockModelV4({
+const jev = new Experimental_DecisionMockModelV4({
   provider: 'gateway',
   modelId: 'typesafe-ai/jev',
-  doEvaluate: async () => ({
+  doDecide: async () => ({
     answers: {
       wantsRefund: { type: 'boolean', probability: 0.99 },
       department: {
@@ -26,7 +26,7 @@ export default defineTool({
   description: 'Classify a support ticket. Call this when asked to classify a ticket.',
   inputSchema: z.object({ ticket: z.string().min(1) }),
   async execute({ ticket }) {
-    const { answers } = await experimental_evaluate({
+    const { answers } = await experimental_decide({
       model: jev,
       state: ticket,
       questions: {

dev-packages/e2e-tests/test-applications/node-eve/package.json

@@ -32,6 +32,7 @@
   "devDependencies": {
     "@playwright/test": "~1.56.0",
     "@sentry-internal/test-utils": "link:../../../test-utils",
+    "@sentry/conventions": "0.27.0",
     "@sentry/core": "file:../../packed/sentry-core-packed.tgz",
     "@types/node": "24.x",
     "typescript": "~5.9.0"

dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts

@@ -1,4 +1,15 @@
 import { expect, test } from '@playwright/test';
+import {
+  GEN_AI_CONVERSATION_ID,
+  GEN_AI_INPUT_MESSAGES,
+  GEN_AI_OPERATION_NAME,
+  GEN_AI_REQUEST_MODEL,
+  GEN_AI_TOOL_NAME,
+  GEN_AI_USAGE_INPUT_TOKENS,
+  GEN_AI_USAGE_OUTPUT_TOKENS,
+  SENTRY_ORIGIN,
+} from '@sentry/conventions/attributes';
+import { GEN_AI_DECIDE, GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op';
 import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils';
 import { runAgentTurn } from './utils';
 
@@ -135,9 +146,9 @@ test('captures errors thrown inside an eve tool', async ({ baseURL }) => {
   });
 });
 
-test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async ({ baseURL }) => {
+test('captures a gen_ai.decide span for a Jev call inside an eve tool', async ({ baseURL }) => {
   const traceSpansPromise = collectStreamedSpans(APP, spansOfTrace =>
-    ['gen_ai.execute_tool', 'gen_ai.evaluate'].every(op => spansOfTrace.some(span => getSpanOp(span) === op)),
+    [GEN_AI_EXECUTE_TOOL, GEN_AI_DECIDE].every(op => spansOfTrace.some(span => getSpanOp(span) === op)),
   );
 
   const sessionId = await runAgentTurn(
@@ -147,21 +158,21 @@ test('captures a gen_ai.evaluate span for a Jev call inside an eve tool', async
 
   const traceSpans = await traceSpansPromise;
 
-  const executeTool = traceSpans.find(span => getSpanOp(span) === 'gen_ai.execute_tool');
-  const evaluate = traceSpans.find(span => getSpanOp(span) === 'gen_ai.evaluate');
-
-  expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('classify_ticket');
-
-  expect(evaluate?.name).toBe('evaluate typesafe-ai/jev');
-  expect(evaluate?.status).toBe('ok');
-  expect(evaluate?.attributes?.['sentry.origin']?.value).toBe('auto.vercelai.channel');
-  expect(evaluate?.attributes?.['gen_ai.operation.name']?.value).toBe('evaluate');
-  expect(evaluate?.attributes?.['gen_ai.request.model']?.value).toBe('typesafe-ai/jev');
-  expect(evaluate?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe(275);
-  expect(evaluate?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe(20);
-  expect(evaluate?.attributes?.['gen_ai.input.messages']?.value).toContain('refund for last month');
-  expect(evaluate?.attributes?.['gen_ai.conversation.id']?.value).toBe(sessionId);
-
-  expect(evaluate?.trace_id).toBe(executeTool?.trace_id);
-  expect(evaluate?.parent_span_id).toBe(executeTool?.span_id);
+  const executeTool = traceSpans.find(span => getSpanOp(span) === GEN_AI_EXECUTE_TOOL);
+  const decide = traceSpans.find(span => getSpanOp(span) === GEN_AI_DECIDE);
+
+  expect(executeTool?.attributes?.[GEN_AI_TOOL_NAME]?.value).toBe('classify_ticket');
+
+  expect(decide?.name).toBe('decide typesafe-ai/jev');
+  expect(decide?.status).toBe('ok');
+  expect(decide?.attributes?.[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel');
+  expect(decide?.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('decide');
+  expect(decide?.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev');
+  expect(decide?.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275);
+  expect(decide?.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20);
+  expect(decide?.attributes?.[GEN_AI_INPUT_MESSAGES]?.value).toContain('refund for last month');
+  expect(decide?.attributes?.[GEN_AI_CONVERSATION_ID]?.value).toBe(sessionId);
+
+  expect(decide?.trace_id).toBe(executeTool?.trace_id);
+  expect(decide?.parent_span_id).toBe(executeTool?.span_id);
 });

dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs → dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-decide.mjs

@@ -1,14 +1,14 @@
 import * as Sentry from '@sentry/node';
-import { experimental_evaluate } from 'ai';
-import { Experimental_EvaluationMockModelV4 } from 'ai/test';
+import { experimental_decide } from 'ai';
+import { Experimental_DecisionMockModelV4 } from 'ai/test';
 
 async function run() {
   await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
-    await experimental_evaluate({
-      model: new Experimental_EvaluationMockModelV4({
+    await experimental_decide({
+      model: new Experimental_DecisionMockModelV4({
         provider: 'gateway',
         modelId: 'typesafe-ai/jev',
-        doEvaluate: async () => ({
+        doDecide: async () => ({
           answers: {
             authIssue: { type: 'boolean', probability: 0.97 },
             department: {

dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts

@@ -26,6 +26,7 @@ import {
   SENTRY_OP,
   SENTRY_ORIGIN,
 } from '@sentry/conventions/attributes';
+import { GEN_AI_DECIDE } from '@sentry/conventions/op';
 import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
 import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner';
 
@@ -1275,34 +1276,32 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe
   );
 });
 
-describe('Vercel AI integration experimental_evaluate', () => {
+describe('Vercel AI integration experimental_decide', () => {
   afterAll(() => {
     cleanupChildProcesses();
   });
 
   createEsmTests(
     __dirname,
-    'scenario-evaluate.mjs',
-    'instrument-evaluate.mjs',
+    'scenario-decide.mjs',
+    'instrument-decide.mjs',
     (createRunner, test) => {
-      test('creates an evaluate span', async () => {
+      test('creates a decide span', async () => {
         await createRunner()
           .expect({
             span: container => {
-              const evaluateSpan = container.items.find(
-                span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
-              )!;
-              expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev');
-              expect(evaluateSpan.status).toBe('ok');
-              expect(evaluateSpan.attributes[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel');
-              expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate');
-              expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway');
-              expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev');
-              expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev');
-              expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275);
-              expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20);
-              expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295);
-              expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([
+              const decideSpan = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)!;
+              expect(decideSpan.name).toBe('decide typesafe-ai/jev');
+              expect(decideSpan.status).toBe('ok');
+              expect(decideSpan.attributes[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel');
+              expect(decideSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('decide');
+              expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway');
+              expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev');
+              expect(decideSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev');
+              expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275);
+              expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20);
+              expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295);
+              expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([
                 {
                   type: 'evaluation',
                   state: 'I cannot log in, and I also want a refund for last month.',
@@ -1322,7 +1321,7 @@ describe('Vercel AI integration experimental_evaluate', () => {
                   },
                 },
               ]);
-              expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([
+              expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([
                 {
                   type: 'evaluation',
                   answers: {
@@ -1346,28 +1345,26 @@ describe('Vercel AI integration experimental_evaluate', () => {
     },
     {
       additionalDependencies: {
-        ai: '^7.0.111',
+        ai: '^7.0.128',
       },
     },
   );
 
   createEsmTests(
     __dirname,
-    'scenario-evaluate.mjs',
-    'instrument-evaluate-no-recording.mjs',
+    'scenario-decide.mjs',
+    'instrument-decide-no-recording.mjs',
     (createRunner, test) => {
       test('does not record inputs or outputs when recording is off', async () => {
         await createRunner()
           .expect({
             span: container => {
-              const evaluateSpan = container.items.find(
-                span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
-              )!;
-              expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
-              expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
+              const decideSpan = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)!;
+              expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
+              expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
               // State, questions and answers must not come back through another attribute. Only the attributes
               // are checked (timestamps could match a number), and `probabilities` only occurs in answers.
-              expect(JSON.stringify(evaluateSpan.attributes)).not.toMatch(
+              expect(JSON.stringify(decideSpan.attributes)).not.toMatch(
                 /cannot log in|Charges and refunds|probabilities/,
               );
             },
@@ -1378,7 +1375,7 @@ describe('Vercel AI integration experimental_evaluate', () => {
     },
     {
       additionalDependencies: {
-        ai: '^7.0.111',
+        ai: '^7.0.128',
       },
     },
   );

packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts

@@ -28,8 +28,8 @@ import {
   SENTRY_ORIGIN,
 } from '@sentry/conventions/attributes';
 import {
+  GEN_AI_DECIDE,
   GEN_AI_EMBEDDINGS,
-  GEN_AI_EVALUATE,
   GEN_AI_EXECUTE_TOOL,
   GEN_AI_GENERATE_CONTENT,
   GEN_AI_INVOKE_AGENT,
@@ -71,8 +71,7 @@ const ORIGIN = 'auto.vercelai.channel';
 const GEN_AI_OPERATION_SPAN_OPS = {
   embeddings: GEN_AI_EMBEDDINGS,
   rerank: GEN_AI_RERANK,
-  // oxlint-disable-next-line typescript/no-deprecated -- the rename to `gen_ai.decide` comes in a follow-up
-  evaluate: GEN_AI_EVALUATE,
+  decide: GEN_AI_DECIDE,
   invoke_agent: GEN_AI_INVOKE_AGENT,
   execute_tool: GEN_AI_EXECUTE_TOOL,
   // The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than
@@ -488,7 +487,7 @@ export function createSpanFromMessage(
     // `ai` 7.0.128 renamed `experimental_evaluate` to `experimental_decide`; older 7.x still publishes the old name.
     case 'experimental_evaluate':
     case 'experimental_decide':
-      return startGenAiSpan('evaluate', modelId, {
+      return startGenAiSpan('decide', modelId, {
         ...baseAttributes,
         ...(recordInputs
           ? {
@@ -703,7 +702,7 @@ function getOutputMessages(
 
 /**
  * The AI SDK TypeSafe provider moves each answer's `confidence` out of the answers into
- * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so evaluate answers keep it.
+ * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so decide answers keep it.
  */
 function withProviderConfidence(result: Record<string, unknown>): unknown {
   const { answers, providerMetadata } = result;

packages/server-utils/test/integrations/vercel-ai/evaluate.test.ts → packages/server-utils/test/integrations/vercel-ai/decide.test.ts

@@ -1,12 +1,13 @@
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { GEN_AI_DECIDE } from '@sentry/conventions/op';
 import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core';
 import type { Span } from '@sentry/core';
 import { createSpanFromMessage, enrichSpanOnEnd } from '../../../src/integrations/vercel-ai/vercel-ai-dc-subscriber';
 import { getDefaultTestClientOptions, TestClient } from '../../mocks/client';
 
 type Message = Parameters<typeof createSpanFromMessage>[0];
 
-describe('Vercel AI evaluate spans', () => {
+describe('Vercel AI decide spans', () => {
   let endedSpans: Span[];
 
   beforeEach(() => {
@@ -28,7 +29,7 @@ describe('Vercel AI evaluate spans', () => {
   });
 
   // `ai` 7.0.128 renamed the channel event from `experimental_evaluate` to `experimental_decide`.
-  it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates an evaluate span for `%s`', type => {
+  it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates a decide span for `%s`', type => {
     const message = {
       type,
       event: {
@@ -48,8 +49,8 @@ describe('Vercel AI evaluate spans', () => {
     span.end();
 
     const json = spanToStaticSpanJSON(endedSpans[0]!);
-    expect(json.description).toBe('evaluate typesafe-ai/jev');
-    expect(json.op).toBe('gen_ai.evaluate');
+    expect(json.description).toBe('decide typesafe-ai/jev');
+    expect(json.op).toBe(GEN_AI_DECIDE);
     expect(JSON.parse(json.data?.['gen_ai.output.messages'] as string)).toEqual([
       { type: 'evaluation', answers: { wantsRefund: { type: 'boolean', probability: 0.99 } } },
     ]);