diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts index 6eee9bb68359..b5e37fdf1856 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -9,20 +9,20 @@ import { SENTRY_ORIGIN, URL_FULL, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils'; const APP = 'node-mastra'; const TICKET = 'My checkout page shows a blank screen after I click Pay.'; -const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE; +const isDecideSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_DECIDE; const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify'); -test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { +test('captures a Mastra classifier (Jev) evaluation as a gen_ai.decide span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( APP, - spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan), + spansOfTrace => spansOfTrace.some(isDecideSpan) && spansOfTrace.some(isClassifyServerSpan), ); const res = await fetch(`${baseURL}/classify`, { @@ -38,18 +38,18 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the // Vercel AI integration does not see it. - const evaluateSpans = spans.filter(isEvaluateSpan); - expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([ - { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' }, + const decideSpans = spans.filter(isDecideSpan); + expect(decideSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([ + { name: 'decide typesafe/jev-1.13', origin: 'auto.ai.mastra' }, ]); - const evaluateSpan = evaluateSpans[0]!; - expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id); - expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); - expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13'); - expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); - expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); - expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ + const decideSpan = decideSpans[0]!; + expect(decideSpan.parent_span_id).toBe(serverSpan.span_id); + expect(decideSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('decide'); + expect(decideSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13'); + expect(decideSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(decideSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); + expect(JSON.parse(String(decideSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ { type: 'evaluation', state: { ticket: TICKET }, @@ -59,7 +59,7 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', }, ]); // The probability comes from the live model, so only its shape is stable. - expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([ + expect(JSON.parse(String(decideSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([ { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } }, ]); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index d1e4309ebded..0c3778daa368 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -20,7 +20,7 @@ import { SENTRY_ORIGIN, URL_FULL, } from '@sentry/conventions/attributes'; -import { GEN_AI_CHAT, GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { supports } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -221,25 +221,25 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { 'scenario-classifier.mjs', 'instrument.mjs', (createRunner, test) => { - test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { + test('maps a classifier evaluation to a gen_ai.decide span', async () => { await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); - expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); - - const evaluateSpan = spans[0]!; - expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); - expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); - expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); - expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); - expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); - expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); - expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); - expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); - expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + expect(spans.map(span => span.name)).toEqual(['decide jev-latest']); + + const decideSpan = spans[0]!; + expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE); + expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); + expect(decideSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide'); + expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); + expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); }, }) .start() @@ -259,8 +259,8 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); - const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!; - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + const decideSpan = container.items.find(span => span.name === 'decide jev-latest')!; + expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', state: { message: 'I was charged twice.', apiKey: '[REDACTED]' }, @@ -269,7 +269,7 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, }, ]); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', answers: { urgent: { type: 'boolean', probability: 0.9 } } }, ]); }, @@ -293,12 +293,12 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { span: container => { expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); - expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); + expect(spans.map(span => span.name)).toEqual(['decide jev-latest']); - const evaluateSpan = spans[0]!; - expect(evaluateSpan.status).toBe('error'); - expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); - expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + const decideSpan = spans[0]!; + expect(decideSpan.status).toBe('error'); + expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); + expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); }, }) .start() diff --git a/packages/server-utils/src/ai/mastra/constants.ts b/packages/server-utils/src/ai/mastra/constants.ts index 4873ca830cd4..ff875db3b201 100644 --- a/packages/server-utils/src/ai/mastra/constants.ts +++ b/packages/server-utils/src/ai/mastra/constants.ts @@ -1,7 +1,7 @@ import { GEN_AI_CHAT, + GEN_AI_DECIDE, GEN_AI_EMBEDDINGS, - GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT, } from '@sentry/conventions/op'; @@ -42,8 +42,7 @@ export const SPAN_TYPE_OPS: Readonly = new Set([ diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 844cc9e82514..d2b897a59a55 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -27,8 +27,8 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, + GEN_AI_DECIDE, GEN_AI_EMBEDDINGS, - GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT, } from '@sentry/conventions/op'; @@ -165,11 +165,11 @@ describe('SentryMastraExporter', () => { ); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.description).toBe('evaluate jev-1'); + expect(json.description).toBe('decide jev-1'); expect(json.data).toMatchObject({ - [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_OP]: GEN_AI_DECIDE, [SENTRY_ORIGIN]: 'auto.ai.mastra', - [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_OPERATION_NAME]: 'decide', [GEN_AI_REQUEST_MODEL]: 'jev-1', [GEN_AI_PROVIDER_NAME]: 'typesafe.evaluation', [GEN_AI_USAGE_INPUT_TOKENS]: 30, @@ -187,7 +187,7 @@ describe('SentryMastraExporter', () => { expect(endedSpans).toHaveLength(0); expect(call.endTime).toEqual(span.endTime); - expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1'); + expect(spanToStaticSpanJSON(call.span!).description).toBe('decide jev-1'); }); it('passes its own recording options to the classifier evaluation call', async () => {