Skip to content

feat(server-utils): Map Mastra classifier evaluations to gen_ai.decide spans #25281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andreiborza merged 2 commits into from
Oct 12, 2026
Merged

feat(server-utils): Map Mastra classifier evaluations to gen_ai.decide spans #25281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81ccf1c
feat(server-utils): Map Mastra classifier evaluations to gen_ai.decid…
andreiborza Oct 12, 2026
0471e64
Merge branch 'develop' into ab/mastra-decide-spans
andreiborza Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,20 @@ import {
SENTRY_ORIGIN,
URL_FULL,
} from '@sentry/conventions/attributes';
import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op';
import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op';
import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils';

const APP = 'node-mastra';
const TICKET = 'My checkout page shows a blank screen after I click Pay.';

const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE;
const isDecideSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_DECIDE;
const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean =>
getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify');

test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => {
test('captures a Mastra classifier (Jev) evaluation as a gen_ai.decide span', async ({ baseURL }) => {
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan),
spansOfTrace => spansOfTrace.some(isDecideSpan) && spansOfTrace.some(isClassifyServerSpan),
);

const res = await fetch(`${baseURL}/classify`, {
Expand All @@ -38,18 +38,18 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span',

// Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the
// Vercel AI integration does not see it.
const evaluateSpans = spans.filter(isEvaluateSpan);
expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([
{ name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' },
const decideSpans = spans.filter(isDecideSpan);
expect(decideSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([
{ name: 'decide typesafe/jev-1.13', origin: 'auto.ai.mastra' },
]);

const evaluateSpan = evaluateSpans[0]!;
expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id);
expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate');
expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13');
expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter');
expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0);
expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([
const decideSpan = decideSpans[0]!;
expect(decideSpan.parent_span_id).toBe(serverSpan.span_id);
expect(decideSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('decide');
expect(decideSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13');
expect(decideSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter');
expect(decideSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0);
expect(JSON.parse(String(decideSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([
{
type: 'evaluation',
state: { ticket: TICKET },
Expand All @@ -59,7 +59,7 @@ test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span',
},
]);
// The probability comes from the live model, so only its shape is stable.
expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([
expect(JSON.parse(String(decideSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([
{ type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } },
]);
});
46 changes: 23 additions & 23 deletions dev-packages/node-integration-tests/suites/tracing/mastra/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import {
SENTRY_ORIGIN,
URL_FULL,
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT, GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
import { afterAll, describe, expect } from 'vitest';
import { supports } from '../../../utils';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';
Expand Down Expand Up @@ -221,25 +221,25 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
'scenario-classifier.mjs',
'instrument.mjs',
(createRunner, test) => {
test('maps a classifier evaluation to a gen_ai.evaluate span', async () => {
test('maps a classifier evaluation to a gen_ai.decide span', async () => {
await createRunner()
.expect({
span: container => {
expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined();
const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra');
expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']);

const evaluateSpan = spans[0]!;
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra');
expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate');
expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation');
expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
expect(spans.map(span => span.name)).toEqual(['decide jev-latest']);

const decideSpan = spans[0]!;
expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE);
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra');
expect(decideSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide');
expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation');
expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
},
})
.start()
Expand All @@ -259,8 +259,8 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
.expect({
span: container => {
expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined();
const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!;
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
const decideSpan = container.items.find(span => span.name === 'decide jev-latest')!;
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
{
type: 'evaluation',
state: { message: 'I was charged twice.', apiKey: '[REDACTED]' },
Expand All @@ -269,7 +269,7 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
},
},
]);
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
{ type: 'evaluation', answers: { urgent: { type: 'boolean', probability: 0.9 } } },
]);
},
Expand All @@ -293,12 +293,12 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
span: container => {
expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined();
const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra');
expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']);
expect(spans.map(span => span.name)).toEqual(['decide jev-latest']);

const evaluateSpan = spans[0]!;
expect(evaluateSpan.status).toBe('error');
expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.');
expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
const decideSpan = spans[0]!;
expect(decideSpan.status).toBe('error');
expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.');
expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
},
})
.start()
Expand Down
5 changes: 2 additions & 3 deletions packages/server-utils/src/ai/mastra/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {
GEN_AI_CHAT,
GEN_AI_DECIDE,
GEN_AI_EMBEDDINGS,
GEN_AI_EVALUATE,
GEN_AI_EXECUTE_TOOL,
GEN_AI_INVOKE_AGENT,
} from '@sentry/conventions/op';
Expand Down Expand Up @@ -42,8 +42,7 @@ export const SPAN_TYPE_OPS: Readonly<Record<string, { op: string; operationName:
provider_tool_call: { op: GEN_AI_EXECUTE_TOOL, operationName: 'execute_tool' },
client_tool_call: { op: GEN_AI_EXECUTE_TOOL, operationName: 'execute_tool' },
rag_embedding: { op: GEN_AI_EMBEDDINGS, operationName: 'embeddings' },
// oxlint-disable-next-line typescript/no-deprecated -- the rename to `gen_ai.decide` comes in a follow-up
classifier_evaluation: { op: GEN_AI_EVALUATE, operationName: 'evaluate' },
classifier_evaluation: { op: GEN_AI_DECIDE, operationName: 'decide' },
};

export const TOOL_SPAN_TYPES: ReadonlySet<string> = new Set<MastraSpanType>([
Expand Down
10 changes: 5 additions & 5 deletions packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import {
} from '@sentry/conventions/attributes';
import {
GEN_AI_CHAT,
GEN_AI_DECIDE,
GEN_AI_EMBEDDINGS,
GEN_AI_EVALUATE,
GEN_AI_EXECUTE_TOOL,
GEN_AI_INVOKE_AGENT,
} from '@sentry/conventions/op';
Expand Down Expand Up @@ -165,11 +165,11 @@ describe('SentryMastraExporter', () => {
);

const json = spanToStaticSpanJSON(endedSpans[0]!);
expect(json.description).toBe('evaluate jev-1');
expect(json.description).toBe('decide jev-1');
expect(json.data).toMatchObject({
[SENTRY_OP]: GEN_AI_EVALUATE,
[SENTRY_OP]: GEN_AI_DECIDE,
[SENTRY_ORIGIN]: 'auto.ai.mastra',
[GEN_AI_OPERATION_NAME]: 'evaluate',
[GEN_AI_OPERATION_NAME]: 'decide',
[GEN_AI_REQUEST_MODEL]: 'jev-1',
[GEN_AI_PROVIDER_NAME]: 'typesafe.evaluation',
[GEN_AI_USAGE_INPUT_TOKENS]: 30,
Expand All @@ -187,7 +187,7 @@ describe('SentryMastraExporter', () => {

expect(endedSpans).toHaveLength(0);
expect(call.endTime).toEqual(span.endTime);
expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1');
expect(spanToStaticSpanJSON(call.span!).description).toBe('decide jev-1');
});

it('passes its own recording options to the classifier evaluation call', async () => {
Expand Down
Loading