From db16762abc89b47e7cb7718ab63780d00d6d471c Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 13:35:21 +0200 Subject: [PATCH 1/2] feat(server-utils): Record TypeSafe and LangChain Jev calls as gen_ai.decide spans Co-Authored-By: Claude Opus 5.5 --- .../langchain-typesafe-classifier/test.ts | 20 ++++---- .../suites/tracing/langchain/v1/test.ts | 50 +++++++++---------- .../suites/tracing/langgraph/v1/test.ts | 24 ++++----- .../suites/tracing/typesafe/test.ts | 30 +++++------ .../server-utils/src/ai/langchain/index.ts | 8 +-- .../src/ai/langchain/typesafe-classifier.ts | 6 +-- .../server-utils/src/ai/typesafe/index.ts | 12 ++--- .../server-utils/src/integrations/typesafe.ts | 4 +- .../tracing/langchain-invoke-agent.test.ts | 6 +-- 9 files changed, 80 insertions(+), 80 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts index b5681919fe82..347f79787c80 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts @@ -10,7 +10,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { expect, it } from 'vitest'; import { createRunner } from '../../../runner'; import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils'; @@ -29,28 +29,28 @@ it('traces a TypeSafeClassifier call through the Vite plugin instrumentation', a expect(genAiSpans).toHaveLength(1); - const evaluateSpan = genAiSpans[0]!; - expect(evaluateSpan.name).toBe('evaluate jev-latest'); - expect(evaluateSpan.parent_span_id).toBe(segmentSpan?.span_id); - expect(evaluateSpan.status).toBe('ok'); - expect(evaluateSpan.attributes).toMatchObject({ - [SENTRY_OP]: { value: GEN_AI_EVALUATE, type: 'string' }, + const decideSpan = genAiSpans[0]!; + expect(decideSpan.name).toBe('decide jev-latest'); + expect(decideSpan.parent_span_id).toBe(segmentSpan?.span_id); + expect(decideSpan.status).toBe('ok'); + expect(decideSpan.attributes).toMatchObject({ + [SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' }, [SENTRY_ORIGIN]: { value: 'auto.ai.langchain', type: 'string' }, - [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, + [GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'typesafe', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'jev-latest', type: 'string' }, [GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 30, type: 'integer' }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 2, type: 'integer' }, }); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ { type: 'evaluation', state: ['user: My payouts have been failing.'], questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, }, ]); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, ]); }) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index f14ccb93169d..f6de0aecca94 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -20,7 +20,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests'; @@ -241,7 +241,7 @@ describe('LangChain integration (v1)', () => { 'scenario-typesafe-classifier.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => { + test('records a TypeSafeClassifier run as a gen_ai.decide span', async () => { const runner = createRunner().ignore('event'); const spansPromise = runner.collectStreamedSpansUntilSegment('main'); @@ -250,19 +250,19 @@ describe('LangChain integration (v1)', () => { const spans = (await spansPromise).filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', ); - expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']); - - const [evaluateSpan, messageEvaluateSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp); - expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); - expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); - expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe'); - expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); - expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13'); - expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); - expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); - expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + expect(spans.map(span => span.name)).toEqual(['decide jev-latest', 'decide jev-latest']); + + const [decideSpan, messageDecideSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE); + expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(decideSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide'); + expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe'); + expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(decideSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13'); + expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', state: 'My payouts have been failing.', @@ -270,14 +270,14 @@ describe('LangChain integration (v1)', () => { }, ]); // A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form. - expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + expect(JSON.parse(messageDecideSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', state: 'user: My card was charged twice.', questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, }, ]); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, ]); }); @@ -305,7 +305,7 @@ describe('LangChain integration (v1)', () => { const spans = await spansPromise; const segment = spans.find(span => span.is_segment && span.name === 'main'); expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child'); - expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1); + expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)).toHaveLength(1); }); }, { @@ -330,15 +330,15 @@ describe('LangChain integration (v1)', () => { await runner.start().completed(); const spans = await spansPromise; - const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!; + const decideSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)!; const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!; // The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped. - expect(evaluateSpan.name).toBe('evaluate jev-latest'); - expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent'); - expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + expect(decideSpan.name).toBe('decide jev-latest'); + expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(decideSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent'); + expect(decideSpan.parent_span_id).toBe(chatSpan.parent_span_id); + expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', // The router passes the latest human message, which the classifier sends as a transcript line. @@ -352,7 +352,7 @@ describe('LangChain integration (v1)', () => { }, }, ]); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', answers: { diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts index c6a71539de1a..b09261efc29e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts @@ -6,7 +6,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; describe('LangGraph integration (v1)', () => { @@ -28,21 +28,21 @@ describe('LangGraph integration (v1)', () => { const spans = await spansPromise; const rootSpan = spans.find(span => span.is_segment && span.name === 'main')!; const agentSpan = spans.find(span => span.name === 'invoke_agent triage_graph')!; - const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + const decideSpan = spans.find(span => span.name === 'decide jev-latest')!; expect(agentSpan.parent_span_id).toBe(rootSpan.span_id); - expect(evaluateSpan.parent_span_id).toBe(agentSpan.span_id); - expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); - expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph'); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + expect(decideSpan.parent_span_id).toBe(agentSpan.span_id); + expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE); + expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(decideSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph'); + expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', state: 'My payouts have been failing.', questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, }, ]); - expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, ]); }); @@ -69,11 +69,11 @@ describe('LangGraph integration (v1)', () => { await runner.start().completed(); const spans = await spansPromise; - const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + const decideSpan = spans.find(span => span.name === 'decide jev-latest')!; - expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); - expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE); + expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index e5feb8919745..e7b8d7dd97c9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -28,22 +28,22 @@ describe('TypeSafe integration', () => { scenarioFile, instrumentFile, (createRunner, test) => { - test('creates evaluate spans for systemOne', async () => { + test('creates decide spans for systemOne', async () => { await createRunner() .unordered() .expect({ span: container => { - const evaluateSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + const decideSpans = container.items.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', ); - expect(evaluateSpans).toHaveLength(4); - for (const span of evaluateSpans) { + expect(decideSpans).toHaveLength(4); + for (const span of decideSpans) { expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); - expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('decide'); expect(span.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('typesafe'); } - const defaultModelSpan = evaluateSpans.find(span => span.name === 'evaluate jev-latest')!; + const defaultModelSpan = decideSpans.find(span => span.name === 'decide jev-latest')!; expect(defaultModelSpan).toBeDefined(); expect(defaultModelSpan.status).toBe('ok'); expect(defaultModelSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('jev-latest'); @@ -92,18 +92,18 @@ describe('TypeSafe integration', () => { }, ]); - const asResponseSpan = evaluateSpans.find(span => span.name === 'evaluate jev-1.13')!; + const asResponseSpan = decideSpans.find(span => span.name === 'decide jev-1.13')!; expect(asResponseSpan).toBeDefined(); expect(asResponseSpan.status).toBe('ok'); expect(asResponseSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); - const errorSpan = evaluateSpans.find(span => span.name === 'evaluate error-model')!; + const errorSpan = decideSpans.find(span => span.name === 'decide error-model')!; expect(errorSpan).toBeDefined(); expect(errorSpan.status).toBe('error'); expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); - const validationErrorSpan = evaluateSpans.find(span => span.name === 'evaluate validation-error')!; + const validationErrorSpan = decideSpans.find(span => span.name === 'decide validation-error')!; expect(validationErrorSpan).toBeDefined(); expect(validationErrorSpan.status).toBe('error'); expect(validationErrorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); @@ -130,12 +130,12 @@ describe('TypeSafe integration', () => { .unordered() .expect({ span: container => { - const evaluateSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + const decideSpans = container.items.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', ); - expect(evaluateSpans).toHaveLength(4); - expect(evaluateSpans.filter(span => span.status === 'error')).toHaveLength(2); - for (const span of evaluateSpans) { + expect(decideSpans).toHaveLength(4); + expect(decideSpans.filter(span => span.status === 'error')).toHaveLength(2); + for (const span of decideSpans) { expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); // State, questions and answers must not come back through another attribute (e.g. an error message). diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index f7b5408d1d59..11a1afb7d391 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -48,7 +48,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Internal state - single instance tracks all spans const spanMap = new Map(); - const evaluateRunIds = new Set(); + const decideRunIds = new Set(); /** * Exit a span and clean up @@ -59,7 +59,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.end(); } spanMap.delete(runId); - evaluateRunIds.delete(runId); + decideRunIds.delete(runId); }; /** @@ -225,7 +225,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // A Jev call is a real model call, so it is recorded inside an agent too. if (isTypeSafeClassifier(chain)) { spanMap.set(runId, startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs)); - evaluateRunIds.add(runId); + decideRunIds.add(runId); return; } @@ -278,7 +278,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChainEnd(outputs: unknown, runId: string) { const span = spanMap.get(runId); if (span?.isRecording()) { - if (evaluateRunIds.has(runId)) { + if (decideRunIds.has(runId)) { addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs); } else if (recordOutputs) { span.setAttributes({ diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts index e254685a4561..0167e60f2c22 100644 --- a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -1,7 +1,7 @@ import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import type { Span } from '@sentry/core'; import { isObjectLike } from '@sentry/core'; -import { addResponseAttributes, startEvaluateSpan } from '../typesafe'; +import { addResponseAttributes, startDecideSpan } from '../typesafe'; import { LANGCHAIN_ORIGIN } from './constants'; import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils'; @@ -40,7 +40,7 @@ export function isTypeSafeClassifier(chain: unknown): boolean { return isObjectLike(chain) && Array.isArray(chain.id) && chain.id[chain.id.length - 1] === TYPESAFE_CLASSIFIER_NAME; } -/** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */ +/** Start a `decide` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */ export function startTypeSafeClassifierSpan( chain: unknown, inputs: Record, @@ -50,7 +50,7 @@ export function startTypeSafeClassifierSpan( const kwargs = isObjectLike(chain) && isObjectLike(chain.kwargs) ? chain.kwargs : {}; const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL; - return startEvaluateSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, { + return startDecideSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, { [SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, ...getAgentNameFromMetadata(metadata), ...getConversationIdFromMetadata(metadata), diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 81dbdd666cd0..65d24e6fc3f8 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -10,7 +10,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import type { Span, SpanAttributes } from '@sentry/core'; import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, withActiveSpan } from '@sentry/core'; import type { GenAiOptions } from '../core/utils'; @@ -21,7 +21,7 @@ import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; * Start the span for a `systemOne(request)` call. The request model falls back to the client's * `defaultModel`, the same way the SDK resolves it. */ -export function startEvaluateSpan( +export function startDecideSpan( request: unknown, client: unknown, recordInputs: boolean, @@ -33,9 +33,9 @@ export function startEvaluateSpan( typeof params.model === 'string' ? params.model : typeof defaultModel === 'string' ? defaultModel : undefined; return startInactiveSpan({ - name: model ? `evaluate ${model}` : 'evaluate', + name: model ? `decide ${model}` : 'decide', // oxlint-disable-next-line typescript/no-deprecated -- the rename to `gen_ai.decide` comes in a follow-up - op: GEN_AI_EVALUATE, + op: GEN_AI_DECIDE, // Extra attributes (e.g. another integration's origin) are set at start, so samplers see them. attributes: { ...getRequestAttributes(params, model, recordInputs), ...attributes }, }); @@ -48,7 +48,7 @@ function getRequestAttributes( ): SpanAttributes { return { [SENTRY_ORIGIN]: TYPESAFE_ORIGIN, - [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_OPERATION_NAME]: 'decide', [GEN_AI_PROVIDER_NAME]: TYPESAFE_PROVIDER_NAME, ...(model ? { [GEN_AI_REQUEST_MODEL]: model } : {}), ...(recordInputs ? { [GEN_AI_INPUT_MESSAGES]: getEvaluationInputMessages(request) } : {}), @@ -143,7 +143,7 @@ function instrumentSystemOne( options: GenAiOptions | undefined, ): unknown { const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); - const span = startEvaluateSpan(args[0], client, recordInputs); + const span = startDecideSpan(args[0], client, recordInputs); const endWithError = (): void => { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); span.end(); diff --git a/packages/server-utils/src/integrations/typesafe.ts b/packages/server-utils/src/integrations/typesafe.ts index 9575e1206435..70f2b39c020b 100644 --- a/packages/server-utils/src/integrations/typesafe.ts +++ b/packages/server-utils/src/integrations/typesafe.ts @@ -3,7 +3,7 @@ import type { IntegrationFn } from '@sentry/core'; import { defineIntegration } from '@sentry/core'; import type { GenAiOptions } from '../ai/core/utils'; import { resolveAIRecordingOptions } from '../ai/core/utils'; -import { addResponseAttributes, onSystemOneResponse, startEvaluateSpan } from '../ai/typesafe'; +import { addResponseAttributes, onSystemOneResponse, startDecideSpan } from '../ai/typesafe'; import { TYPESAFE_INTEGRATION_NAME } from '../ai/typesafe/constants'; import type { OrchestrionChannelContext } from '../orchestrion/types'; import { CHANNELS } from '../orchestrion/channels'; @@ -23,7 +23,7 @@ const _typesafeIntegration = ((options: GenAiOptions = {}) => { function instrumentTypeSafe(options: GenAiOptions): void { bindTracingChannelToSpan( diagnosticsChannel.tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE), - data => startEvaluateSpan(data.arguments?.[0], data.self, resolveAIRecordingOptions(options).recordInputs), + data => startDecideSpan(data.arguments?.[0], data.self, resolveAIRecordingOptions(options).recordInputs), { beforeSpanEnd: (span, data) => { if (!('error' in data)) { diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts index 3a36432167ea..592a9aa9f362 100644 --- a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts @@ -6,7 +6,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getClient, getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; @@ -120,11 +120,11 @@ describe('LangChain invoke_agent span names', () => { expect(startAttributes).toEqual([ expect.objectContaining({ - [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_OP]: GEN_AI_DECIDE, [SENTRY_ORIGIN]: 'auto.ai.langchain', [GEN_AI_AGENT_NAME]: 'support_agent', }), ]); - expect(endedSpans.map(span => spanToStaticSpanJSON(span).description)).toEqual(['evaluate jev-latest']); + expect(endedSpans.map(span => spanToStaticSpanJSON(span).description)).toEqual(['decide jev-latest']); }); }); From abb55130398eb759b02cb7fc7f747f8790db5007 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 13:44:57 +0200 Subject: [PATCH 2/2] Use GEN_AI_DECIDE constant in TypeSafe test assertions --- .../suites/tracing/typesafe/test.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index e7b8d7dd97c9..0640f7282c7d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -11,6 +11,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -33,9 +34,7 @@ describe('TypeSafe integration', () => { .unordered() .expect({ span: container => { - const decideSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', - ); + const decideSpans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); expect(decideSpans).toHaveLength(4); for (const span of decideSpans) { expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); @@ -130,9 +129,7 @@ describe('TypeSafe integration', () => { .unordered() .expect({ span: container => { - const decideSpans = container.items.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide', - ); + const decideSpans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); expect(decideSpans).toHaveLength(4); expect(decideSpans.filter(span => span.status === 'error')).toHaveLength(2); for (const span of decideSpans) {