Skip to content

Commit db16762

Browse files
andreiborzaclaude
andreiborza
and
claude
committed
feat(server-utils): Record TypeSafe and LangChain Jev calls as gen_ai.decide spans
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 12a84a9 commit db16762

9 files changed

Lines changed: 80 additions & 80 deletions

File tree

‎dev-packages/cloudflare-integration-tests/suites/tracing/langchain-typesafe-classifier/test.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
SENTRY_OP,
1111
SENTRY_ORIGIN,
1212
} from '@sentry/conventions/attributes';
13-
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
13+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
1414
import { expect, it } from 'vitest';
1515
import { createRunner } from '../../../runner';
1616
import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils';
@@ -29,28 +29,28 @@ it('traces a TypeSafeClassifier call through the Vite plugin instrumentation', a
2929

3030
expect(genAiSpans).toHaveLength(1);
3131

32-
const evaluateSpan = genAiSpans[0]!;
33-
expect(evaluateSpan.name).toBe('evaluate jev-latest');
34-
expect(evaluateSpan.parent_span_id).toBe(segmentSpan?.span_id);
35-
expect(evaluateSpan.status).toBe('ok');
36-
expect(evaluateSpan.attributes).toMatchObject({
37-
[SENTRY_OP]: { value: GEN_AI_EVALUATE, type: 'string' },
32+
const decideSpan = genAiSpans[0]!;
33+
expect(decideSpan.name).toBe('decide jev-latest');
34+
expect(decideSpan.parent_span_id).toBe(segmentSpan?.span_id);
35+
expect(decideSpan.status).toBe('ok');
36+
expect(decideSpan.attributes).toMatchObject({
37+
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
3838
[SENTRY_ORIGIN]: { value: 'auto.ai.langchain', type: 'string' },
39-
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
39+
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
4040
[GEN_AI_PROVIDER_NAME]: { value: 'typesafe', type: 'string' },
4141
[GEN_AI_REQUEST_MODEL]: { value: 'jev-latest', type: 'string' },
4242
[GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13', type: 'string' },
4343
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 30, type: 'integer' },
4444
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 2, type: 'integer' },
4545
});
46-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
46+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
4747
{
4848
type: 'evaluation',
4949
state: ['user: My payouts have been failing.'],
5050
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
5151
},
5252
]);
53-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
53+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
5454
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
5555
]);
5656
})

‎dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import {
2020
SENTRY_OP,
2121
SENTRY_ORIGIN,
2222
} from '@sentry/conventions/attributes';
23-
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
23+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
2424
import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
2525
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
2626
import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests';
@@ -241,7 +241,7 @@ describe('LangChain integration (v1)', () => {
241241
'scenario-typesafe-classifier.mjs',
242242
'instrument-with-pii.mjs',
243243
(createRunner, test) => {
244-
test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => {
244+
test('records a TypeSafeClassifier run as a gen_ai.decide span', async () => {
245245
const runner = createRunner().ignore('event');
246246
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
247247

@@ -250,34 +250,34 @@ describe('LangChain integration (v1)', () => {
250250
const spans = (await spansPromise).filter(
251251
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain',
252252
);
253-
expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']);
254-
255-
const [evaluateSpan, messageEvaluateSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp);
256-
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
257-
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
258-
expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate');
259-
expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe');
260-
expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
261-
expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13');
262-
expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
263-
expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
264-
expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
265-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
253+
expect(spans.map(span => span.name)).toEqual(['decide jev-latest', 'decide jev-latest']);
254+
255+
const [decideSpan, messageDecideSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp);
256+
expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE);
257+
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
258+
expect(decideSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide');
259+
expect(decideSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe');
260+
expect(decideSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
261+
expect(decideSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13');
262+
expect(decideSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
263+
expect(decideSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
264+
expect(decideSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
265+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
266266
{
267267
type: 'evaluation',
268268
state: 'My payouts have been failing.',
269269
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
270270
},
271271
]);
272272
// A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form.
273-
expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
273+
expect(JSON.parse(messageDecideSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
274274
{
275275
type: 'evaluation',
276276
state: 'user: My card was charged twice.',
277277
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
278278
},
279279
]);
280-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
280+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
281281
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
282282
]);
283283
});
@@ -305,7 +305,7 @@ describe('LangChain integration (v1)', () => {
305305
const spans = await spansPromise;
306306
const segment = spans.find(span => span.is_segment && span.name === 'main');
307307
expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child');
308-
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1);
308+
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)).toHaveLength(1);
309309
});
310310
},
311311
{
@@ -330,15 +330,15 @@ describe('LangChain integration (v1)', () => {
330330
await runner.start().completed();
331331

332332
const spans = await spansPromise;
333-
const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!;
333+
const decideSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE)!;
334334
const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!;
335335

336336
// The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped.
337-
expect(evaluateSpan.name).toBe('evaluate jev-latest');
338-
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
339-
expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent');
340-
expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id);
341-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
337+
expect(decideSpan.name).toBe('decide jev-latest');
338+
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
339+
expect(decideSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent');
340+
expect(decideSpan.parent_span_id).toBe(chatSpan.parent_span_id);
341+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
342342
{
343343
type: 'evaluation',
344344
// The router passes the latest human message, which the classifier sends as a transcript line.
@@ -352,7 +352,7 @@ describe('LangChain integration (v1)', () => {
352352
},
353353
},
354354
]);
355-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
355+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
356356
{
357357
type: 'evaluation',
358358
answers: {

‎dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
SENTRY_OP,
77
SENTRY_ORIGIN,
88
} from '@sentry/conventions/attributes';
9-
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
9+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
1010
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
1111

1212
describe('LangGraph integration (v1)', () => {
@@ -28,21 +28,21 @@ describe('LangGraph integration (v1)', () => {
2828
const spans = await spansPromise;
2929
const rootSpan = spans.find(span => span.is_segment && span.name === 'main')!;
3030
const agentSpan = spans.find(span => span.name === 'invoke_agent triage_graph')!;
31-
const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!;
31+
const decideSpan = spans.find(span => span.name === 'decide jev-latest')!;
3232

3333
expect(agentSpan.parent_span_id).toBe(rootSpan.span_id);
34-
expect(evaluateSpan.parent_span_id).toBe(agentSpan.span_id);
35-
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
36-
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
37-
expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph');
38-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
34+
expect(decideSpan.parent_span_id).toBe(agentSpan.span_id);
35+
expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE);
36+
expect(decideSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
37+
expect(decideSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph');
38+
expect(JSON.parse(decideSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
3939
{
4040
type: 'evaluation',
4141
state: 'My payouts have been failing.',
4242
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
4343
},
4444
]);
45-
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
45+
expect(JSON.parse(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
4646
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
4747
]);
4848
});
@@ -69,11 +69,11 @@ describe('LangGraph integration (v1)', () => {
6969
await runner.start().completed();
7070

7171
const spans = await spansPromise;
72-
const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!;
72+
const decideSpan = spans.find(span => span.name === 'decide jev-latest')!;
7373

74-
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
75-
expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
76-
expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
74+
expect(decideSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_DECIDE);
75+
expect(decideSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
76+
expect(decideSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
7777
});
7878
},
7979
{

‎dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,22 +28,22 @@ describe('TypeSafe integration', () => {
2828
scenarioFile,
2929
instrumentFile,
3030
(createRunner, test) => {
31-
test('creates evaluate spans for systemOne', async () => {
31+
test('creates decide spans for systemOne', async () => {
3232
await createRunner()
3333
.unordered()
3434
.expect({
3535
span: container => {
36-
const evaluateSpans = container.items.filter(
37-
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
36+
const decideSpans = container.items.filter(
37+
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
3838
);
39-
expect(evaluateSpans).toHaveLength(4);
40-
for (const span of evaluateSpans) {
39+
expect(decideSpans).toHaveLength(4);
40+
for (const span of decideSpans) {
4141
expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe');
42-
expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate');
42+
expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('decide');
4343
expect(span.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('typesafe');
4444
}
4545

46-
const defaultModelSpan = evaluateSpans.find(span => span.name === 'evaluate jev-latest')!;
46+
const defaultModelSpan = decideSpans.find(span => span.name === 'decide jev-latest')!;
4747
expect(defaultModelSpan).toBeDefined();
4848
expect(defaultModelSpan.status).toBe('ok');
4949
expect(defaultModelSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('jev-latest');
@@ -92,18 +92,18 @@ describe('TypeSafe integration', () => {
9292
},
9393
]);
9494

95-
const asResponseSpan = evaluateSpans.find(span => span.name === 'evaluate jev-1.13')!;
95+
const asResponseSpan = decideSpans.find(span => span.name === 'decide jev-1.13')!;
9696
expect(asResponseSpan).toBeDefined();
9797
expect(asResponseSpan.status).toBe('ok');
9898
expect(asResponseSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0');
9999

100-
const errorSpan = evaluateSpans.find(span => span.name === 'evaluate error-model')!;
100+
const errorSpan = decideSpans.find(span => span.name === 'decide error-model')!;
101101
expect(errorSpan).toBeDefined();
102102
expect(errorSpan.status).toBe('error');
103103
expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined();
104104
expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
105105

106-
const validationErrorSpan = evaluateSpans.find(span => span.name === 'evaluate validation-error')!;
106+
const validationErrorSpan = decideSpans.find(span => span.name === 'decide validation-error')!;
107107
expect(validationErrorSpan).toBeDefined();
108108
expect(validationErrorSpan.status).toBe('error');
109109
expect(validationErrorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
@@ -130,12 +130,12 @@ describe('TypeSafe integration', () => {
130130
.unordered()
131131
.expect({
132132
span: container => {
133-
const evaluateSpans = container.items.filter(
134-
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate',
133+
const decideSpans = container.items.filter(
134+
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.decide',
135135
);
136-
expect(evaluateSpans).toHaveLength(4);
137-
expect(evaluateSpans.filter(span => span.status === 'error')).toHaveLength(2);
138-
for (const span of evaluateSpans) {
136+
expect(decideSpans).toHaveLength(4);
137+
expect(decideSpans.filter(span => span.status === 'error')).toHaveLength(2);
138+
for (const span of decideSpans) {
139139
expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
140140
expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
141141
// State, questions and answers must not come back through another attribute (e.g. an error message).

‎packages/server-utils/src/ai/langchain/index.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
4848

4949
// Internal state - single instance tracks all spans
5050
const spanMap = new Map<string, Span>();
51-
const evaluateRunIds = new Set<string>();
51+
const decideRunIds = new Set<string>();
5252

5353
/**
5454
* Exit a span and clean up
@@ -59,7 +59,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
5959
span.end();
6060
}
6161
spanMap.delete(runId);
62-
evaluateRunIds.delete(runId);
62+
decideRunIds.delete(runId);
6363
};
6464

6565
/**
@@ -225,7 +225,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
225225
// A Jev call is a real model call, so it is recorded inside an agent too.
226226
if (isTypeSafeClassifier(chain)) {
227227
spanMap.set(runId, startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs));
228-
evaluateRunIds.add(runId);
228+
decideRunIds.add(runId);
229229
return;
230230
}
231231

@@ -278,7 +278,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
278278
handleChainEnd(outputs: unknown, runId: string) {
279279
const span = spanMap.get(runId);
280280
if (span?.isRecording()) {
281-
if (evaluateRunIds.has(runId)) {
281+
if (decideRunIds.has(runId)) {
282282
addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs);
283283
} else if (recordOutputs) {
284284
span.setAttributes({

‎packages/server-utils/src/ai/langchain/typesafe-classifier.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { SENTRY_ORIGIN } from '@sentry/conventions/attributes';
22
import type { Span } from '@sentry/core';
33
import { isObjectLike } from '@sentry/core';
4-
import { addResponseAttributes, startEvaluateSpan } from '../typesafe';
4+
import { addResponseAttributes, startDecideSpan } from '../typesafe';
55
import { LANGCHAIN_ORIGIN } from './constants';
66
import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils';
77

@@ -40,7 +40,7 @@ export function isTypeSafeClassifier(chain: unknown): boolean {
4040
return isObjectLike(chain) && Array.isArray(chain.id) && chain.id[chain.id.length - 1] === TYPESAFE_CLASSIFIER_NAME;
4141
}
4242

43-
/** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */
43+
/** Start a `decide` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */
4444
export function startTypeSafeClassifierSpan(
4545
chain: unknown,
4646
inputs: Record<string, unknown>,
@@ -50,7 +50,7 @@ export function startTypeSafeClassifierSpan(
5050
const kwargs = isObjectLike(chain) && isObjectLike(chain.kwargs) ? chain.kwargs : {};
5151
const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL;
5252

53-
return startEvaluateSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, {
53+
return startDecideSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, {
5454
[SENTRY_ORIGIN]: LANGCHAIN_ORIGIN,
5555
...getAgentNameFromMetadata(metadata),
5656
...getConversationIdFromMetadata(metadata),

0 commit comments

Comments
 (0)