From 4ca936f30f737e222accded9cf74b860dd6d1632 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 14:30:02 +0200 Subject: [PATCH 01/24] feat(server-utils): Instrument the OpenAI decisions API as gen_ai.decide spans Co-Authored-By: Claude Opus 5.5 --- .../openai/v7/instrument-with-recording.mjs | 9 ++ .../tracing/openai/v7/scenario-decisions.mjs | 105 ++++++++++++++++++ .../suites/tracing/openai/v7/test.ts | 98 ++++++++++++++++ .../server-utils/src/ai/openai/constants.ts | 2 + packages/server-utils/src/ai/openai/index.ts | 10 +- packages/server-utils/src/ai/openai/utils.ts | 101 ++++++++++++----- .../server-utils/src/integrations/openai.ts | 3 +- .../src/orchestrion/config/openai.ts | 7 ++ .../test/ai/lib/tracing/openai.test.ts | 21 ++++ 9 files changed, 328 insertions(+), 28 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs new file mode 100644 index 000000000000..23526265b021 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -0,0 +1,105 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json({ limit: '1mb' })); + + app.post('/openai/decisions', (req, res) => { + const { model } = req.body; + + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + res.send({ + model: 'gpt-6-luna-2026-09-01', + answers: [ + { type: 'predicate', name: 'wants_refund', probability: 0.97 }, + { + type: 'choice', + name: 'department', + choice: 'billing', + confidence: 0.82, + probabilities: [ + { value: 'billing', probability: 0.82 }, + { value: 'other', probability: 0.18 }, + ], + }, + { type: 'refusal', name: 'urgency' }, + ], + usage: { + input_tokens: 120, + input_tokens_details: { cached_tokens: 40, cache_write_tokens: 0 }, + output_tokens: 3, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 123, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +const questions = [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [{ value: 'billing' }, { value: 'other' }], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, +]; + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + await client.decisions.create({ + model: 'gpt-6-luna', + input: 'I was charged twice and want a refund.', + questions, + }); + + await client.decisions.create({ + model: 'gpt-6-luna', + input: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is the package damaged?' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' }, + ], + }, + ], + questions: [{ type: 'predicate', name: 'damaged', instructions: 'Is there visible damage?' }], + }); + + try { + await client.decisions.create({ model: 'error-model', input: 'Error test', questions }); + } catch { + // Error is expected and handled + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 34199f155036..78a72be31685 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -1,19 +1,24 @@ import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, expect } from 'vitest'; import { + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { conditionalTest } from '../../../../utils/index'; @@ -158,4 +163,97 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { }, }, ); + + // The decisions API (`openai` >= 7.30.0) returns typed answers, so it is recorded like a Jev call. + createEsmAndCjsTests( + __dirname, + 'scenario-decisions.mjs', + 'instrument-with-recording.mjs', + (createRunner, test) => { + test('instruments the decisions API as decide spans', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + expect(spans).toHaveLength(3); + + const [textSpan, imageSpan] = spans.filter(span => span.status === 'ok'); + expect(textSpan!.name).toBe('decide gpt-6-luna'); + expect(textSpan!.attributes).toMatchObject({ + [SENTRY_ORIGIN]: { value: 'auto.ai.openai' }, + [GEN_AI_OPERATION_NAME]: { value: 'decide' }, + [GEN_AI_PROVIDER_NAME]: { value: 'openai' }, + [GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' }, + [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna-2026-09-01' }, + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 120 }, + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 40 }, + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 3 }, + [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 }, + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 123 }, + }); + expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice and want a refund.', + questions: [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [{ value: 'billing' }, { value: 'other' }], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, + ], + }, + ]); + expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ + { + type: 'evaluation', + answers: [ + { type: 'predicate', name: 'wants_refund', probability: 0.97 }, + { + type: 'choice', + name: 'department', + choice: 'billing', + confidence: 0.82, + probabilities: [ + { value: 'billing', probability: 0.82 }, + { value: 'other', probability: 0.18 }, + ], + }, + { type: 'refusal', name: 'urgency' }, + ], + }, + ]); + + // Images are base64 data URLs, so only their media type is recorded. + expect(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value).not.toContain('base64'); + expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is the package damaged?' }, + { type: 'input_image', mime_type: 'image/png' }, + ], + }, + ]); + + const errorSpan = spans.find(span => span.name === 'decide error-model'); + expect(errorSpan!.status).not.toBe('ok'); + }); + }, + { + additionalDependencies: { + openai: '7.30.0', + }, + }, + ); }); diff --git a/packages/server-utils/src/ai/openai/constants.ts b/packages/server-utils/src/ai/openai/constants.ts index af163af81b6a..60bf73f94bf1 100644 --- a/packages/server-utils/src/ai/openai/constants.ts +++ b/packages/server-utils/src/ai/openai/constants.ts @@ -12,6 +12,8 @@ export const OPENAI_METHOD_REGISTRY = { // Conversations API - for conversation state management // https://platform.openai.com/docs/guides/conversation-state 'conversations.create': { operation: 'chat' }, + // https://developers.openai.com/api/docs/guides/decisions + 'decisions.create': { operation: 'decide' }, } as const satisfies InstrumentedMethodRegistry; export const RESPONSES_TOOL_CALL_EVENT_TYPES = [ 'response.output_item.added', diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 981da4281ef5..08f1e4541feb 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -31,7 +31,7 @@ import { import { OPENAI_METHOD_REGISTRY } from './constants'; import { instrumentStream } from './streaming'; import type { ChatCompletionChunk, OpenAiOptions, OpenAIStream, ResponseStreamingEvent } from './types'; -import { addResponseAttributes, extractRequestParameters } from './utils'; +import { addResponseAttributes, extractRequestParameters, getDecisionInputMessages } from './utils'; /** * Extract available tools from request parameters @@ -88,6 +88,14 @@ export function extractRequestAttributes( // Extract and record AI request inputs, if present. This is intentionally separate from response attributes. export function addRequestAttributes(span: Span, params: Record, operationName: string): void { + if (operationName === 'decide') { + const inputMessages = getDecisionInputMessages(params); + if (inputMessages) { + span.setAttribute(GEN_AI_INPUT_MESSAGES, inputMessages); + } + return; + } + // Store embeddings input on a separate attribute if (operationName === 'embeddings' && 'input' in params) { const input = params.input; diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index 49b39eb85013..f90c86840d23 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -3,6 +3,7 @@ import { isObjectLike } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_CONVERSATION_ID, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_REQUEST_FREQUENCY_PENALTY, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_PRESENCE_PENALTY, @@ -17,6 +18,7 @@ import { GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { @@ -24,6 +26,7 @@ import { GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE, GEN_AI_REQUEST_STREAM_ATTRIBUTE, } from '../core/gen-ai-attributes'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../core/utils'; import type { ChatCompletionChunk, ResponseStreamingEvent } from './types'; /** @@ -76,32 +79,8 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs attrs[GEN_AI_CONVERSATION_ID] = response.id; } - // Token usage — supports both naming conventions (chat: prompt_tokens/completion_tokens, responses: input_tokens/output_tokens) - if (response.usage && typeof response.usage === 'object') { - const usage = response.usage as Record; - - const inputTokens = usage.prompt_tokens ?? usage.input_tokens; - if (typeof inputTokens === 'number') { - attrs[GEN_AI_USAGE_INPUT_TOKENS] = inputTokens; - } - - const outputTokens = usage.completion_tokens ?? usage.output_tokens; - if (typeof outputTokens === 'number') { - attrs[GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens; - } - - if (typeof usage.total_tokens === 'number') { - attrs[GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens; - } - const inputDetails = usage.prompt_tokens_details ?? usage.input_tokens_details; - if (isObjectLike(inputDetails)) { - if (typeof inputDetails.cached_tokens === 'number') { - attrs[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = inputDetails.cached_tokens; - } - if (typeof inputDetails.cache_write_tokens === 'number') { - attrs[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] = inputDetails.cache_write_tokens; - } - } + if (isObjectLike(response.usage)) { + addUsageAttributes(attrs, response.usage); } // Finish reasons from choices (chat completions) @@ -151,6 +130,14 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs attrs[GEN_AI_RESPONSE_TEXT] = response.output_text; } + // Typed answers (decisions API) + if (Array.isArray(response.answers)) { + const outputMessages = getEvaluationOutputMessages(response.answers); + if (outputMessages) { + attrs[GEN_AI_OUTPUT_MESSAGES] = outputMessages; + } + } + // Tool calls from output array (responses API) if (Array.isArray(response.output) && response.output.length > 0 && !attrs[GEN_AI_RESPONSE_TOOL_CALLS]) { const functionCalls = (response.output as Array>).filter( @@ -165,6 +152,39 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs span.setAttributes(attrs); } +/** + * Token usage, in both naming conventions (chat: prompt_tokens/completion_tokens, + * responses and decisions: input_tokens/output_tokens). + */ +function addUsageAttributes(attrs: Record, usage: Record): void { + const inputTokens = usage.prompt_tokens ?? usage.input_tokens; + if (typeof inputTokens === 'number') { + attrs[GEN_AI_USAGE_INPUT_TOKENS] = inputTokens; + } + + const outputTokens = usage.completion_tokens ?? usage.output_tokens; + if (typeof outputTokens === 'number') { + attrs[GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens; + } + + if (typeof usage.total_tokens === 'number') { + attrs[GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens; + } + const inputDetails = usage.prompt_tokens_details ?? usage.input_tokens_details; + if (isObjectLike(inputDetails)) { + if (typeof inputDetails.cached_tokens === 'number') { + attrs[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = inputDetails.cached_tokens; + } + if (typeof inputDetails.cache_write_tokens === 'number') { + attrs[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] = inputDetails.cache_write_tokens; + } + } + const outputDetails = usage.completion_tokens_details ?? usage.output_tokens_details; + if (isObjectLike(outputDetails) && typeof outputDetails.reasoning_tokens === 'number') { + attrs[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] = outputDetails.reasoning_tokens; + } +} + /** * Extract conversation ID from request parameters * Supports both Conversations API and previous_response_id chaining @@ -206,3 +226,32 @@ export function extractRequestParameters(params: Record): Recor return attributes; } + +/** + * Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state` + * so both render the same. Images are base64 data URLs, so they are reported by media type only. + */ +export function getDecisionInputMessages(params: Record): string | undefined { + return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: params.questions }); +} + +function withoutImageData(input: unknown): unknown { + if (!Array.isArray(input)) { + return input; + } + + return input.map(message => + isObjectLike(message) && Array.isArray(message.content) + ? { ...message, content: message.content.map(withoutImageDataPart) } + : message, + ); +} + +function withoutImageDataPart(part: unknown): unknown { + if (!isObjectLike(part) || part.type !== 'input_image' || typeof part.image_url !== 'string') { + return part; + } + + const mimeType = /^data:([^;,]+)/.exec(part.image_url)?.[1]; + return mimeType ? { type: 'input_image', mime_type: mimeType } : { type: 'input_image' }; +} diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index ec7c852e9010..15c49c217fa3 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -30,6 +30,7 @@ const ORIGIN = 'auto.ai.openai'; const INSTRUMENTED_CHANNELS = [ { channel: CHANNELS.OPENAI_CHAT, operation: 'chat' }, { channel: CHANNELS.OPENAI_EMBEDDINGS, operation: 'embeddings' }, + { channel: CHANNELS.OPENAI_DECIDE, operation: 'decide' }, ] as const; const _openAIIntegration = ((options: OpenAiOptions = {}) => { @@ -134,6 +135,6 @@ function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: /** * Diagnostics-channel-based OpenAI integration. Subscribes to the `orchestrion:openai:*` * diagnostics_channels injected into `openai`'s `create` methods (chat completions, responses, embeddings, - * conversations), so it requires the Sentry runtime hook or bundler plugin. + * conversations, decisions), so it requires the Sentry runtime hook or bundler plugin. */ export const openAIIntegration = defineIntegration(_openAIIntegration); diff --git a/packages/server-utils/src/orchestrion/config/openai.ts b/packages/server-utils/src/orchestrion/config/openai.ts index e68d3a961262..468d7f8392db 100644 --- a/packages/server-utils/src/orchestrion/config/openai.ts +++ b/packages/server-utils/src/orchestrion/config/openai.ts @@ -27,6 +27,12 @@ export const openaiConfig = [ module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Conversations', methodName: 'create', kind: 'Sync' as const }, })), + // OpenAI decisions API (`openai` >= 7.30.0), which returns typed answers rather than generated text. + ...['resources/decisions.js', 'resources/decisions.mjs'].map(filePath => ({ + channelName: 'decide', + module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, + functionQuery: { className: 'Decisions', methodName: 'create', kind: 'Sync' as const }, + })), ] satisfies InstrumentationConfig[]; export const openaiModuleNames = getModuleNames(openaiConfig); @@ -36,4 +42,5 @@ export const openaiChannels = { // identical span handling, so they share one channel. OPENAI_CHAT: 'orchestrion:openai:chat', OPENAI_EMBEDDINGS: 'orchestrion:openai:embeddings', + OPENAI_DECIDE: 'orchestrion:openai:decide', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index 0c6738ce86ac..6946b097f3ed 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,4 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentOpenAiClient } from '../../../../src/ai/openai'; @@ -90,4 +92,23 @@ describe('instrumentOpenAiClient span names', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); }); + + it('records `decisions.create` as a decide span', async () => { + const endedSpans = setupClient('stream'); + const answers = [{ type: 'predicate', name: 'wants_refund', probability: 0.97 }]; + const instrumented = instrumentOpenAiClient({ + decisions: { create: vi.fn().mockResolvedValue({ model: 'gpt-6-luna', answers }) }, + }); + + await instrumented.decisions.create({ + model: 'gpt-6-luna', + input: 'I want a refund', + questions: [{ type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }], + }); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(json.description).toBe('decide gpt-6-luna'); + expect(json.op).toBe(GEN_AI_DECIDE); + expect(JSON.parse(json.data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([{ type: 'evaluation', answers }]); + }); }); From 24f4a8bdf851f71cef856e2ad3d3f8e2c3bf6f96 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 15:26:13 +0200 Subject: [PATCH 02/24] Match the decisions mock to a real gpt-6-luna response --- .../tracing/openai/v7/scenario-decisions.mjs | 24 +++++++++++++------ .../suites/tracing/openai/v7/test.ts | 23 ++++++++++++------ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs index 23526265b021..5de618289c83 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -14,28 +14,38 @@ function startMockServer() { return; } + // Shaped like a real `gpt-6-luna` response. Decisions bills input tokens only. res.send({ - model: 'gpt-6-luna-2026-09-01', + model: 'gpt-6-luna', answers: [ { type: 'predicate', name: 'wants_refund', probability: 0.97 }, { type: 'choice', name: 'department', choice: 'billing', - confidence: 0.82, probabilities: [ { value: 'billing', probability: 0.82 }, { value: 'other', probability: 0.18 }, ], + confidence: 0.82, + }, + { + type: 'score', + name: 'urgency', + score: 0.95, + probabilities: [ + { value: 0, label: 'low', probability: 0.05 }, + { value: 1, label: 'high', probability: 0.95 }, + ], + confidence: 0.93, }, - { type: 'refusal', name: 'urgency' }, ], usage: { - input_tokens: 120, - input_tokens_details: { cached_tokens: 40, cache_write_tokens: 0 }, - output_tokens: 3, + input_tokens: 382, + input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + output_tokens: 0, output_tokens_details: { reasoning_tokens: 0 }, - total_tokens: 123, + total_tokens: 382, }, }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 78a72be31685..da6527d5cd00 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -186,12 +186,12 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { [GEN_AI_OPERATION_NAME]: { value: 'decide' }, [GEN_AI_PROVIDER_NAME]: { value: 'openai' }, [GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' }, - [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna-2026-09-01' }, - [GEN_AI_USAGE_INPUT_TOKENS]: { value: 120 }, - [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 40 }, - [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 3 }, + [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' }, + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 }, + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 0 }, + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 }, [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 }, - [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 123 }, + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 }, }); expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ { @@ -223,13 +223,22 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { type: 'choice', name: 'department', choice: 'billing', - confidence: 0.82, probabilities: [ { value: 'billing', probability: 0.82 }, { value: 'other', probability: 0.18 }, ], + confidence: 0.82, + }, + { + type: 'score', + name: 'urgency', + score: 0.95, + probabilities: [ + { value: 0, label: 'low', probability: 0.05 }, + { value: 1, label: 'high', probability: 0.95 }, + ], + confidence: 0.93, }, - { type: 'refusal', name: 'urgency' }, ], }, ]); From 81b6bc0e51f4425933a697fd0d373ebe2703442b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 15:35:17 +0200 Subject: [PATCH 03/24] Add node-openai-decisions-send-to-sentry e2e test app --- .github/workflows/build.yml | 4 + dev-packages/e2e-tests/.env.example | 4 + .../.gitignore | 5 + .../package.json | 41 ++++++ .../playwright.config.ts | 28 ++++ .../src/app.ts | 90 ++++++++++++ .../src/instrument.ts | 10 ++ .../tests/send-to-sentry.test.ts | 125 +++++++++++++++++ .../tests/utils/sentry-api.ts | 131 ++++++++++++++++++ .../tsconfig.json | 13 ++ 10 files changed, 451 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 658fd862749c..56c0dd2f0536 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1190,6 +1190,10 @@ jobs: E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} + # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. + E2E_OPENAI_DECISIONS_API_KEY: + ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || + '' }} # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index 180fb07078a1..3815cae556f0 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -16,6 +16,10 @@ E2E_TEST_SENTRY_PROJECT= # Only needed to run that test app locally. E2E_OPENROUTER_API_KEY= +# An OpenAI API key for the Decisions API (OpenRouter has no Decisions route). +# Only needed to run node-openai-decisions-send-to-sentry locally. +E2E_OPENAI_DECISIONS_API_KEY= + # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry). # The API token needs "Workers Scripts: Edit" on the account; "Workers KV Storage: Read" additionally silences a # warning when a worker is deleted. Leave it empty to use a `wrangler login` session instead. diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore new file mode 100644 index 000000000000..795f6b6d40d1 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore @@ -0,0 +1,5 @@ +node_modules +pnpm-lock.yaml +dist +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json new file mode 100644 index 000000000000..9b9f26414843 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json @@ -0,0 +1,41 @@ +{ + "name": "node-openai-decisions-send-to-sentry", + "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "tsc", + "start": "node --import ./dist/instrument.js dist/app.js", + "test": "playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", + "test:assert": "pnpm test" + }, + "dependencies": { + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "express": "^4.21.2", + "openai": "7.32.0" + }, + "devDependencies": { + "@playwright/test": "~1.63.0", + "@sentry/conventions": "0.27.0", + "@types/express": "^4.17.21", + "@types/node": "^24.0.0", + "typescript": "~5.9.0" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-openai-decisions-send-to-sentry (latest)" + } + ] + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts new file mode 100644 index 000000000000..ef4764ce8d73 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts @@ -0,0 +1,28 @@ +import { defineConfig } from '@playwright/test'; + +const port = 3030; + +export default defineConfig({ + testDir: './tests', + /* + * Spans take ~2min to become queryable via the trace endpoint, and each poll has its own 180s + * budget. The first test polls twice in a row (the model span, then its parent), so the ceiling + * has to hold two polls back to back. + */ + timeout: 400_000, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: 0, + // Every test spends most of its time polling Sentry, so run them all at once. + workers: '100%', + reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', + use: { + baseURL: `http://localhost:${port}`, + }, + webServer: { + command: 'pnpm start', + port, + stdout: 'pipe', + stderr: 'pipe', + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts new file mode 100644 index 000000000000..77ca388052f4 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts @@ -0,0 +1,90 @@ +// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +// OpenRouter has no `/decisions` route, so this app calls OpenAI directly. CI only passes this key to +// this app. +const apiKey = process.env.E2E_OPENAI_DECISIONS_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENAI_DECISIONS_API_KEY is not set'); +} + +const client = new OpenAI({ apiKey }); + +const MODEL = 'gpt-6-luna'; + +// A 1x1 red PNG, so the image request is cheap. +const RED_PIXEL = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; + +const app = express(); + +/** The trace this request is recorded under, so the test can look its spans up in Sentry. */ +function currentTraceId(): string | undefined { + return Sentry.getActiveSpan()?.spanContext().traceId; +} + +// Each route makes one `decisions.create` call and answers with the trace id, so the test can find +// the request's spans in Sentry. + +app.get('/decide', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: 'I was charged twice this month and want a refund.', + questions: [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [ + { value: 'billing', description: 'Charges and refunds' }, + { value: 'other', description: 'Anything else' }, + ], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, + ], + }); + res.send({ traceId: currentTraceId(), answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.get('/decide-image', async (_req, res, next) => { + try { + const decision = await client.decisions.create({ + model: MODEL, + input: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', image_url: RED_PIXEL }, + ], + }, + ], + questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], + }); + res.send({ traceId: currentTraceId(), answers: decision.answers }); + } catch (error) { + next(error); + } +}); + +app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + res.status(500).send({ message: error.message }); +}); + +const port = Number(process.env.PORT ?? 3030); +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts new file mode 100644 index 000000000000..c6c15df960cb --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; + +// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to +// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The OpenAI integration +// is on by default; its runtime channel injection is what picks up the `openai` client in app.ts. +Sentry.init({ + dsn: process.env.E2E_TEST_DSN, + environment: 'qa', // dynamic sampling bias to keep transactions + tracesSampleRate: 1, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts new file mode 100644 index 000000000000..9797dc210085 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -0,0 +1,125 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; +import type { TraceItem } from './utils/sentry-api'; +import { + EVENT_POLLING_OPTIONS, + fetchSpanAttributes, + fetchTrace, + findSpanInTrace, + flattenTrace, +} from './utils/sentry-api'; + +// Mirrors src/app.ts. +const MODEL = 'gpt-6-luna'; + +type Attributes = Record; + +/** Sends one request to the app and returns the trace its request span was recorded under. */ +async function requestTrace(baseURL: string | undefined, path: string): Promise { + const response = await fetch(`${baseURL}${path}`); + const body = await response.text(); + expect(response.status, `${path} answered ${response.status}: ${body}`).toBe(200); + + const { traceId } = JSON.parse(body) as { traceId?: string }; + expect(traceId, `${path} did not report a trace id: ${body}`).toMatch(/^[0-9a-f]{32}$/); + + console.log(`${path}: https://${process.env.E2E_TEST_SENTRY_ORG_SLUG}.sentry.io/explore/traces/trace/${traceId}/`); + return traceId!; +} + +/** + * Polls Sentry until the trace holds a span with `op`, then returns that span and its attributes. The + * trace endpoint only carries the span's op, name and place in the tree; the attributes need a second + * lookup, which is polled as well since the two can land apart. + */ +async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceItem; attributes: Attributes }> { + let found: { span: TraceItem; attributes: Attributes } | undefined; + + await expect + .poll(async () => { + const span = await findSpanInTrace(traceId, op); + const attributes = span?.event_id ? await fetchSpanAttributes(traceId, span.event_id) : undefined; + found = span && attributes ? { span, attributes } : undefined; + return found; + }, EVENT_POLLING_OPTIONS) + .toBeDefined(); + + return found!; +} + +/** + * The attributes every decide span carries once Sentry has stored it, whatever the model answers. + * Note the names are the stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` + * attributes the SDK sends. Decisions bills input tokens only, so there are no output tokens. + */ +async function expectDecideSpan(traceId: string): Promise { + const { span, attributes } = await waitForSpan(traceId, GEN_AI_DECIDE); + + expect(span.description).toBe(`decide ${MODEL}`); + expect(attributes).toMatchObject({ + 'span.op': GEN_AI_DECIDE, + 'span.status': 'ok', + origin: 'auto.ai.openai', + [GEN_AI_OPERATION_NAME]: 'decide', + [GEN_AI_PROVIDER_NAME]: 'openai', + [GEN_AI_REQUEST_MODEL]: MODEL, + [GEN_AI_RESPONSE_MODEL]: MODEL, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 0, + }); + expect(attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeGreaterThan(0); + expect(attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(attributes[GEN_AI_USAGE_INPUT_TOKENS]); + + const requestSpan = flattenTrace(await fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); + expect(flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === span.event_id)).toBe(true); + + return attributes; +} + +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { + const traceId = await requestTrace(baseURL, '/decide'); + const attributes = await expectDecideSpan(traceId); + + const [input] = JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string); + expect(input).toMatchObject({ type: 'evaluation', state: expect.stringContaining('refund') }); + expect(input.questions.map((question: { name: string }) => question.name)).toEqual([ + 'wants_refund', + 'department', + 'urgency', + ]); + + const [output] = JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string); + expect(output.type).toBe('evaluation'); + expect(output.answers.map((answer: { type: string; name: string }) => [answer.type, answer.name])).toEqual([ + ['predicate', 'wants_refund'], + ['choice', 'department'], + ['score', 'urgency'], + ]); +}); + +test('records an image decision by media type only', async ({ baseURL }) => { + const traceId = await requestTrace(baseURL, '/decide-image'); + const attributes = await expectDecideSpan(traceId); + + expect(attributes[GEN_AI_INPUT_MESSAGES]).not.toContain('base64'); + const [input] = JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string); + expect(input.state).toEqual([ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', mime_type: 'image/png' }, + ], + }, + ]); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts new file mode 100644 index 000000000000..521b830eefcb --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts @@ -0,0 +1,131 @@ +const authToken = process.env.E2E_TEST_AUTH_TOKEN; +const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; +const sentryTestProject = process.env.E2E_TEST_SENTRY_PROJECT; + +/** + * Spans only become queryable once they have made it through to EAP, which takes + * noticeably longer than the error pipeline (~2min vs ~20s when this was measured). + */ +export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] }; + +/** + * A node of the span tree returned by the organization trace endpoint. Spans, errors and + * occurrences all share this shape and are discriminated by `event_type`. + */ +export interface TraceItem { + /** On a span this is the span id. */ + event_id?: string; + event_type?: 'span' | 'error' | 'occurrence' | 'uptime_check'; + op?: string; + /** On a span this is the span name. */ + description?: string; + children?: TraceItem[]; + errors?: TraceItem[]; + occurrences?: TraceItem[]; +} + +/** + * One request to the Sentry API with the E2E token. Returns `undefined` when the connection + * fails or drops mid-response: over the minutes a test polls, the API closes the odd + * connection ("TypeError: terminated"), and an error thrown inside `expect.poll` fails the + * test instead of letting it retry, so a network error is treated like a "not there yet". + */ +async function fetchSentryApi(what: string, path: string): Promise<{ status: number; body: string } | undefined> { + try { + const response = await fetch(`https://sentry.io/api/0/${path}`, { + headers: { Authorization: `Bearer ${authToken}` }, + }); + return { status: response.status, body: await response.text() }; + } catch (error) { + console.log(`${what} failed: ${error}`); + return undefined; + } +} + +export async function fetchTrace(traceId: string): Promise { + const what = `Trace lookup for ${traceId}`; + const response = await fetchSentryApi(what, `organizations/${sentryTestOrgSlug}/trace/${traceId}/?statsPeriod=1h`); + if (!response) { + return []; + } + + // The trace endpoint is org scoped, so the auth token needs `org:read` on top of the + // project scopes the other assertions rely on. That never resolves by waiting, so fail + // loudly instead of polling until the timeout and reporting it as a missing event. + if (response.status === 401 || response.status === 403) { + throw new Error( + `${what} was rejected with ${response.status}: ${response.body}. ` + + 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', + ); + } + + // Empty traces and the occasional rate limit are expected while polling, so treat anything + // else that is not a success as "not there yet" -- but log it, since a rejected request and + // a trace that has not landed are otherwise indistinguishable. + if (response.status !== 200) { + console.log(`${what} returned ${response.status}: ${response.body}`); + return []; + } + + return JSON.parse(response.body); +} + +/** + * Errors attach to whichever span was active when they were captured, and relocate from the + * top level into that span once it lands, so a given event can surface at any depth. + */ +export function flattenTrace(items: TraceItem[]): TraceItem[] { + return items.flatMap(item => [ + item, + ...flattenTrace(item.children ?? []), + ...flattenTrace(item.errors ?? []), + ...flattenTrace(item.occurrences ?? []), + ]); +} + +export async function findSpanInTrace(traceId: string, op: string): Promise { + return flattenTrace(await fetchTrace(traceId)).find(item => item.op === op); +} + +/** + * The attributes of one span, keyed by name, as Sentry stores them. The trace endpoint only carries + * a span's op, name and place in the tree; the attributes come from the project trace-items + * endpoint. Returns `undefined` while the span has not landed. + * + * Ingest normalises what the SDK sent: `sentry.origin` comes back as `origin`, `sentry.op` as + * `span.op`, `gen_ai.response.text` as `gen_ai.output.messages`, and some numbers are indexed as + * tags (`tags[,number]`). Integers are returned as strings and are converted here; everything + * else is passed through as is. + */ +export async function fetchSpanAttributes( + traceId: string, + spanId: string, +): Promise | undefined> { + const what = `Span lookup for ${spanId}`; + const response = await fetchSentryApi( + what, + `projects/${sentryTestOrgSlug}/${sentryTestProject}/trace-items/${spanId}/?trace_id=${traceId}&item_type=spans`, + ); + if (!response) { + return undefined; + } + + if (response.status === 401 || response.status === 403) { + throw new Error( + `${what} was rejected with ${response.status}: ${response.body}. ` + + 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', + ); + } + + if (response.status !== 200) { + if (response.status !== 404) { + console.log(`${what} returned ${response.status}: ${response.body}`); + } + return undefined; + } + + const { attributes } = JSON.parse(response.body) as { attributes?: { name: string; type: string; value: unknown }[] }; + return Object.fromEntries( + (attributes ?? []).map(({ name, type, value }) => [name, type === 'int' ? Number(value) : value]), + ); +} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json new file mode 100644 index 000000000000..795574fd9df2 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} From 99155fb1cc5c101dbdd43f902074e2d7791bb529 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 15:37:56 +0200 Subject: [PATCH 04/24] Match decide spans by input instead of order in the decisions test --- .../node-integration-tests/suites/tracing/openai/v7/test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index da6527d5cd00..68a5a555ca42 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -179,7 +179,10 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); expect(spans).toHaveLength(3); - const [textSpan, imageSpan] = spans.filter(span => span.status === 'ok'); + // Both successful spans are named `decide gpt-6-luna` and may arrive in any order, so match on the input. + const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value); + const textSpan = spans.find(span => inputOf(span).includes('charged twice')); + const imageSpan = spans.find(span => inputOf(span).includes('input_image')); expect(textSpan!.name).toBe('decide gpt-6-luna'); expect(textSpan!.attributes).toMatchObject({ [SENTRY_ORIGIN]: { value: 'auto.ai.openai' }, From bf0b8b89c71cd33502b5bb84fd826df602632232 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 15:43:36 +0200 Subject: [PATCH 05/24] Use the shared test-utils/cli helpers in the decisions e2e app --- .../package.json | 2 + .../tests/send-to-sentry.test.ts | 24 ++-- .../tests/utils/sentry-api.ts | 131 ------------------ 3 files changed, 15 insertions(+), 142 deletions(-) delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json index 9b9f26414843..f0924e7f3193 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json @@ -20,9 +20,11 @@ }, "devDependencies": { "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", "@sentry/conventions": "0.27.0", "@types/express": "^4.17.21", "@types/node": "^24.0.0", + "sentry": "~0.44.1", "typescript": "~5.9.0" }, "sentryTest": { diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 9797dc210085..93b7eb2c1d45 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -11,14 +11,15 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; -import type { TraceItem } from './utils/sentry-api'; +import type { TraceItem } from '@sentry-internal/test-utils/cli'; import { EVENT_POLLING_OPTIONS, fetchSpanAttributes, fetchTrace, findSpanInTrace, flattenTrace, -} from './utils/sentry-api'; + traceTarget, +} from '@sentry-internal/test-utils/cli'; // Mirrors src/app.ts. const MODEL = 'gpt-6-luna'; @@ -34,7 +35,7 @@ async function requestTrace(baseURL: string | undefined, path: string): Promise< const { traceId } = JSON.parse(body) as { traceId?: string }; expect(traceId, `${path} did not report a trace id: ${body}`).toMatch(/^[0-9a-f]{32}$/); - console.log(`${path}: https://${process.env.E2E_TEST_SENTRY_ORG_SLUG}.sentry.io/explore/traces/trace/${traceId}/`); + console.log(`${path}: sentry trace view ${traceTarget(traceId!)}`); return traceId!; } @@ -47,9 +48,9 @@ async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceIt let found: { span: TraceItem; attributes: Attributes } | undefined; await expect - .poll(async () => { - const span = await findSpanInTrace(traceId, op); - const attributes = span?.event_id ? await fetchSpanAttributes(traceId, span.event_id) : undefined; + .poll(() => { + const span = findSpanInTrace(traceId, op); + const attributes = span?.event_id ? fetchSpanAttributes(traceId, span.event_id) : undefined; found = span && attributes ? { span, attributes } : undefined; return found; }, EVENT_POLLING_OPTIONS) @@ -61,7 +62,8 @@ async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceIt /** * The attributes every decide span carries once Sentry has stored it, whatever the model answers. * Note the names are the stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` - * attributes the SDK sends. Decisions bills input tokens only, so there are no output tokens. + * attributes the SDK sends. Integers come back as strings. Decisions bills input tokens only, so there + * are no output tokens. */ async function expectDecideSpan(traceId: string): Promise { const { span, attributes } = await waitForSpan(traceId, GEN_AI_DECIDE); @@ -75,12 +77,12 @@ async function expectDecideSpan(traceId: string): Promise { [GEN_AI_PROVIDER_NAME]: 'openai', [GEN_AI_REQUEST_MODEL]: MODEL, [GEN_AI_RESPONSE_MODEL]: MODEL, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 0, }); - expect(attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeGreaterThan(0); - expect(attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(attributes[GEN_AI_USAGE_INPUT_TOKENS]); + expect(Number(attributes[GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); + expect(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(attributes[GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])); - const requestSpan = flattenTrace(await fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); + const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); expect(flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === span.event_id)).toBe(true); return attributes; diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts deleted file mode 100644 index 521b830eefcb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/utils/sentry-api.ts +++ /dev/null @@ -1,131 +0,0 @@ -const authToken = process.env.E2E_TEST_AUTH_TOKEN; -const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG; -const sentryTestProject = process.env.E2E_TEST_SENTRY_PROJECT; - -/** - * Spans only become queryable once they have made it through to EAP, which takes - * noticeably longer than the error pipeline (~2min vs ~20s when this was measured). - */ -export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] }; - -/** - * A node of the span tree returned by the organization trace endpoint. Spans, errors and - * occurrences all share this shape and are discriminated by `event_type`. - */ -export interface TraceItem { - /** On a span this is the span id. */ - event_id?: string; - event_type?: 'span' | 'error' | 'occurrence' | 'uptime_check'; - op?: string; - /** On a span this is the span name. */ - description?: string; - children?: TraceItem[]; - errors?: TraceItem[]; - occurrences?: TraceItem[]; -} - -/** - * One request to the Sentry API with the E2E token. Returns `undefined` when the connection - * fails or drops mid-response: over the minutes a test polls, the API closes the odd - * connection ("TypeError: terminated"), and an error thrown inside `expect.poll` fails the - * test instead of letting it retry, so a network error is treated like a "not there yet". - */ -async function fetchSentryApi(what: string, path: string): Promise<{ status: number; body: string } | undefined> { - try { - const response = await fetch(`https://sentry.io/api/0/${path}`, { - headers: { Authorization: `Bearer ${authToken}` }, - }); - return { status: response.status, body: await response.text() }; - } catch (error) { - console.log(`${what} failed: ${error}`); - return undefined; - } -} - -export async function fetchTrace(traceId: string): Promise { - const what = `Trace lookup for ${traceId}`; - const response = await fetchSentryApi(what, `organizations/${sentryTestOrgSlug}/trace/${traceId}/?statsPeriod=1h`); - if (!response) { - return []; - } - - // The trace endpoint is org scoped, so the auth token needs `org:read` on top of the - // project scopes the other assertions rely on. That never resolves by waiting, so fail - // loudly instead of polling until the timeout and reporting it as a missing event. - if (response.status === 401 || response.status === 403) { - throw new Error( - `${what} was rejected with ${response.status}: ${response.body}. ` + - 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', - ); - } - - // Empty traces and the occasional rate limit are expected while polling, so treat anything - // else that is not a success as "not there yet" -- but log it, since a rejected request and - // a trace that has not landed are otherwise indistinguishable. - if (response.status !== 200) { - console.log(`${what} returned ${response.status}: ${response.body}`); - return []; - } - - return JSON.parse(response.body); -} - -/** - * Errors attach to whichever span was active when they were captured, and relocate from the - * top level into that span once it lands, so a given event can surface at any depth. - */ -export function flattenTrace(items: TraceItem[]): TraceItem[] { - return items.flatMap(item => [ - item, - ...flattenTrace(item.children ?? []), - ...flattenTrace(item.errors ?? []), - ...flattenTrace(item.occurrences ?? []), - ]); -} - -export async function findSpanInTrace(traceId: string, op: string): Promise { - return flattenTrace(await fetchTrace(traceId)).find(item => item.op === op); -} - -/** - * The attributes of one span, keyed by name, as Sentry stores them. The trace endpoint only carries - * a span's op, name and place in the tree; the attributes come from the project trace-items - * endpoint. Returns `undefined` while the span has not landed. - * - * Ingest normalises what the SDK sent: `sentry.origin` comes back as `origin`, `sentry.op` as - * `span.op`, `gen_ai.response.text` as `gen_ai.output.messages`, and some numbers are indexed as - * tags (`tags[,number]`). Integers are returned as strings and are converted here; everything - * else is passed through as is. - */ -export async function fetchSpanAttributes( - traceId: string, - spanId: string, -): Promise | undefined> { - const what = `Span lookup for ${spanId}`; - const response = await fetchSentryApi( - what, - `projects/${sentryTestOrgSlug}/${sentryTestProject}/trace-items/${spanId}/?trace_id=${traceId}&item_type=spans`, - ); - if (!response) { - return undefined; - } - - if (response.status === 401 || response.status === 403) { - throw new Error( - `${what} was rejected with ${response.status}: ${response.body}. ` + - 'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.', - ); - } - - if (response.status !== 200) { - if (response.status !== 404) { - console.log(`${what} returned ${response.status}: ${response.body}`); - } - return undefined; - } - - const { attributes } = JSON.parse(response.body) as { attributes?: { name: string; type: string; value: unknown }[] }; - return Object.fromEntries( - (attributes ?? []).map(({ name, type, value }) => [name, type === 'int' ? Number(value) : value]), - ); -} From 19ed67b2255d676cb78c6c4a27de491648ce14bd Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 15:48:50 +0200 Subject: [PATCH 06/24] Inline the trace id lookup in the decisions e2e app --- .../node-openai-decisions-send-to-sentry/src/app.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts index 77ca388052f4..f37d3198784c 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts @@ -20,11 +20,6 @@ const RED_PIXEL = const app = express(); -/** The trace this request is recorded under, so the test can look its spans up in Sentry. */ -function currentTraceId(): string | undefined { - return Sentry.getActiveSpan()?.spanContext().traceId; -} - // Each route makes one `decisions.create` call and answers with the trace id, so the test can find // the request's spans in Sentry. @@ -52,7 +47,7 @@ app.get('/decide', async (_req, res, next) => { }, ], }); - res.send({ traceId: currentTraceId(), answers: decision.answers }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); } catch (error) { next(error); } @@ -73,7 +68,7 @@ app.get('/decide-image', async (_req, res, next) => { ], questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], }); - res.send({ traceId: currentTraceId(), answers: decision.answers }); + res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); } catch (error) { next(error); } From 28c77c18c275581500c85d6f2791bd2d8b92c790 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 16:05:34 +0200 Subject: [PATCH 07/24] Assert the decisions e2e messages against the full recorded JSON --- .../tests/send-to-sentry.test.ts | 85 ++++++++++++++----- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 93b7eb2c1d45..7f1d3a82941c 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -92,20 +92,59 @@ test('records a text decision with its state, questions and answers', async ({ b const traceId = await requestTrace(baseURL, '/decide'); const attributes = await expectDecideSpan(traceId); - const [input] = JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string); - expect(input).toMatchObject({ type: 'evaluation', state: expect.stringContaining('refund') }); - expect(input.questions.map((question: { name: string }) => question.name)).toEqual([ - 'wants_refund', - 'department', - 'urgency', + expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice this month and want a refund.', + questions: [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [ + { value: 'billing', description: 'Charges and refunds' }, + { value: 'other', description: 'Anything else' }, + ], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high' }], + }, + ], + }, ]); - const [output] = JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string); - expect(output.type).toBe('evaluation'); - expect(output.answers.map((answer: { type: string; name: string }) => [answer.type, answer.name])).toEqual([ - ['predicate', 'wants_refund'], - ['choice', 'department'], - ['score', 'urgency'], + // The probabilities depend on the model, the shape does not. + expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: [ + { type: 'predicate', name: 'wants_refund', probability: expect.any(Number) }, + { + type: 'choice', + name: 'department', + choice: 'billing', + probabilities: [ + { value: 'billing', probability: expect.any(Number) }, + { value: 'other', probability: expect.any(Number) }, + ], + confidence: expect.any(Number), + }, + { + type: 'score', + name: 'urgency', + score: expect.any(Number), + probabilities: [ + { value: 0, label: 'low', probability: expect.any(Number) }, + { value: 1, label: 'high', probability: expect.any(Number) }, + ], + confidence: expect.any(Number), + }, + ], + }, ]); }); @@ -113,15 +152,23 @@ test('records an image decision by media type only', async ({ baseURL }) => { const traceId = await requestTrace(baseURL, '/decide-image'); const attributes = await expectDecideSpan(traceId); - expect(attributes[GEN_AI_INPUT_MESSAGES]).not.toContain('base64'); - const [input] = JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string); - expect(input.state).toEqual([ + expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ { - role: 'user', - content: [ - { type: 'input_text', text: 'Is this image mostly red?' }, - { type: 'input_image', mime_type: 'image/png' }, + type: 'evaluation', + state: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is this image mostly red?' }, + { type: 'input_image', mime_type: 'image/png' }, + ], + }, ], + questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], }, ]); + + expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { type: 'evaluation', answers: [{ type: 'predicate', name: 'is_red', probability: expect.any(Number) }] }, + ]); }); From ec536bcd19a48407b90ed6907beee071d21c9f8b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 16:13:13 +0200 Subject: [PATCH 08/24] Poll for the request span parent in the decisions e2e test --- .../tests/send-to-sentry.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 7f1d3a82941c..99f58f11044a 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -82,8 +82,13 @@ async function expectDecideSpan(traceId: string): Promise { expect(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); expect(Number(attributes[GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])); - const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); - expect(flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === span.event_id)).toBe(true); + // The request span is the segment; it ends last, so it can land after the decide span. + await expect + .poll(() => { + const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); + return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === span.event_id); + }, EVENT_POLLING_OPTIONS) + .toBe(true); return attributes; } From 12dae22e1b1604aaae6484fd682cfef4610ca403 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 09:37:25 +0200 Subject: [PATCH 09/24] Assert the stored span name in the decisions e2e test --- .../tests/send-to-sentry.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 99f58f11044a..bd7326e11b9c 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -68,8 +68,9 @@ async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceIt async function expectDecideSpan(traceId: string): Promise { const { span, attributes } = await waitForSpan(traceId, GEN_AI_DECIDE); - expect(span.description).toBe(`decide ${MODEL}`); + // `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. expect(attributes).toMatchObject({ + 'span.name': `decide ${MODEL}`, 'span.op': GEN_AI_DECIDE, 'span.status': 'ok', origin: 'auto.ai.openai', From 4fefcd6096676e5d97945b15115f10b45c6cff0b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 11:33:17 +0200 Subject: [PATCH 10/24] Lay out decision choice and score answers like Jev answers, keyed by question name --- .../tests/send-to-sentry.test.ts | 39 ++++++--------- .../suites/tracing/openai/v7/test.ts | 35 +++++-------- packages/server-utils/src/ai/openai/utils.ts | 49 ++++++++++++++++++- .../test/ai/lib/tracing/openai.test.ts | 13 +++-- 4 files changed, 85 insertions(+), 51 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index bd7326e11b9c..3ecd2142e769 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -102,24 +102,22 @@ test('records a text decision with its state, questions and answers', async ({ b { type: 'evaluation', state: 'I was charged twice this month and want a refund.', - questions: [ - { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, - { + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { type: 'choice', - name: 'department', instructions: 'Which team should handle this?', choices: [ { value: 'billing', description: 'Charges and refunds' }, { value: 'other', description: 'Anything else' }, ], }, - { + urgency: { type: 'score', - name: 'urgency', instructions: 'How urgent is this ticket?', levels: [{ label: 'low' }, { label: 'high' }], }, - ], + }, }, ]); @@ -127,29 +125,22 @@ test('records a text decision with its state, questions and answers', async ({ b expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', - answers: [ - { type: 'predicate', name: 'wants_refund', probability: expect.any(Number) }, - { + answers: { + wants_refund: { type: 'predicate', probability: expect.any(Number) }, + department: { type: 'choice', - name: 'department', choice: 'billing', - probabilities: [ - { value: 'billing', probability: expect.any(Number) }, - { value: 'other', probability: expect.any(Number) }, - ], + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, confidence: expect.any(Number), }, - { + urgency: { type: 'score', - name: 'urgency', score: expect.any(Number), - probabilities: [ - { value: 0, label: 'low', probability: expect.any(Number) }, - { value: 1, label: 'high', probability: expect.any(Number) }, - ], + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, + legend: { 0: 'low', 1: 'high' }, confidence: expect.any(Number), }, - ], + }, }, ]); }); @@ -170,11 +161,11 @@ test('records an image decision by media type only', async ({ baseURL }) => { ], }, ], - questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], + questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, }, ]); expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ - { type: 'evaluation', answers: [{ type: 'predicate', name: 'is_red', probability: expect.any(Number) }] }, + { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, ]); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 68a5a555ca42..0039d1826700 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -200,49 +200,40 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { { type: 'evaluation', state: 'I was charged twice and want a refund.', - questions: [ - { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, - { + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { type: 'choice', - name: 'department', instructions: 'Which team should handle this?', choices: [{ value: 'billing' }, { value: 'other' }], }, - { + urgency: { type: 'score', - name: 'urgency', instructions: 'How urgent is this ticket?', levels: [{ label: 'low' }, { label: 'high' }], }, - ], + }, }, ]); expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ { type: 'evaluation', - answers: [ - { type: 'predicate', name: 'wants_refund', probability: 0.97 }, - { + answers: { + wants_refund: { type: 'predicate', probability: 0.97 }, + department: { type: 'choice', - name: 'department', choice: 'billing', - probabilities: [ - { value: 'billing', probability: 0.82 }, - { value: 'other', probability: 0.18 }, - ], + probabilities: { billing: 0.82, other: 0.18 }, confidence: 0.82, }, - { + urgency: { type: 'score', - name: 'urgency', score: 0.95, - probabilities: [ - { value: 0, label: 'low', probability: 0.05 }, - { value: 1, label: 'high', probability: 0.95 }, - ], + probabilities: { 0: 0.05, 1: 0.95 }, + legend: { 0: 'low', 1: 'high' }, confidence: 0.93, }, - ], + }, }, ]); diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index f90c86840d23..d50bb9cd6bc5 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -132,7 +132,7 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs // Typed answers (decisions API) if (Array.isArray(response.answers)) { - const outputMessages = getEvaluationOutputMessages(response.answers); + const outputMessages = getEvaluationOutputMessages(keyByName(response.answers.map(toJevAnswerFormat))); if (outputMessages) { attrs[GEN_AI_OUTPUT_MESSAGES] = outputMessages; } @@ -232,7 +232,52 @@ export function extractRequestParameters(params: Record): Recor * so both render the same. Images are base64 data URLs, so they are reported by media type only. */ export function getDecisionInputMessages(params: Record): string | undefined { - return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: params.questions }); + return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(params.questions) }); +} + +/** + * Lay out choice and score answers like Jev answers: probabilities keyed by option or score value, with + * score labels in a `legend`. Predicate answers are already in the shape Sentry renders. + */ +function toJevAnswerFormat(answer: unknown): unknown { + if ( + !isObjectLike(answer) || + (answer.type !== 'choice' && answer.type !== 'score') || + !Array.isArray(answer.probabilities) + ) { + return answer; + } + + const options = answer.probabilities.filter(isObjectLike); + const probabilities = Object.fromEntries(options.map(option => [String(option.value), option.probability])); + if (answer.type === 'choice') { + return { ...answer, probabilities }; + } + + const legend = Object.fromEntries( + options.filter(option => typeof option.label === 'string').map(option => [String(option.value), option.label]), + ); + return { ...answer, probabilities, legend }; +} + +/** + * The decisions API lists questions and answers as arrays with an optional `name`, where the conventions + * key them by question name. An unnamed item is keyed by its position. + */ +function keyByName(items: unknown): unknown { + if (!Array.isArray(items)) { + return items; + } + + return Object.fromEntries( + items.map((item, index) => { + if (!isObjectLike(item)) { + return [String(index), item]; + } + const { name, ...rest } = item; + return [typeof name === 'string' ? name : String(index), rest]; + }), + ); } function withoutImageData(input: unknown): unknown { diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index 6946b097f3ed..fa07c337dca4 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -95,9 +95,13 @@ describe('instrumentOpenAiClient span names', () => { it('records `decisions.create` as a decide span', async () => { const endedSpans = setupClient('stream'); - const answers = [{ type: 'predicate', name: 'wants_refund', probability: 0.97 }]; const instrumented = instrumentOpenAiClient({ - decisions: { create: vi.fn().mockResolvedValue({ model: 'gpt-6-luna', answers }) }, + decisions: { + create: vi.fn().mockResolvedValue({ + model: 'gpt-6-luna', + answers: [{ type: 'predicate', name: 'wants_refund', probability: 0.97 }], + }), + }, }); await instrumented.decisions.create({ @@ -109,6 +113,9 @@ describe('instrumentOpenAiClient span names', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('decide gpt-6-luna'); expect(json.op).toBe(GEN_AI_DECIDE); - expect(JSON.parse(json.data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([{ type: 'evaluation', answers }]); + expect(JSON.parse(json.data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + // Keyed by question name, as the conventions define it. + { type: 'evaluation', answers: { wants_refund: { type: 'predicate', probability: 0.97 } } }, + ]); }); }); From 95ff42d45997a44b2c566d0110bb204f27135d6d Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 11:56:35 +0200 Subject: [PATCH 11/24] Lay out decision questions like Jev questions, with options in criteria --- .../tests/send-to-sentry.test.ts | 7 ++--- .../tracing/openai/v7/scenario-decisions.mjs | 2 +- .../suites/tracing/openai/v7/test.ts | 5 +-- packages/server-utils/src/ai/openai/utils.ts | 31 ++++++++++++++++++- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 3ecd2142e769..0c0041ed9282 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -107,15 +107,12 @@ test('records a text decision with its state, questions and answers', async ({ b department: { type: 'choice', instructions: 'Which team should handle this?', - choices: [ - { value: 'billing', description: 'Charges and refunds' }, - { value: 'other', description: 'Anything else' }, - ], + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, }, urgency: { type: 'score', instructions: 'How urgent is this ticket?', - levels: [{ label: 'low' }, { label: 'high' }], + criteria: ['low', 'high'], }, }, }, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs index 5de618289c83..70d5dbbd9037 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -63,7 +63,7 @@ const questions = [ type: 'choice', name: 'department', instructions: 'Which team should handle this?', - choices: [{ value: 'billing' }, { value: 'other' }], + choices: [{ value: 'billing', description: 'Charges and refunds' }, { value: 'other' }], }, { type: 'score', diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 0039d1826700..9bb080c31963 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -205,12 +205,13 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { department: { type: 'choice', instructions: 'Which team should handle this?', - choices: [{ value: 'billing' }, { value: 'other' }], + // `other` has no description, so it is left out of the criteria. + criteria: { billing: 'Charges and refunds' }, }, urgency: { type: 'score', instructions: 'How urgent is this ticket?', - levels: [{ label: 'low' }, { label: 'high' }], + criteria: ['low', 'high'], }, }, }, diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index d50bb9cd6bc5..7eceab317224 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -232,7 +232,36 @@ export function extractRequestParameters(params: Record): Recor * so both render the same. Images are base64 data URLs, so they are reported by media type only. */ export function getDecisionInputMessages(params: Record): string | undefined { - return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(params.questions) }); + const questions = Array.isArray(params.questions) ? params.questions.map(toJevQuestionFormat) : params.questions; + return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(questions) }); +} + +/** + * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as + * `{ [value]: description }` (undescribed choices are left out), and score levels as their labels, + * indexed by score value. + */ +function toJevQuestionFormat(question: unknown): unknown { + if (!isObjectLike(question)) { + return question; + } + + if (question.type === 'choice' && Array.isArray(question.choices)) { + const { choices, ...rest } = question; + const described = choices.filter( + (choice): choice is Record => isObjectLike(choice) && typeof choice.description === 'string', + ); + return described.length + ? { ...rest, criteria: Object.fromEntries(described.map(choice => [String(choice.value), choice.description])) } + : rest; + } + + if (question.type === 'score' && Array.isArray(question.levels)) { + const { levels, ...rest } = question; + return { ...rest, criteria: levels.map(level => (isObjectLike(level) ? level.label : level)) }; + } + + return question; } /** From 2c6bbafe5a8f4038ebc52d6f20e9e26caf7a08de Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 13:09:41 +0200 Subject: [PATCH 12/24] Drop image data from top-level decisions input parts --- packages/server-utils/src/ai/openai/utils.ts | 9 +++---- .../test/ai/lib/tracing/openai.test.ts | 24 ++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index ac663f8c8129..de18792e3821 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -307,10 +307,11 @@ function withoutImageData(input: unknown): unknown { return input; } - return input.map(message => - isObjectLike(message) && Array.isArray(message.content) - ? { ...message, content: message.content.map(withoutImageDataPart) } - : message, + // Parts are also handled at the top level, in case a flat list of parts is passed instead of messages. + return input.map(item => + isObjectLike(item) && Array.isArray(item.content) + ? { ...item, content: item.content.map(withoutImageDataPart) } + : withoutImageDataPart(item), ); } diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index fa07c337dca4..66d5fde0eb14 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; @@ -118,4 +118,26 @@ describe('instrumentOpenAiClient span names', () => { { type: 'evaluation', answers: { wants_refund: { type: 'predicate', probability: 0.97 } } }, ]); }); + + it('records only the media type of a decisions image passed as a top-level part', async () => { + const endedSpans = setupClient('stream'); + const instrumented = instrumentOpenAiClient({ + decisions: { create: vi.fn().mockResolvedValue({ model: 'gpt-6-luna', answers: [] }) }, + }); + + await instrumented.decisions.create({ + model: 'gpt-6-luna', + input: [ + { type: 'input_text', text: 'Is this red?' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' }, + ], + questions: [], + }); + + const [input] = JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_INPUT_MESSAGES] as string); + expect(input.state).toEqual([ + { type: 'input_text', text: 'Is this red?' }, + { type: 'input_image', mime_type: 'image/png' }, + ]); + }); }); From 50bba017fb5c3be03fccf5ec1858fd9910a04516 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 13:13:27 +0200 Subject: [PATCH 13/24] Record the decisions input when the span starts --- packages/server-utils/src/ai/openai/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 08f1e4541feb..18a919761956 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -79,6 +79,11 @@ export function extractRequestAttributes( } Object.assign(attributes, extractRequestParameters(params)); + + const decisionInput = recordInputs && operationName === 'decide' ? getDecisionInputMessages(params) : undefined; + if (decisionInput) { + attributes[GEN_AI_INPUT_MESSAGES] = decisionInput; + } } else { attributes[GEN_AI_REQUEST_MODEL] = 'unknown'; } @@ -88,11 +93,8 @@ export function extractRequestAttributes( // Extract and record AI request inputs, if present. This is intentionally separate from response attributes. export function addRequestAttributes(span: Span, params: Record, operationName: string): void { + // A decision's input is recorded when its span starts, in `extractRequestAttributes`. if (operationName === 'decide') { - const inputMessages = getDecisionInputMessages(params); - if (inputMessages) { - span.setAttribute(GEN_AI_INPUT_MESSAGES, inputMessages); - } return; } From b61e351e98f389f89345de5ca03c125b58c2a6b6 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 13:34:42 +0200 Subject: [PATCH 14/24] Write the decisions e2e test steps inline, like the Cloudflare app --- .../tests/send-to-sentry.test.ts | 116 ++++++++---------- 1 file changed, 50 insertions(+), 66 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts index 0c0041ed9282..6f1f1155f54d 100644 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts @@ -11,7 +11,6 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; -import type { TraceItem } from '@sentry-internal/test-utils/cli'; import { EVENT_POLLING_OPTIONS, fetchSpanAttributes, @@ -24,81 +23,53 @@ import { // Mirrors src/app.ts. const MODEL = 'gpt-6-luna'; -type Attributes = Record; - -/** Sends one request to the app and returns the trace its request span was recorded under. */ -async function requestTrace(baseURL: string | undefined, path: string): Promise { - const response = await fetch(`${baseURL}${path}`); - const body = await response.text(); - expect(response.status, `${path} answered ${response.status}: ${body}`).toBe(200); - - const { traceId } = JSON.parse(body) as { traceId?: string }; - expect(traceId, `${path} did not report a trace id: ${body}`).toMatch(/^[0-9a-f]{32}$/); - - console.log(`${path}: sentry trace view ${traceTarget(traceId!)}`); - return traceId!; -} - /** - * Polls Sentry until the trace holds a span with `op`, then returns that span and its attributes. The - * trace endpoint only carries the span's op, name and place in the tree; the attributes need a second - * lookup, which is polled as well since the two can land apart. + * What every decide span carries once Sentry has stored it, whatever the model answers. The names are the + * stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` attributes the SDK sends. + * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. */ -async function waitForSpan(traceId: string, op: string): Promise<{ span: TraceItem; attributes: Attributes }> { - let found: { span: TraceItem; attributes: Attributes } | undefined; +const DECIDE_SPAN_ATTRIBUTES = { + 'span.name': `decide ${MODEL}`, + 'span.op': GEN_AI_DECIDE, + 'span.status': 'ok', + origin: 'auto.ai.openai', + [GEN_AI_OPERATION_NAME]: 'decide', + [GEN_AI_PROVIDER_NAME]: 'openai', + [GEN_AI_REQUEST_MODEL]: MODEL, + [GEN_AI_RESPONSE_MODEL]: MODEL, +}; + +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { + const response = await fetch(`${baseURL}/decide`); + expect(response.status).toBe(200); + const { traceId }: { traceId: string } = await response.json(); + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; await expect - .poll(() => { - const span = findSpanInTrace(traceId, op); - const attributes = span?.event_id ? fetchSpanAttributes(traceId, span.event_id) : undefined; - found = span && attributes ? { span, attributes } : undefined; - return found; - }, EVENT_POLLING_OPTIONS) + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) .toBeDefined(); - return found!; -} + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); -/** - * The attributes every decide span carries once Sentry has stored it, whatever the model answers. - * Note the names are the stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` - * attributes the SDK sends. Integers come back as strings. Decisions bills input tokens only, so there - * are no output tokens. - */ -async function expectDecideSpan(traceId: string): Promise { - const { span, attributes } = await waitForSpan(traceId, GEN_AI_DECIDE); - - // `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. - expect(attributes).toMatchObject({ - 'span.name': `decide ${MODEL}`, - 'span.op': GEN_AI_DECIDE, - 'span.status': 'ok', - origin: 'auto.ai.openai', - [GEN_AI_OPERATION_NAME]: 'decide', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: MODEL, - [GEN_AI_RESPONSE_MODEL]: MODEL, - }); - expect(Number(attributes[GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); - expect(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); - expect(Number(attributes[GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes[GEN_AI_USAGE_INPUT_TOKENS])); + // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. + expect(Number(attributes![GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); + expect(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); + expect(Number(attributes![GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])); // The request span is the segment; it ends last, so it can land after the decide span. await expect .poll(() => { const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); - return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === span.event_id); + return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); }, EVENT_POLLING_OPTIONS) .toBe(true); - return attributes; -} - -test('records a text decision with its state, questions and answers', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/decide'); - const attributes = await expectDecideSpan(traceId); - - expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', state: 'I was charged twice this month and want a refund.', @@ -119,7 +90,7 @@ test('records a text decision with its state, questions and answers', async ({ b ]); // The probabilities depend on the model, the shape does not. - expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { @@ -143,10 +114,23 @@ test('records a text decision with its state, questions and answers', async ({ b }); test('records an image decision by media type only', async ({ baseURL }) => { - const traceId = await requestTrace(baseURL, '/decide-image'); - const attributes = await expectDecideSpan(traceId); + const response = await fetch(`${baseURL}/decide-image`); + expect(response.status).toBe(200); + const { traceId }: { traceId: string } = await response.json(); + + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); + + let spanId: string | undefined; + await expect + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) + .toBeDefined(); + + let attributes: Record | undefined; + await expect + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', state: [ @@ -162,7 +146,7 @@ test('records an image decision by media type only', async ({ baseURL }) => { }, ]); - expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, ]); }); From d09c00937bc1e1f52772535b3a6c3f8134d71288 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 13:56:29 +0200 Subject: [PATCH 15/24] Record decision answers only for decide calls and move the mapping to mapping.ts --- packages/server-utils/src/ai/openai/index.ts | 5 +- .../server-utils/src/ai/openai/mapping.ts | 115 +++++++++++++++++ packages/server-utils/src/ai/openai/utils.ts | 120 ++---------------- .../server-utils/src/integrations/openai.ts | 2 +- 4 files changed, 130 insertions(+), 112 deletions(-) create mode 100644 packages/server-utils/src/ai/openai/mapping.ts diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 18a919761956..dde6801dc5a1 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -31,7 +31,8 @@ import { import { OPENAI_METHOD_REGISTRY } from './constants'; import { instrumentStream } from './streaming'; import type { ChatCompletionChunk, OpenAiOptions, OpenAIStream, ResponseStreamingEvent } from './types'; -import { addResponseAttributes, extractRequestParameters, getDecisionInputMessages } from './utils'; +import { getDecisionInputMessages } from './mapping'; +import { addResponseAttributes, extractRequestParameters } from './utils'; /** * Extract available tools from request parameters @@ -213,7 +214,7 @@ function instrumentMethod( } return originalResult.then(result => { - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, options.recordOutputs, operationName); return result; }); }); diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts new file mode 100644 index 000000000000..b965738993f2 --- /dev/null +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -0,0 +1,115 @@ +/** + * Maps OpenAI decisions requests and responses to the format Jev calls are recorded in, so Sentry renders + * both the same way. + */ +import { isObjectLike } from '@sentry/core'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../core/utils'; + +/** + * Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state` + * so both render the same. Images are base64 data URLs, so they are reported by media type only. + */ +export function getDecisionInputMessages(params: Record): string | undefined { + const questions = Array.isArray(params.questions) ? params.questions.map(toJevQuestionFormat) : params.questions; + return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(questions) }); +} + +/** Serialize the answers of a decisions response, laid out and keyed like Jev answers. */ +export function getDecisionOutputMessages(answers: unknown[]): string | undefined { + return getEvaluationOutputMessages(keyByName(answers.map(toJevAnswerFormat))); +} + +/** + * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as + * `{ [value]: description }` (undescribed choices are left out), and score levels as their labels, + * indexed by score value. + */ +function toJevQuestionFormat(question: unknown): unknown { + if (!isObjectLike(question)) { + return question; + } + + if (question.type === 'choice' && Array.isArray(question.choices)) { + const { choices, ...rest } = question; + const described = choices.filter( + (choice): choice is Record => isObjectLike(choice) && typeof choice.description === 'string', + ); + return described.length + ? { ...rest, criteria: Object.fromEntries(described.map(choice => [String(choice.value), choice.description])) } + : rest; + } + + if (question.type === 'score' && Array.isArray(question.levels)) { + const { levels, ...rest } = question; + return { ...rest, criteria: levels.map(level => (isObjectLike(level) ? level.label : level)) }; + } + + return question; +} + +/** + * Lay out choice and score answers like Jev answers: probabilities keyed by option or score value, with + * score labels in a `legend`. Predicate answers are already in the shape Sentry renders. + */ +function toJevAnswerFormat(answer: unknown): unknown { + if ( + !isObjectLike(answer) || + (answer.type !== 'choice' && answer.type !== 'score') || + !Array.isArray(answer.probabilities) + ) { + return answer; + } + + const options = answer.probabilities.filter(isObjectLike); + const probabilities = Object.fromEntries(options.map(option => [String(option.value), option.probability])); + if (answer.type === 'choice') { + return { ...answer, probabilities }; + } + + const legend = Object.fromEntries( + options.filter(option => typeof option.label === 'string').map(option => [String(option.value), option.label]), + ); + return { ...answer, probabilities, legend }; +} + +/** + * The decisions API lists questions and answers as arrays with an optional `name`, where the conventions + * key them by question name. An unnamed item is keyed by its position. + */ +function keyByName(items: unknown): unknown { + if (!Array.isArray(items)) { + return items; + } + + return Object.fromEntries( + items.map((item, index) => { + if (!isObjectLike(item)) { + return [String(index), item]; + } + const { name, ...rest } = item; + return [typeof name === 'string' ? name : String(index), rest]; + }), + ); +} + +function withoutImageData(input: unknown): unknown { + if (!Array.isArray(input)) { + return input; + } + + // Parts are also handled at the top level, in case a flat list of parts is passed instead of messages. + return input.map(item => + isObjectLike(item) && Array.isArray(item.content) + ? { ...item, content: item.content.map(withoutImageDataPart) } + : withoutImageDataPart(item), + ); +} + +function withoutImageDataPart(part: unknown): unknown { + if (!isObjectLike(part) || part.type !== 'input_image' || typeof part.image_url !== 'string') { + return part; + } + + const mimeType = /^data:([^;,]+)/.exec(part.image_url)?.[1]; + return mimeType ? { type: 'input_image', mime_type: mimeType } : { type: 'input_image' }; +} diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index de18792e3821..7c2190585e9a 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -26,7 +26,7 @@ import { GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE, GEN_AI_REQUEST_STREAM_ATTRIBUTE, } from '../core/gen-ai-attributes'; -import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../core/utils'; +import { getDecisionOutputMessages } from './mapping'; import type { ChatCompletionChunk, ResponseStreamingEvent } from './types'; /** @@ -56,9 +56,15 @@ export function isChatCompletionChunk(event: unknown): event is ChatCompletionCh /** * Add response attributes to a span using duck-typing. - * Works for Chat Completions, Responses API, Embeddings, and Conversations API responses. + * Works for Chat Completions, Responses API, Embeddings, and Conversations API responses. Decision + * answers are only recorded when `operationName` is `decide`. */ -export function addResponseAttributes(span: Span, result: unknown, recordOutputs?: boolean): void { +export function addResponseAttributes( + span: Span, + result: unknown, + recordOutputs?: boolean, + operationName?: string, +): void { if (!result || typeof result !== 'object') return; const response = result as Record; @@ -131,8 +137,8 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs } // Typed answers (decisions API) - if (Array.isArray(response.answers)) { - const outputMessages = getEvaluationOutputMessages(keyByName(response.answers.map(toJevAnswerFormat))); + if (operationName === 'decide' && Array.isArray(response.answers)) { + const outputMessages = getDecisionOutputMessages(response.answers); if (outputMessages) { attrs[GEN_AI_OUTPUT_MESSAGES] = outputMessages; } @@ -219,107 +225,3 @@ export function extractRequestParameters(params: Record): Recor return attributes; } - -/** - * Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state` - * so both render the same. Images are base64 data URLs, so they are reported by media type only. - */ -export function getDecisionInputMessages(params: Record): string | undefined { - const questions = Array.isArray(params.questions) ? params.questions.map(toJevQuestionFormat) : params.questions; - return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(questions) }); -} - -/** - * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as - * `{ [value]: description }` (undescribed choices are left out), and score levels as their labels, - * indexed by score value. - */ -function toJevQuestionFormat(question: unknown): unknown { - if (!isObjectLike(question)) { - return question; - } - - if (question.type === 'choice' && Array.isArray(question.choices)) { - const { choices, ...rest } = question; - const described = choices.filter( - (choice): choice is Record => isObjectLike(choice) && typeof choice.description === 'string', - ); - return described.length - ? { ...rest, criteria: Object.fromEntries(described.map(choice => [String(choice.value), choice.description])) } - : rest; - } - - if (question.type === 'score' && Array.isArray(question.levels)) { - const { levels, ...rest } = question; - return { ...rest, criteria: levels.map(level => (isObjectLike(level) ? level.label : level)) }; - } - - return question; -} - -/** - * Lay out choice and score answers like Jev answers: probabilities keyed by option or score value, with - * score labels in a `legend`. Predicate answers are already in the shape Sentry renders. - */ -function toJevAnswerFormat(answer: unknown): unknown { - if ( - !isObjectLike(answer) || - (answer.type !== 'choice' && answer.type !== 'score') || - !Array.isArray(answer.probabilities) - ) { - return answer; - } - - const options = answer.probabilities.filter(isObjectLike); - const probabilities = Object.fromEntries(options.map(option => [String(option.value), option.probability])); - if (answer.type === 'choice') { - return { ...answer, probabilities }; - } - - const legend = Object.fromEntries( - options.filter(option => typeof option.label === 'string').map(option => [String(option.value), option.label]), - ); - return { ...answer, probabilities, legend }; -} - -/** - * The decisions API lists questions and answers as arrays with an optional `name`, where the conventions - * key them by question name. An unnamed item is keyed by its position. - */ -function keyByName(items: unknown): unknown { - if (!Array.isArray(items)) { - return items; - } - - return Object.fromEntries( - items.map((item, index) => { - if (!isObjectLike(item)) { - return [String(index), item]; - } - const { name, ...rest } = item; - return [typeof name === 'string' ? name : String(index), rest]; - }), - ); -} - -function withoutImageData(input: unknown): unknown { - if (!Array.isArray(input)) { - return input; - } - - // Parts are also handled at the top level, in case a flat list of parts is passed instead of messages. - return input.map(item => - isObjectLike(item) && Array.isArray(item.content) - ? { ...item, content: item.content.map(withoutImageDataPart) } - : withoutImageDataPart(item), - ); -} - -function withoutImageDataPart(part: unknown): unknown { - if (!isObjectLike(part) || part.type !== 'input_image' || typeof part.image_url !== 'string') { - return part; - } - - const mimeType = /^data:([^;,]+)/.exec(part.image_url)?.[1]; - return mimeType ? { type: 'input_image', mime_type: mimeType } : { type: 'input_image' }; -} diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index 15c49c217fa3..eaa02e867a5d 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -49,7 +49,7 @@ function instrumentOpenai(options: OpenAiOptions): void { data => createGenAiSpan(data, operation, options), { beforeSpanEnd: (span, data) => { - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs, operation); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( From 0ba719e97aa760da997405ed09511b8f11b8095b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 14:17:15 +0200 Subject: [PATCH 16/24] Limit the decisions hook to openai 7.30.0 and later --- packages/server-utils/src/orchestrion/config/openai.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server-utils/src/orchestrion/config/openai.ts b/packages/server-utils/src/orchestrion/config/openai.ts index 468d7f8392db..8024cb4b0b67 100644 --- a/packages/server-utils/src/orchestrion/config/openai.ts +++ b/packages/server-utils/src/orchestrion/config/openai.ts @@ -30,7 +30,7 @@ export const openaiConfig = [ // OpenAI decisions API (`openai` >= 7.30.0), which returns typed answers rather than generated text. ...['resources/decisions.js', 'resources/decisions.mjs'].map(filePath => ({ channelName: 'decide', - module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, + module: { name: 'openai', versionRange: '>=7.30.0 <8', filePath }, functionQuery: { className: 'Decisions', methodName: 'create', kind: 'Sync' as const }, })), ] satisfies InstrumentationConfig[]; From 2eec966c14c13302ef387303539dcc9ad6d81d5b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Tue, 13 Oct 2026 15:41:20 +0200 Subject: [PATCH 17/24] Move the decisions e2e app to its own PR --- .github/workflows/build.yml | 4 - dev-packages/e2e-tests/.env.example | 4 - .../.gitignore | 5 - .../package.json | 43 ----- .../playwright.config.ts | 28 ---- .../src/app.ts | 85 ---------- .../src/instrument.ts | 10 -- .../tests/send-to-sentry.test.ts | 152 ------------------ .../tsconfig.json | 13 -- 9 files changed, 344 deletions(-) delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99ad92d4515d..55dcd184d04c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1197,10 +1197,6 @@ jobs: E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks' E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests' E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }} - # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key. - E2E_OPENAI_DECISIONS_API_KEY: - ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY || - '' }} # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} diff --git a/dev-packages/e2e-tests/.env.example b/dev-packages/e2e-tests/.env.example index 3815cae556f0..180fb07078a1 100644 --- a/dev-packages/e2e-tests/.env.example +++ b/dev-packages/e2e-tests/.env.example @@ -16,10 +16,6 @@ E2E_TEST_SENTRY_PROJECT= # Only needed to run that test app locally. E2E_OPENROUTER_API_KEY= -# An OpenAI API key for the Decisions API (OpenRouter has no Decisions route). -# Only needed to run node-openai-decisions-send-to-sentry locally. -E2E_OPENAI_DECISIONS_API_KEY= - # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry). # The API token needs "Workers Scripts: Edit" on the account; "Workers KV Storage: Read" additionally silences a # warning when a worker is deleted. Leave it empty to use a `wrangler login` session instead. diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore deleted file mode 100644 index 795f6b6d40d1..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -pnpm-lock.yaml -dist -test-results -playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json deleted file mode 100644 index f0924e7f3193..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "node-openai-decisions-send-to-sentry", - "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "build": "tsc", - "start": "node --import ./dist/instrument.js dist/app.js", - "test": "playwright test", - "clean": "npx rimraf node_modules dist pnpm-lock.yaml", - "test:build": "pnpm install && pnpm build", - "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build", - "test:assert": "pnpm test" - }, - "dependencies": { - "@sentry/node": "file:../../packed/sentry-node-packed.tgz", - "express": "^4.21.2", - "openai": "7.32.0" - }, - "devDependencies": { - "@playwright/test": "~1.63.0", - "@sentry-internal/test-utils": "link:../../../test-utils", - "@sentry/conventions": "0.27.0", - "@types/express": "^4.17.21", - "@types/node": "^24.0.0", - "sentry": "~0.44.1", - "typescript": "~5.9.0" - }, - "sentryTest": { - "optional": true, - "optionalVariants": [ - { - "build-command": "pnpm test:build-latest", - "label": "node-openai-decisions-send-to-sentry (latest)" - } - ] - }, - "volta": { - "node": "24.15.0", - "extends": "../../package.json" - } -} diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts deleted file mode 100644 index ef4764ce8d73..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { defineConfig } from '@playwright/test'; - -const port = 3030; - -export default defineConfig({ - testDir: './tests', - /* - * Spans take ~2min to become queryable via the trace endpoint, and each poll has its own 180s - * budget. The first test polls twice in a row (the model span, then its parent), so the ceiling - * has to hold two polls back to back. - */ - timeout: 400_000, - fullyParallel: true, - forbidOnly: !!process.env.CI, - retries: 0, - // Every test spends most of its time polling Sentry, so run them all at once. - workers: '100%', - reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list', - use: { - baseURL: `http://localhost:${port}`, - }, - webServer: { - command: 'pnpm start', - port, - stdout: 'pipe', - stderr: 'pipe', - }, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts deleted file mode 100644 index f37d3198784c..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts +++ /dev/null @@ -1,85 +0,0 @@ -// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here. -import * as Sentry from '@sentry/node'; -import express from 'express'; -import OpenAI from 'openai'; - -// OpenRouter has no `/decisions` route, so this app calls OpenAI directly. CI only passes this key to -// this app. -const apiKey = process.env.E2E_OPENAI_DECISIONS_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENAI_DECISIONS_API_KEY is not set'); -} - -const client = new OpenAI({ apiKey }); - -const MODEL = 'gpt-6-luna'; - -// A 1x1 red PNG, so the image request is cheap. -const RED_PIXEL = - 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; - -const app = express(); - -// Each route makes one `decisions.create` call and answers with the trace id, so the test can find -// the request's spans in Sentry. - -app.get('/decide', async (_req, res, next) => { - try { - const decision = await client.decisions.create({ - model: MODEL, - input: 'I was charged twice this month and want a refund.', - questions: [ - { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, - { - type: 'choice', - name: 'department', - instructions: 'Which team should handle this?', - choices: [ - { value: 'billing', description: 'Charges and refunds' }, - { value: 'other', description: 'Anything else' }, - ], - }, - { - type: 'score', - name: 'urgency', - instructions: 'How urgent is this ticket?', - levels: [{ label: 'low' }, { label: 'high' }], - }, - ], - }); - res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); - } catch (error) { - next(error); - } -}); - -app.get('/decide-image', async (_req, res, next) => { - try { - const decision = await client.decisions.create({ - model: MODEL, - input: [ - { - role: 'user', - content: [ - { type: 'input_text', text: 'Is this image mostly red?' }, - { type: 'input_image', image_url: RED_PIXEL }, - ], - }, - ], - questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }], - }); - res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers }); - } catch (error) { - next(error); - } -}); - -app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { - res.status(500).send({ message: error.message }); -}); - -const port = Number(process.env.PORT ?? 3030); -app.listen(port, () => { - // eslint-disable-next-line no-console - console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts deleted file mode 100644 index c6c15df960cb..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; - -// The setup from the docs, nothing more: preloaded with `node --import`, no tunnel, so the spans go to -// the real Sentry project behind `E2E_TEST_DSN`, where the tests read them back. The OpenAI integration -// is on by default; its runtime channel injection is what picks up the `openai` client in app.ts. -Sentry.init({ - dsn: process.env.E2E_TEST_DSN, - environment: 'qa', // dynamic sampling bias to keep transactions - tracesSampleRate: 1, -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts deleted file mode 100644 index 6f1f1155f54d..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_PROVIDER_NAME, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, -} from '@sentry/conventions/attributes'; -import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; -import { - EVENT_POLLING_OPTIONS, - fetchSpanAttributes, - fetchTrace, - findSpanInTrace, - flattenTrace, - traceTarget, -} from '@sentry-internal/test-utils/cli'; - -// Mirrors src/app.ts. -const MODEL = 'gpt-6-luna'; - -/** - * What every decide span carries once Sentry has stored it, whatever the model answers. The names are the - * stored ones: `span.op`, `span.status` and `origin` rather than the `sentry.*` attributes the SDK sends. - * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. - */ -const DECIDE_SPAN_ATTRIBUTES = { - 'span.name': `decide ${MODEL}`, - 'span.op': GEN_AI_DECIDE, - 'span.status': 'ok', - origin: 'auto.ai.openai', - [GEN_AI_OPERATION_NAME]: 'decide', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: MODEL, - [GEN_AI_RESPONSE_MODEL]: MODEL, -}; - -test('records a text decision with its state, questions and answers', async ({ baseURL }) => { - const response = await fetch(`${baseURL}/decide`); - expect(response.status).toBe(200); - const { traceId }: { traceId: string } = await response.json(); - - console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); - - let spanId: string | undefined; - await expect - .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) - .toBeDefined(); - - let attributes: Record | undefined; - await expect - .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) - .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - - // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. - expect(Number(attributes![GEN_AI_USAGE_OUTPUT_TOKENS])).toBe(0); - expect(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])).toBeGreaterThan(0); - expect(Number(attributes![GEN_AI_USAGE_TOTAL_TOKENS])).toBe(Number(attributes![GEN_AI_USAGE_INPUT_TOKENS])); - - // The request span is the segment; it ends last, so it can land after the decide span. - await expect - .poll(() => { - const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); - return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); - }, EVENT_POLLING_OPTIONS) - .toBe(true); - - expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - state: 'I was charged twice this month and want a refund.', - questions: { - wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, - department: { - type: 'choice', - instructions: 'Which team should handle this?', - criteria: { billing: 'Charges and refunds', other: 'Anything else' }, - }, - urgency: { - type: 'score', - instructions: 'How urgent is this ticket?', - criteria: ['low', 'high'], - }, - }, - }, - ]); - - // The probabilities depend on the model, the shape does not. - expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - answers: { - wants_refund: { type: 'predicate', probability: expect.any(Number) }, - department: { - type: 'choice', - choice: 'billing', - probabilities: { billing: expect.any(Number), other: expect.any(Number) }, - confidence: expect.any(Number), - }, - urgency: { - type: 'score', - score: expect.any(Number), - probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, - legend: { 0: 'low', 1: 'high' }, - confidence: expect.any(Number), - }, - }, - }, - ]); -}); - -test('records an image decision by media type only', async ({ baseURL }) => { - const response = await fetch(`${baseURL}/decide-image`); - expect(response.status).toBe(200); - const { traceId }: { traceId: string } = await response.json(); - - console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); - - let spanId: string | undefined; - await expect - .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) - .toBeDefined(); - - let attributes: Record | undefined; - await expect - .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) - .toMatchObject(DECIDE_SPAN_ATTRIBUTES); - - expect(JSON.parse(attributes![GEN_AI_INPUT_MESSAGES] as string)).toEqual([ - { - type: 'evaluation', - state: [ - { - role: 'user', - content: [ - { type: 'input_text', text: 'Is this image mostly red?' }, - { type: 'input_image', mime_type: 'image/png' }, - ], - }, - ], - questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, - }, - ]); - - expect(JSON.parse(attributes![GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ - { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, - ]); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json deleted file mode 100644 index 795574fd9df2..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "outDir": "dist", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "types": ["node"] - }, - "include": ["src/**/*.ts"] -} From 511265334bddfb3bd6b04fdb93a6e69b8943dd0d Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 09:45:46 +0200 Subject: [PATCH 18/24] Cover cache tokens in the decisions integration test --- .../suites/tracing/openai/v7/scenario-decisions.mjs | 2 +- .../node-integration-tests/suites/tracing/openai/v7/test.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs index 70d5dbbd9037..beccdcd3d11f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -42,7 +42,7 @@ function startMockServer() { ], usage: { input_tokens: 382, - input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + input_tokens_details: { cached_tokens: 256, cache_write_tokens: 64 }, output_tokens: 0, output_tokens_details: { reasoning_tokens: 0 }, total_tokens: 382, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 317be507ccc5..eb19f8c0be46 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -10,6 +10,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, @@ -190,8 +191,10 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { [GEN_AI_PROVIDER_NAME]: { value: 'openai' }, [GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' }, [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' }, + // OpenAI's input tokens already include the cached ones, so they are not added again. [GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 }, - [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 0 }, + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 }, + [GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: { value: 64 }, [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 }, [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 }, [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 }, From 72df2c5942eb31b0db2fdfaf673f23c376644ca1 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 09:48:37 +0200 Subject: [PATCH 19/24] Keep decision choices without a description in the criteria --- .../suites/tracing/openai/v7/test.ts | 4 ++-- packages/server-utils/src/ai/openai/mapping.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index eb19f8c0be46..358fb15698ab 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -208,8 +208,8 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { department: { type: 'choice', instructions: 'Which team should handle this?', - // `other` has no description, so it is left out of the criteria. - criteria: { billing: 'Charges and refunds' }, + // `other` has no description, so it gets an empty one. + criteria: { billing: 'Charges and refunds', other: '' }, }, urgency: { type: 'score', diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts index b965738993f2..af5b563d1462 100644 --- a/packages/server-utils/src/ai/openai/mapping.ts +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -21,7 +21,7 @@ export function getDecisionOutputMessages(answers: unknown[]): string | undefine /** * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as - * `{ [value]: description }` (undescribed choices are left out), and score levels as their labels, + * `{ [value]: description }` (an empty description when there is none), and score levels as their labels, * indexed by score value. */ function toJevQuestionFormat(question: unknown): unknown { @@ -31,12 +31,12 @@ function toJevQuestionFormat(question: unknown): unknown { if (question.type === 'choice' && Array.isArray(question.choices)) { const { choices, ...rest } = question; - const described = choices.filter( - (choice): choice is Record => isObjectLike(choice) && typeof choice.description === 'string', + const criteria = Object.fromEntries( + choices + .filter(isObjectLike) + .map(choice => [String(choice.value), typeof choice.description === 'string' ? choice.description : '']), ); - return described.length - ? { ...rest, criteria: Object.fromEntries(described.map(choice => [String(choice.value), choice.description])) } - : rest; + return { ...rest, criteria }; } if (question.type === 'score' && Array.isArray(question.levels)) { From 13e90d7bd29995fec468be3be843a910940e372a Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 10:24:51 +0200 Subject: [PATCH 20/24] Set the decision input and tool definitions without a guard --- packages/server-utils/src/ai/openai/index.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index dde6801dc5a1..a832688c8cb3 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -74,16 +74,14 @@ export function extractRequestAttributes( if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) { const params = args[0] as Record; - const availableTools = recordInputs ? extractAvailableTools(params) : undefined; - if (availableTools) { - attributes[GEN_AI_TOOL_DEFINITIONS] = availableTools; + if (recordInputs) { + attributes[GEN_AI_TOOL_DEFINITIONS] = extractAvailableTools(params); } Object.assign(attributes, extractRequestParameters(params)); - const decisionInput = recordInputs && operationName === 'decide' ? getDecisionInputMessages(params) : undefined; - if (decisionInput) { - attributes[GEN_AI_INPUT_MESSAGES] = decisionInput; + if (recordInputs && operationName === 'decide') { + attributes[GEN_AI_INPUT_MESSAGES] = getDecisionInputMessages(params); } } else { attributes[GEN_AI_REQUEST_MODEL] = 'unknown'; From c316841c967a957a23ef5f9451483c72eaf56fef Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 10:32:39 +0200 Subject: [PATCH 21/24] Record the decisions input images as they are --- .../suites/tracing/openai/v7/test.ts | 4 +-- .../server-utils/src/ai/openai/mapping.ts | 26 ++----------------- .../test/ai/lib/tracing/openai.test.ts | 24 +---------------- 3 files changed, 4 insertions(+), 50 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index 358fb15698ab..fa1e28db9e86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -241,14 +241,12 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { }, ]); - // Images are base64 data URLs, so only their media type is recorded. - expect(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value).not.toContain('base64'); expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([ { role: 'user', content: [ { type: 'input_text', text: 'Is the package damaged?' }, - { type: 'input_image', mime_type: 'image/png' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' }, ], }, ]); diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts index af5b563d1462..ae52d9f4fcaa 100644 --- a/packages/server-utils/src/ai/openai/mapping.ts +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -7,11 +7,11 @@ import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../core /** * Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state` - * so both render the same. Images are base64 data URLs, so they are reported by media type only. + * so both render the same. */ export function getDecisionInputMessages(params: Record): string | undefined { const questions = Array.isArray(params.questions) ? params.questions.map(toJevQuestionFormat) : params.questions; - return getEvaluationInputMessages({ state: withoutImageData(params.input), questions: keyByName(questions) }); + return getEvaluationInputMessages({ state: params.input, questions: keyByName(questions) }); } /** Serialize the answers of a decisions response, laid out and keyed like Jev answers. */ @@ -91,25 +91,3 @@ function keyByName(items: unknown): unknown { }), ); } - -function withoutImageData(input: unknown): unknown { - if (!Array.isArray(input)) { - return input; - } - - // Parts are also handled at the top level, in case a flat list of parts is passed instead of messages. - return input.map(item => - isObjectLike(item) && Array.isArray(item.content) - ? { ...item, content: item.content.map(withoutImageDataPart) } - : withoutImageDataPart(item), - ); -} - -function withoutImageDataPart(part: unknown): unknown { - if (!isObjectLike(part) || part.type !== 'input_image' || typeof part.image_url !== 'string') { - return part; - } - - const mimeType = /^data:([^;,]+)/.exec(part.image_url)?.[1]; - return mimeType ? { type: 'input_image', mime_type: mimeType } : { type: 'input_image' }; -} diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index 66d5fde0eb14..fa07c337dca4 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; @@ -118,26 +118,4 @@ describe('instrumentOpenAiClient span names', () => { { type: 'evaluation', answers: { wants_refund: { type: 'predicate', probability: 0.97 } } }, ]); }); - - it('records only the media type of a decisions image passed as a top-level part', async () => { - const endedSpans = setupClient('stream'); - const instrumented = instrumentOpenAiClient({ - decisions: { create: vi.fn().mockResolvedValue({ model: 'gpt-6-luna', answers: [] }) }, - }); - - await instrumented.decisions.create({ - model: 'gpt-6-luna', - input: [ - { type: 'input_text', text: 'Is this red?' }, - { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' }, - ], - questions: [], - }); - - const [input] = JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_INPUT_MESSAGES] as string); - expect(input.state).toEqual([ - { type: 'input_text', text: 'Is this red?' }, - { type: 'input_image', mime_type: 'image/png' }, - ]); - }); }); From 8cf3f78a1e88f27117e510370b2e384199d13495 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 11:37:46 +0200 Subject: [PATCH 22/24] Record decision score level descriptions --- .../suites/tracing/openai/v7/scenario-decisions.mjs | 2 +- .../suites/tracing/openai/v7/test.ts | 2 ++ packages/server-utils/src/ai/openai/mapping.ts | 9 +++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs index beccdcd3d11f..0934cebcd554 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -69,7 +69,7 @@ const questions = [ type: 'score', name: 'urgency', instructions: 'How urgent is this ticket?', - levels: [{ label: 'low' }, { label: 'high' }], + levels: [{ label: 'low' }, { label: 'high', description: 'Blocks the customer' }], }, ]; diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index fa1e28db9e86..ecbe7f3ce1c6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -215,6 +215,8 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'], + // `low` has no description, so it gets an empty one. + descriptions: ['', 'Blocks the customer'], }, }, }, diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts index ae52d9f4fcaa..9cb1cc305225 100644 --- a/packages/server-utils/src/ai/openai/mapping.ts +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -22,7 +22,7 @@ export function getDecisionOutputMessages(answers: unknown[]): string | undefine /** * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as * `{ [value]: description }` (an empty description when there is none), and score levels as their labels, - * indexed by score value. + * indexed by score value, with any level descriptions in `descriptions` at the same index. */ function toJevQuestionFormat(question: unknown): unknown { if (!isObjectLike(question)) { @@ -41,7 +41,12 @@ function toJevQuestionFormat(question: unknown): unknown { if (question.type === 'score' && Array.isArray(question.levels)) { const { levels, ...rest } = question; - return { ...rest, criteria: levels.map(level => (isObjectLike(level) ? level.label : level)) }; + const criteria = levels.map(level => (isObjectLike(level) ? level.label : level)); + // Sentry renders score criteria as a list of labels only, so level descriptions get their own list. + const descriptions = levels.map(level => + isObjectLike(level) && typeof level.description === 'string' ? level.description : '', + ); + return descriptions.some(Boolean) ? { ...rest, criteria, descriptions } : { ...rest, criteria }; } return question; From 4ee73958c2c338983e8f595528a0b06e432a9319 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 12:33:18 +0200 Subject: [PATCH 23/24] Key decision score level descriptions by score value --- .../suites/tracing/openai/v7/test.ts | 4 ++-- packages/server-utils/src/ai/openai/mapping.ts | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index ecbe7f3ce1c6..cea32de437ab 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -215,8 +215,8 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'], - // `low` has no description, so it gets an empty one. - descriptions: ['', 'Blocks the customer'], + // Keyed by score value; `low` has no description, so it is left out. + descriptions: { 1: 'Blocks the customer' }, }, }, }, diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts index 9cb1cc305225..22099af88c13 100644 --- a/packages/server-utils/src/ai/openai/mapping.ts +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -22,7 +22,7 @@ export function getDecisionOutputMessages(answers: unknown[]): string | undefine /** * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as * `{ [value]: description }` (an empty description when there is none), and score levels as their labels, - * indexed by score value, with any level descriptions in `descriptions` at the same index. + * indexed by score value, with any level descriptions in `descriptions`, keyed by score value. */ function toJevQuestionFormat(question: unknown): unknown { if (!isObjectLike(question)) { @@ -42,11 +42,14 @@ function toJevQuestionFormat(question: unknown): unknown { if (question.type === 'score' && Array.isArray(question.levels)) { const { levels, ...rest } = question; const criteria = levels.map(level => (isObjectLike(level) ? level.label : level)); - // Sentry renders score criteria as a list of labels only, so level descriptions get their own list. - const descriptions = levels.map(level => - isObjectLike(level) && typeof level.description === 'string' ? level.description : '', + // Sentry renders score criteria as a list of labels only, so level descriptions go in their own field, + // keyed by score value like an answer's `legend`. + const descriptions = Object.fromEntries( + levels.flatMap((level, index) => + isObjectLike(level) && typeof level.description === 'string' ? [[String(index), level.description]] : [], + ), ); - return descriptions.some(Boolean) ? { ...rest, criteria, descriptions } : { ...rest, criteria }; + return Object.keys(descriptions).length ? { ...rest, criteria, descriptions } : { ...rest, criteria }; } return question; From b4cc2a9ba06b95bbf5956b7ffa8e36ef511eef87 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 14 Oct 2026 15:33:15 +0200 Subject: [PATCH 24/24] Record decision choice answers as strings --- .../server-utils/src/ai/openai/mapping.ts | 3 +- .../test/ai/lib/tracing/openai.test.ts | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts index 22099af88c13..e43237570d6c 100644 --- a/packages/server-utils/src/ai/openai/mapping.ts +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -71,7 +71,8 @@ function toJevAnswerFormat(answer: unknown): unknown { const options = answer.probabilities.filter(isObjectLike); const probabilities = Object.fromEntries(options.map(option => [String(option.value), option.probability])); if (answer.type === 'choice') { - return { ...answer, probabilities }; + // Choice values can be booleans, but Sentry renders a choice as a string, like its `probabilities` keys. + return { ...answer, choice: String(answer.choice), probabilities }; } const legend = Object.fromEntries( diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index fa07c337dca4..523d040e8a4c 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -118,4 +118,42 @@ describe('instrumentOpenAiClient span names', () => { { type: 'evaluation', answers: { wants_refund: { type: 'predicate', probability: 0.97 } } }, ]); }); + + it('records a boolean decision choice as a string', async () => { + const endedSpans = setupClient('stream'); + const instrumented = instrumentOpenAiClient({ + decisions: { + create: vi.fn().mockResolvedValue({ + model: 'gpt-6-luna', + answers: [ + { + type: 'choice', + name: 'is_spam', + choice: true, + probabilities: [ + { value: true, probability: 0.91 }, + { value: false, probability: 0.09 }, + ], + confidence: 0.91, + }, + ], + }), + }, + }); + + await instrumented.decisions.create({ + model: 'gpt-6-luna', + input: 'Win a free cruise!', + questions: [{ type: 'choice', name: 'is_spam', choices: [{ value: true }, { value: false }] }], + }); + + expect(JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: { + is_spam: { type: 'choice', choice: 'true', probabilities: { true: 0.91, false: 0.09 }, confidence: 0.91 }, + }, + }, + ]); + }); });