diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-with-recording.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs new file mode 100644 index 000000000000..0934cebcd554 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/scenario-decisions.mjs @@ -0,0 +1,115 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json({ limit: '1mb' })); + + app.post('/openai/decisions', (req, res) => { + const { model } = req.body; + + if (model === 'error-model') { + res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error'); + return; + } + + // Shaped like a real `gpt-6-luna` response. Decisions bills input tokens only. + res.send({ + model: 'gpt-6-luna', + answers: [ + { type: 'predicate', name: 'wants_refund', probability: 0.97 }, + { + type: 'choice', + name: 'department', + choice: 'billing', + probabilities: [ + { value: 'billing', probability: 0.82 }, + { value: 'other', probability: 0.18 }, + ], + confidence: 0.82, + }, + { + type: 'score', + name: 'urgency', + score: 0.95, + probabilities: [ + { value: 0, label: 'low', probability: 0.05 }, + { value: 1, label: 'high', probability: 0.95 }, + ], + confidence: 0.93, + }, + ], + usage: { + input_tokens: 382, + input_tokens_details: { cached_tokens: 256, cache_write_tokens: 64 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 382, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +const questions = [ + { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }, + { + type: 'choice', + name: 'department', + instructions: 'Which team should handle this?', + choices: [{ value: 'billing', description: 'Charges and refunds' }, { value: 'other' }], + }, + { + type: 'score', + name: 'urgency', + instructions: 'How urgent is this ticket?', + levels: [{ label: 'low' }, { label: 'high', description: 'Blocks the customer' }], + }, +]; + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + + await client.decisions.create({ + model: 'gpt-6-luna', + input: 'I was charged twice and want a refund.', + questions, + }); + + await client.decisions.create({ + model: 'gpt-6-luna', + input: [ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is the package damaged?' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' }, + ], + }, + ], + questions: [{ type: 'predicate', name: 'damaged', instructions: 'Is there visible damage?' }], + }); + + try { + await client.decisions.create({ model: 'error-model', input: 'Error test', questions }); + } catch { + // Error is expected and handled + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index d24c08130d84..cea32de437ab 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -1,19 +1,25 @@ import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { supports } from '../../../../utils/index'; @@ -158,4 +164,103 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { }, }, ); + + // The decisions API (`openai` >= 7.30.0) returns typed answers, so it is recorded like a Jev call. + createEsmAndCjsTests( + __dirname, + 'scenario-decisions.mjs', + 'instrument-with-recording.mjs', + (createRunner, test) => { + test('instruments the decisions API as decide spans', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + expect(spans).toHaveLength(3); + + // Both successful spans are named `decide gpt-6-luna` and may arrive in any order, so match on the input. + const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value); + const textSpan = spans.find(span => inputOf(span).includes('charged twice')); + const imageSpan = spans.find(span => inputOf(span).includes('input_image')); + expect(textSpan!.name).toBe('decide gpt-6-luna'); + expect(textSpan!.attributes).toMatchObject({ + [SENTRY_ORIGIN]: { value: 'auto.ai.openai' }, + [GEN_AI_OPERATION_NAME]: { value: 'decide' }, + [GEN_AI_PROVIDER_NAME]: { value: 'openai' }, + [GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' }, + [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' }, + // OpenAI's input tokens already include the cached ones, so they are not added again. + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 }, + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 }, + [GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: { value: 64 }, + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 }, + [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 }, + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 }, + }); + expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice and want a refund.', + questions: { + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + // `other` has no description, so it gets an empty one. + criteria: { billing: 'Charges and refunds', other: '' }, + }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'high'], + // Keyed by score value; `low` has no description, so it is left out. + descriptions: { 1: 'Blocks the customer' }, + }, + }, + }, + ]); + expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ + { + type: 'evaluation', + answers: { + wants_refund: { type: 'predicate', probability: 0.97 }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.82, other: 0.18 }, + confidence: 0.82, + }, + urgency: { + type: 'score', + score: 0.95, + probabilities: { 0: 0.05, 1: 0.95 }, + legend: { 0: 'low', 1: 'high' }, + confidence: 0.93, + }, + }, + }, + ]); + + expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([ + { + role: 'user', + content: [ + { type: 'input_text', text: 'Is the package damaged?' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' }, + ], + }, + ]); + + const errorSpan = spans.find(span => span.name === 'decide error-model'); + expect(errorSpan!.status).not.toBe('ok'); + }); + }, + { + additionalDependencies: { + openai: '7.30.0', + }, + }, + ); }); diff --git a/packages/server-utils/src/ai/openai/constants.ts b/packages/server-utils/src/ai/openai/constants.ts index af163af81b6a..60bf73f94bf1 100644 --- a/packages/server-utils/src/ai/openai/constants.ts +++ b/packages/server-utils/src/ai/openai/constants.ts @@ -12,6 +12,8 @@ export const OPENAI_METHOD_REGISTRY = { // Conversations API - for conversation state management // https://platform.openai.com/docs/guides/conversation-state 'conversations.create': { operation: 'chat' }, + // https://developers.openai.com/api/docs/guides/decisions + 'decisions.create': { operation: 'decide' }, } as const satisfies InstrumentedMethodRegistry; export const RESPONSES_TOOL_CALL_EVENT_TYPES = [ 'response.output_item.added', diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 981da4281ef5..a832688c8cb3 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -31,6 +31,7 @@ import { import { OPENAI_METHOD_REGISTRY } from './constants'; import { instrumentStream } from './streaming'; import type { ChatCompletionChunk, OpenAiOptions, OpenAIStream, ResponseStreamingEvent } from './types'; +import { getDecisionInputMessages } from './mapping'; import { addResponseAttributes, extractRequestParameters } from './utils'; /** @@ -73,12 +74,15 @@ export function extractRequestAttributes( if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) { const params = args[0] as Record; - const availableTools = recordInputs ? extractAvailableTools(params) : undefined; - if (availableTools) { - attributes[GEN_AI_TOOL_DEFINITIONS] = availableTools; + if (recordInputs) { + attributes[GEN_AI_TOOL_DEFINITIONS] = extractAvailableTools(params); } Object.assign(attributes, extractRequestParameters(params)); + + if (recordInputs && operationName === 'decide') { + attributes[GEN_AI_INPUT_MESSAGES] = getDecisionInputMessages(params); + } } else { attributes[GEN_AI_REQUEST_MODEL] = 'unknown'; } @@ -88,6 +92,11 @@ export function extractRequestAttributes( // Extract and record AI request inputs, if present. This is intentionally separate from response attributes. export function addRequestAttributes(span: Span, params: Record, operationName: string): void { + // A decision's input is recorded when its span starts, in `extractRequestAttributes`. + if (operationName === 'decide') { + return; + } + // Store embeddings input on a separate attribute if (operationName === 'embeddings' && 'input' in params) { const input = params.input; @@ -203,7 +212,7 @@ function instrumentMethod( } return originalResult.then(result => { - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, options.recordOutputs, operationName); return result; }); }); diff --git a/packages/server-utils/src/ai/openai/mapping.ts b/packages/server-utils/src/ai/openai/mapping.ts new file mode 100644 index 000000000000..e43237570d6c --- /dev/null +++ b/packages/server-utils/src/ai/openai/mapping.ts @@ -0,0 +1,102 @@ +/** + * Maps OpenAI decisions requests and responses to the format Jev calls are recorded in, so Sentry renders + * both the same way. + */ +import { isObjectLike } from '@sentry/core'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../core/utils'; + +/** + * Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state` + * so both render the same. + */ +export function getDecisionInputMessages(params: Record): string | undefined { + const questions = Array.isArray(params.questions) ? params.questions.map(toJevQuestionFormat) : params.questions; + return getEvaluationInputMessages({ state: params.input, questions: keyByName(questions) }); +} + +/** Serialize the answers of a decisions response, laid out and keyed like Jev answers. */ +export function getDecisionOutputMessages(answers: unknown[]): string | undefined { + return getEvaluationOutputMessages(keyByName(answers.map(toJevAnswerFormat))); +} + +/** + * Lay out a decisions question like a Jev question, with its options in `criteria`: choices as + * `{ [value]: description }` (an empty description when there is none), and score levels as their labels, + * indexed by score value, with any level descriptions in `descriptions`, keyed by score value. + */ +function toJevQuestionFormat(question: unknown): unknown { + if (!isObjectLike(question)) { + return question; + } + + if (question.type === 'choice' && Array.isArray(question.choices)) { + const { choices, ...rest } = question; + const criteria = Object.fromEntries( + choices + .filter(isObjectLike) + .map(choice => [String(choice.value), typeof choice.description === 'string' ? choice.description : '']), + ); + return { ...rest, criteria }; + } + + if (question.type === 'score' && Array.isArray(question.levels)) { + const { levels, ...rest } = question; + const criteria = levels.map(level => (isObjectLike(level) ? level.label : level)); + // Sentry renders score criteria as a list of labels only, so level descriptions go in their own field, + // keyed by score value like an answer's `legend`. + const descriptions = Object.fromEntries( + levels.flatMap((level, index) => + isObjectLike(level) && typeof level.description === 'string' ? [[String(index), level.description]] : [], + ), + ); + return Object.keys(descriptions).length ? { ...rest, criteria, descriptions } : { ...rest, criteria }; + } + + return question; +} + +/** + * Lay out choice and score answers like Jev answers: probabilities keyed by option or score value, with + * score labels in a `legend`. Predicate answers are already in the shape Sentry renders. + */ +function toJevAnswerFormat(answer: unknown): unknown { + if ( + !isObjectLike(answer) || + (answer.type !== 'choice' && answer.type !== 'score') || + !Array.isArray(answer.probabilities) + ) { + return answer; + } + + const options = answer.probabilities.filter(isObjectLike); + const probabilities = Object.fromEntries(options.map(option => [String(option.value), option.probability])); + if (answer.type === 'choice') { + // Choice values can be booleans, but Sentry renders a choice as a string, like its `probabilities` keys. + return { ...answer, choice: String(answer.choice), probabilities }; + } + + const legend = Object.fromEntries( + options.filter(option => typeof option.label === 'string').map(option => [String(option.value), option.label]), + ); + return { ...answer, probabilities, legend }; +} + +/** + * The decisions API lists questions and answers as arrays with an optional `name`, where the conventions + * key them by question name. An unnamed item is keyed by its position. + */ +function keyByName(items: unknown): unknown { + if (!Array.isArray(items)) { + return items; + } + + return Object.fromEntries( + items.map((item, index) => { + if (!isObjectLike(item)) { + return [String(index), item]; + } + const { name, ...rest } = item; + return [typeof name === 'string' ? name : String(index), rest]; + }), + ); +} diff --git a/packages/server-utils/src/ai/openai/utils.ts b/packages/server-utils/src/ai/openai/utils.ts index 3f6afeac1c44..7c2190585e9a 100644 --- a/packages/server-utils/src/ai/openai/utils.ts +++ b/packages/server-utils/src/ai/openai/utils.ts @@ -3,6 +3,7 @@ import { isObjectLike } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_CONVERSATION_ID, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_REQUEST_FREQUENCY_PENALTY, GEN_AI_REQUEST_MODEL, GEN_AI_REQUEST_PRESENCE_PENALTY, @@ -17,6 +18,7 @@ import { GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { @@ -24,6 +26,7 @@ import { GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE, GEN_AI_REQUEST_STREAM_ATTRIBUTE, } from '../core/gen-ai-attributes'; +import { getDecisionOutputMessages } from './mapping'; import type { ChatCompletionChunk, ResponseStreamingEvent } from './types'; /** @@ -53,9 +56,15 @@ export function isChatCompletionChunk(event: unknown): event is ChatCompletionCh /** * Add response attributes to a span using duck-typing. - * Works for Chat Completions, Responses API, Embeddings, and Conversations API responses. + * Works for Chat Completions, Responses API, Embeddings, and Conversations API responses. Decision + * answers are only recorded when `operationName` is `decide`. */ -export function addResponseAttributes(span: Span, result: unknown, recordOutputs?: boolean): void { +export function addResponseAttributes( + span: Span, + result: unknown, + recordOutputs?: boolean, + operationName?: string, +): void { if (!result || typeof result !== 'object') return; const response = result as Record; @@ -76,32 +85,8 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs attrs[GEN_AI_CONVERSATION_ID] = response.id; } - // Token usage — supports both naming conventions (chat: prompt_tokens/completion_tokens, responses: input_tokens/output_tokens) - if (response.usage && typeof response.usage === 'object') { - const usage = response.usage as Record; - - const inputTokens = usage.prompt_tokens ?? usage.input_tokens; - if (typeof inputTokens === 'number') { - attrs[GEN_AI_USAGE_INPUT_TOKENS] = inputTokens; - } - - const outputTokens = usage.completion_tokens ?? usage.output_tokens; - if (typeof outputTokens === 'number') { - attrs[GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens; - } - - if (typeof usage.total_tokens === 'number') { - attrs[GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens; - } - const inputDetails = usage.prompt_tokens_details ?? usage.input_tokens_details; - if (isObjectLike(inputDetails)) { - if (typeof inputDetails.cached_tokens === 'number') { - attrs[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = inputDetails.cached_tokens; - } - if (typeof inputDetails.cache_write_tokens === 'number') { - attrs[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] = inputDetails.cache_write_tokens; - } - } + if (isObjectLike(response.usage)) { + addUsageAttributes(attrs, response.usage); } // Finish reasons from choices (chat completions) @@ -151,6 +136,14 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs attrs[GEN_AI_RESPONSE_TEXT] = response.output_text; } + // Typed answers (decisions API) + if (operationName === 'decide' && Array.isArray(response.answers)) { + const outputMessages = getDecisionOutputMessages(response.answers); + if (outputMessages) { + attrs[GEN_AI_OUTPUT_MESSAGES] = outputMessages; + } + } + // Tool calls from output array (responses API) if (Array.isArray(response.output) && response.output.length > 0 && !attrs[GEN_AI_RESPONSE_TOOL_CALLS]) { const functionCalls = (response.output as Array>).filter( @@ -165,6 +158,39 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs span.setAttributes(attrs); } +/** + * Token usage, in both naming conventions (chat: prompt_tokens/completion_tokens, + * responses and decisions: input_tokens/output_tokens). + */ +function addUsageAttributes(attrs: Record, usage: Record): void { + const inputTokens = usage.prompt_tokens ?? usage.input_tokens; + if (typeof inputTokens === 'number') { + attrs[GEN_AI_USAGE_INPUT_TOKENS] = inputTokens; + } + + const outputTokens = usage.completion_tokens ?? usage.output_tokens; + if (typeof outputTokens === 'number') { + attrs[GEN_AI_USAGE_OUTPUT_TOKENS] = outputTokens; + } + + if (typeof usage.total_tokens === 'number') { + attrs[GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens; + } + const inputDetails = usage.prompt_tokens_details ?? usage.input_tokens_details; + if (isObjectLike(inputDetails)) { + if (typeof inputDetails.cached_tokens === 'number') { + attrs[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] = inputDetails.cached_tokens; + } + if (typeof inputDetails.cache_write_tokens === 'number') { + attrs[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] = inputDetails.cache_write_tokens; + } + } + const outputDetails = usage.completion_tokens_details ?? usage.output_tokens_details; + if (isObjectLike(outputDetails) && typeof outputDetails.reasoning_tokens === 'number') { + attrs[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] = outputDetails.reasoning_tokens; + } +} + /** * Extract the Conversations API id (`conversation: "conv_..."`) from request parameters. * `previous_response_id` is deliberately not read: it names the previous response, so it changes on diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index ec7c852e9010..eaa02e867a5d 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -30,6 +30,7 @@ const ORIGIN = 'auto.ai.openai'; const INSTRUMENTED_CHANNELS = [ { channel: CHANNELS.OPENAI_CHAT, operation: 'chat' }, { channel: CHANNELS.OPENAI_EMBEDDINGS, operation: 'embeddings' }, + { channel: CHANNELS.OPENAI_DECIDE, operation: 'decide' }, ] as const; const _openAIIntegration = ((options: OpenAiOptions = {}) => { @@ -48,7 +49,7 @@ function instrumentOpenai(options: OpenAiOptions): void { data => createGenAiSpan(data, operation, options), { beforeSpanEnd: (span, data) => { - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs, operation); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( @@ -134,6 +135,6 @@ function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: /** * Diagnostics-channel-based OpenAI integration. Subscribes to the `orchestrion:openai:*` * diagnostics_channels injected into `openai`'s `create` methods (chat completions, responses, embeddings, - * conversations), so it requires the Sentry runtime hook or bundler plugin. + * conversations, decisions), so it requires the Sentry runtime hook or bundler plugin. */ export const openAIIntegration = defineIntegration(_openAIIntegration); diff --git a/packages/server-utils/src/orchestrion/config/openai.ts b/packages/server-utils/src/orchestrion/config/openai.ts index e68d3a961262..8024cb4b0b67 100644 --- a/packages/server-utils/src/orchestrion/config/openai.ts +++ b/packages/server-utils/src/orchestrion/config/openai.ts @@ -27,6 +27,12 @@ export const openaiConfig = [ module: { name: 'openai', versionRange: '>=4.0.0 <8', filePath }, functionQuery: { className: 'Conversations', methodName: 'create', kind: 'Sync' as const }, })), + // OpenAI decisions API (`openai` >= 7.30.0), which returns typed answers rather than generated text. + ...['resources/decisions.js', 'resources/decisions.mjs'].map(filePath => ({ + channelName: 'decide', + module: { name: 'openai', versionRange: '>=7.30.0 <8', filePath }, + functionQuery: { className: 'Decisions', methodName: 'create', kind: 'Sync' as const }, + })), ] satisfies InstrumentationConfig[]; export const openaiModuleNames = getModuleNames(openaiConfig); @@ -36,4 +42,5 @@ export const openaiChannels = { // identical span handling, so they share one channel. OPENAI_CHAT: 'orchestrion:openai:chat', OPENAI_EMBEDDINGS: 'orchestrion:openai:embeddings', + OPENAI_DECIDE: 'orchestrion:openai:decide', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index 0c6738ce86ac..523d040e8a4c 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,4 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentOpenAiClient } from '../../../../src/ai/openai'; @@ -90,4 +92,68 @@ describe('instrumentOpenAiClient span names', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).description).toBe('chat'); }); + + it('records `decisions.create` as a decide span', async () => { + const endedSpans = setupClient('stream'); + const instrumented = instrumentOpenAiClient({ + decisions: { + create: vi.fn().mockResolvedValue({ + model: 'gpt-6-luna', + answers: [{ type: 'predicate', name: 'wants_refund', probability: 0.97 }], + }), + }, + }); + + await instrumented.decisions.create({ + model: 'gpt-6-luna', + input: 'I want a refund', + questions: [{ type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' }], + }); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(json.description).toBe('decide gpt-6-luna'); + expect(json.op).toBe(GEN_AI_DECIDE); + expect(JSON.parse(json.data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + // Keyed by question name, as the conventions define it. + { type: 'evaluation', answers: { wants_refund: { type: 'predicate', probability: 0.97 } } }, + ]); + }); + + it('records a boolean decision choice as a string', async () => { + const endedSpans = setupClient('stream'); + const instrumented = instrumentOpenAiClient({ + decisions: { + create: vi.fn().mockResolvedValue({ + model: 'gpt-6-luna', + answers: [ + { + type: 'choice', + name: 'is_spam', + choice: true, + probabilities: [ + { value: true, probability: 0.91 }, + { value: false, probability: 0.09 }, + ], + confidence: 0.91, + }, + ], + }), + }, + }); + + await instrumented.decisions.create({ + model: 'gpt-6-luna', + input: 'Win a free cruise!', + questions: [{ type: 'choice', name: 'is_spam', choices: [{ value: true }, { value: false }] }], + }); + + expect(JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { + type: 'evaluation', + answers: { + is_spam: { type: 'choice', choice: 'true', probabilities: { true: 0.91, false: 0.09 }, confidence: 0.91 }, + }, + }, + ]); + }); });