Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
4ca936f
feat(server-utils): Instrument the OpenAI decisions API as gen_ai.dec…
andreiborza Oct 12, 2026
24f4a8b
Match the decisions mock to a real gpt-6-luna response
andreiborza Oct 12, 2026
81b6bc0
Add node-openai-decisions-send-to-sentry e2e test app
andreiborza Oct 12, 2026
99155fb
Match decide spans by input instead of order in the decisions test
andreiborza Oct 12, 2026
bf0b8b8
Use the shared test-utils/cli helpers in the decisions e2e app
andreiborza Oct 12, 2026
19ed67b
Inline the trace id lookup in the decisions e2e app
andreiborza Oct 12, 2026
28c77c1
Assert the decisions e2e messages against the full recorded JSON
andreiborza Oct 12, 2026
ec536bc
Poll for the request span parent in the decisions e2e test
andreiborza Oct 12, 2026
12dae22
Assert the stored span name in the decisions e2e test
andreiborza Oct 13, 2026
4fefcd6
Lay out decision choice and score answers like Jev answers, keyed by …
andreiborza Oct 13, 2026
95ff42d
Lay out decision questions like Jev questions, with options in criteria
andreiborza Oct 13, 2026
0cf6661
Merge remote-tracking branch 'origin/develop' into ab/openai-decisions
andreiborza Oct 13, 2026
2c6bbaf
Drop image data from top-level decisions input parts
andreiborza Oct 13, 2026
50bba01
Record the decisions input when the span starts
andreiborza Oct 13, 2026
b61e351
Write the decisions e2e test steps inline, like the Cloudflare app
andreiborza Oct 13, 2026
d09c009
Record decision answers only for decide calls and move the mapping to…
andreiborza Oct 13, 2026
0ba719e
Limit the decisions hook to openai 7.30.0 and later
andreiborza Oct 13, 2026
836145b
Merge branch 'develop' into ab/openai-decisions
andreiborza Oct 13, 2026
10f4154
Merge branch 'develop' into ab/openai-decisions
andreiborza Oct 13, 2026
2eec966
Move the decisions e2e app to its own PR
andreiborza Oct 13, 2026
5112653
Cover cache tokens in the decisions integration test
andreiborza Oct 14, 2026
72df2c5
Keep decision choices without a description in the criteria
andreiborza Oct 14, 2026
13e90d7
Set the decision input and tool definitions without a guard
andreiborza Oct 14, 2026
c316841
Record the decisions input images as they are
andreiborza Oct 14, 2026
8cf3f78
Record decision score level descriptions
andreiborza Oct 14, 2026
4ee7395
Key decision score level descriptions by score value
andreiborza Oct 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import * as Sentry from '@sentry/node';
import express from 'express';
import OpenAI from 'openai';

function startMockServer() {
const app = express();
app.use(express.json({ limit: '1mb' }));

app.post('/openai/decisions', (req, res) => {
const { model } = req.body;

if (model === 'error-model') {
res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error');
return;
}

// Shaped like a real `gpt-6-luna` response. Decisions bills input tokens only.
res.send({
model: 'gpt-6-luna',
answers: [
{ type: 'predicate', name: 'wants_refund', probability: 0.97 },
{
type: 'choice',
name: 'department',
choice: 'billing',
probabilities: [
{ value: 'billing', probability: 0.82 },
{ value: 'other', probability: 0.18 },
],
confidence: 0.82,
},
{
type: 'score',
name: 'urgency',
score: 0.95,
probabilities: [
{ value: 0, label: 'low', probability: 0.05 },
{ value: 1, label: 'high', probability: 0.95 },
],
confidence: 0.93,
},
],
usage: {
input_tokens: 382,
input_tokens_details: { cached_tokens: 256, cache_write_tokens: 64 },
output_tokens: 0,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 382,
},
});
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

const questions = [
{ type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' },
{
type: 'choice',
name: 'department',
instructions: 'Which team should handle this?',
choices: [{ value: 'billing', description: 'Charges and refunds' }, { value: 'other' }],
},
{
type: 'score',
name: 'urgency',
instructions: 'How urgent is this ticket?',
levels: [{ label: 'low' }, { label: 'high', description: 'Blocks the customer' }],
},
];

async function run() {
const server = await startMockServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new OpenAI({
baseURL: `http://localhost:${server.address().port}/openai`,
apiKey: 'mock-api-key',
});

await client.decisions.create({
model: 'gpt-6-luna',
input: 'I was charged twice and want a refund.',
questions,
});

await client.decisions.create({
model: 'gpt-6-luna',
input: [
{
role: 'user',
content: [
{ type: 'input_text', text: 'Is the package damaged?' },
{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' },
],
},
],
questions: [{ type: 'predicate', name: 'damaged', instructions: 'Is there visible damage?' }],
});

try {
await client.decisions.create({ model: 'error-model', input: 'Error test', questions });
} catch {
// Error is expected and handled
}
});

server.close();
}

run();
105 changes: 105 additions & 0 deletions dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
import type { SerializedStreamedSpanContainer } from '@sentry/core';
import { afterAll, describe, expect } from 'vitest';
import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_MODEL,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
import { supports } from '../../../../utils/index';

Expand Down Expand Up @@ -158,4 +164,103 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => {
},
},
);

// The decisions API (`openai` >= 7.30.0) returns typed answers, so it is recorded like a Jev call.
createEsmAndCjsTests(
__dirname,
'scenario-decisions.mjs',
'instrument-with-recording.mjs',
(createRunner, test) => {
test('instruments the decisions API as decide spans', async () => {
const runner = createRunner().ignore('event');
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

await runner.start().completed();

const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE);
expect(spans).toHaveLength(3);

// Both successful spans are named `decide gpt-6-luna` and may arrive in any order, so match on the input.
const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value);
const textSpan = spans.find(span => inputOf(span).includes('charged twice'));
const imageSpan = spans.find(span => inputOf(span).includes('input_image'));
expect(textSpan!.name).toBe('decide gpt-6-luna');
expect(textSpan!.attributes).toMatchObject({
[SENTRY_ORIGIN]: { value: 'auto.ai.openai' },
[GEN_AI_OPERATION_NAME]: { value: 'decide' },
[GEN_AI_PROVIDER_NAME]: { value: 'openai' },
[GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' },
[GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' },
// OpenAI's input tokens already include the cached ones, so they are not added again.
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 },
[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 },
[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: { value: 64 },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 },
[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 },
});
expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
{
type: 'evaluation',
state: 'I was charged twice and want a refund.',
questions: {
wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' },
department: {
type: 'choice',
instructions: 'Which team should handle this?',
// `other` has no description, so it gets an empty one.
criteria: { billing: 'Charges and refunds', other: '' },
},
urgency: {
type: 'score',
instructions: 'How urgent is this ticket?',
criteria: ['low', 'high'],
// Keyed by score value; `low` has no description, so it is left out.
descriptions: { 1: 'Blocks the customer' },
},
},
},
]);
expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
{
type: 'evaluation',
answers: {
wants_refund: { type: 'predicate', probability: 0.97 },
department: {
type: 'choice',
choice: 'billing',
probabilities: { billing: 0.82, other: 0.18 },
confidence: 0.82,
},
urgency: {
type: 'score',
score: 0.95,
probabilities: { 0: 0.05, 1: 0.95 },
legend: { 0: 'low', 1: 'high' },
confidence: 0.93,
},
},
},
]);

expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([
{
role: 'user',
content: [
{ type: 'input_text', text: 'Is the package damaged?' },
{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' },
],
},
]);

const errorSpan = spans.find(span => span.name === 'decide error-model');
expect(errorSpan!.status).not.toBe('ok');
});
},
{
additionalDependencies: {
openai: '7.30.0',
},
},
);
});
2 changes: 2 additions & 0 deletions packages/server-utils/src/ai/openai/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export const OPENAI_METHOD_REGISTRY = {
// Conversations API - for conversation state management
// https://platform.openai.com/docs/guides/conversation-state
'conversations.create': { operation: 'chat' },
// https://developers.openai.com/api/docs/guides/decisions
'decisions.create': { operation: 'decide' },
} as const satisfies InstrumentedMethodRegistry;
export const RESPONSES_TOOL_CALL_EVENT_TYPES = [
'response.output_item.added',
Expand Down
17 changes: 13 additions & 4 deletions packages/server-utils/src/ai/openai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
import { OPENAI_METHOD_REGISTRY } from './constants';
import { instrumentStream } from './streaming';
import type { ChatCompletionChunk, OpenAiOptions, OpenAIStream, ResponseStreamingEvent } from './types';
import { getDecisionInputMessages } from './mapping';
import { addResponseAttributes, extractRequestParameters } from './utils';

/**
Expand Down Expand Up @@ -73,12 +74,15 @@ export function extractRequestAttributes(
if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) {
const params = args[0] as Record<string, unknown>;

const availableTools = recordInputs ? extractAvailableTools(params) : undefined;
if (availableTools) {
attributes[GEN_AI_TOOL_DEFINITIONS] = availableTools;
if (recordInputs) {
attributes[GEN_AI_TOOL_DEFINITIONS] = extractAvailableTools(params);
}

Object.assign(attributes, extractRequestParameters(params));

if (recordInputs && operationName === 'decide') {
attributes[GEN_AI_INPUT_MESSAGES] = getDecisionInputMessages(params);
}
} else {
attributes[GEN_AI_REQUEST_MODEL] = 'unknown';
Comment thread
nicohrubec marked this conversation as resolved.
}
Expand All @@ -88,6 +92,11 @@ export function extractRequestAttributes(

// Extract and record AI request inputs, if present. This is intentionally separate from response attributes.
export function addRequestAttributes(span: Span, params: Record<string, unknown>, operationName: string): void {
// A decision's input is recorded when its span starts, in `extractRequestAttributes`.
if (operationName === 'decide') {
return;
Comment thread
cursor[bot] marked this conversation as resolved.
}

// Store embeddings input on a separate attribute
if (operationName === 'embeddings' && 'input' in params) {
const input = params.input;
Expand Down Expand Up @@ -203,7 +212,7 @@ function instrumentMethod<T extends unknown[], R>(
}

return originalResult.then(result => {
addResponseAttributes(span, result, options.recordOutputs);
addResponseAttributes(span, result, options.recordOutputs, operationName);
return result;
});
});
Expand Down
Loading
Loading