test(e2e): Add node-openai-decisions-send-to-sentry test app

.github/workflows/build.yml

@@ -1197,6 +1197,10 @@ jobs:
       E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
       E2E_OPENROUTER_API_KEY: ${{ secrets.E2E_OPENROUTER_API_KEY }}
+      # Only this app talks to OpenAI directly (OpenRouter has no Decisions route), so only it gets the key.
+      E2E_OPENAI_DECISIONS_API_KEY:
+        ${{ matrix.test-application == 'node-openai-decisions-send-to-sentry' && secrets.E2E_OPENAI_DECISIONS_API_KEY ||
+        '' }}
       # Used by test apps that deploy a real Cloudflare Worker, e.g. cloudflare-workers-send-to-sentry
       CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
       CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}

dev-packages/e2e-tests/.env.example

@@ -16,6 +16,10 @@ E2E_TEST_SENTRY_PROJECT=
 # Only needed to run that test app locally.
 E2E_OPENROUTER_API_KEY=
 
+# An OpenAI API key for the Decisions API (OpenRouter has no Decisions route).
+# Only needed to run node-openai-decisions-send-to-sentry locally.
+E2E_OPENAI_DECISIONS_API_KEY=
+
 # Cloudflare credentials for E2E tests that deploy a real Worker (e.g. cloudflare-workers-send-to-sentry).
 # The API token needs "Workers Scripts: Edit" on the account; "Workers KV Storage: Read" additionally silences a
 # warning when a worker is deleted. Leave it empty to use a `wrangler login` session instead.

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/.gitignore

@@ -0,0 +1,5 @@
+node_modules
+pnpm-lock.yaml
+dist
+test-results
+playwright-report

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "node-openai-decisions-send-to-sentry",
+  "description": "The OpenAI integration for the Decisions API as a user runs it: a plain Sentry.init on an express app, real decisions.create requests against OpenAI, and the gen_ai.decide spans read back from a real Sentry project",
+  "version": "1.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "start": "node --import ./dist/instrument.js dist/app.js",
+    "test": "playwright test",
+    "clean": "npx rimraf node_modules dist pnpm-lock.yaml",
+    "test:build": "pnpm install && pnpm build",
+    "test:build-latest": "pnpm install && pnpm add openai@latest && pnpm build",
+    "test:assert": "pnpm test"
+  },
+  "dependencies": {
+    "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
+    "express": "^4.21.2",
+    "openai": "7.32.0"
+  },
+  "devDependencies": {
+    "@playwright/test": "~1.63.0",
+    "@sentry-internal/test-utils": "link:../../../test-utils",
+    "@sentry/conventions": "0.27.0",
+    "@types/express": "^4.17.21",
+    "@types/node": "^24.0.0",
+    "sentry": "~0.44.1",
+    "typescript": "~5.9.0"
+  },
+  "sentryTest": {
+    "optional": true,
+    "optionalVariants": [
+      {
+        "build-command": "pnpm test:build-latest",
+        "label": "node-openai-decisions-send-to-sentry (latest)"
+      }
+    ]
+  },
+  "volta": {
+    "node": "24.15.0",
+    "extends": "../../package.json"
+  }
+}

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/playwright.config.ts

@@ -0,0 +1,27 @@
+import { defineConfig } from '@playwright/test';
+
+const port = 3030;
+
+export default defineConfig({
+  testDir: './tests',
+  /*
+   * Each Sentry poll can take up to 180s. The first poll also waits the ~2min until the span can be
+   * queried, so the later polls usually pass at once. This leaves room for two full polls.
+   */
+  timeout: 400_000,
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  // Every test spends most of its time polling Sentry, so run them all at once.
+  workers: '100%',
+  reporter: process.env.CI ? [['list'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+  use: {
+    baseURL: `http://localhost:${port}`,
+  },
+  webServer: {
+    command: 'pnpm start',
+    port,
+    stdout: 'pipe',
+    stderr: 'pipe',
+  },
+});

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/app.ts

@@ -0,0 +1,85 @@
+// `instrument.ts` is preloaded via `node --import`, so Sentry is already initialised here.
+import * as Sentry from '@sentry/node';
+import express from 'express';
+import OpenAI from 'openai';
+
+// OpenRouter has no `/decisions` route, so this app calls OpenAI directly. CI only passes this key to
+// this app.
+const apiKey = process.env.E2E_OPENAI_DECISIONS_API_KEY;
+if (!apiKey) {
+  throw new Error('E2E_OPENAI_DECISIONS_API_KEY is not set');
+}
+
+const client = new OpenAI({ apiKey });
+
+const MODEL = 'gpt-6-luna';
+
+// A 1x1 red PNG, so the image request is cheap.
+const RED_PIXEL =
+  'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==';
+
+const app = express();
+
+// Each route makes one `decisions.create` call and answers with the trace id, so the test can find
+// the request's spans in Sentry.
+
+app.get('/decide', async (_req, res, next) => {
+  try {
+    const decision = await client.decisions.create({
+      model: MODEL,
+      input: 'I was charged twice this month and want a refund.',
+      questions: [
+        { type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' },
+        {
+          type: 'choice',
+          name: 'department',
+          instructions: 'Which team should handle this?',
+          choices: [
+            { value: 'billing', description: 'Charges and refunds' },
+            { value: 'other', description: 'Anything else' },
+          ],
+        },
+        {
+          type: 'score',
+          name: 'urgency',
+          instructions: 'How urgent is this ticket?',
+          levels: [{ label: 'low' }, { label: 'high' }],
+        },
+      ],
+    });
+    res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers });
+  } catch (error) {
+    next(error);
+  }
+});
+
+app.get('/decide-image', async (_req, res, next) => {
+  try {
+    const decision = await client.decisions.create({
+      model: MODEL,
+      input: [
+        {
+          role: 'user',
+          content: [
+            { type: 'input_text', text: 'Is this image mostly red?' },
+            { type: 'input_image', image_url: RED_PIXEL },
+          ],
+        },
+      ],
+      questions: [{ type: 'predicate', name: 'is_red', instructions: 'Is the image mostly red?' }],
+    });
+    res.send({ traceId: Sentry.getActiveSpan()?.spanContext().traceId, answers: decision.answers });
+  } catch (error) {
+    next(error);
+  }
+});
+
+app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
+  res.status(500).send({ message: error.message });
+});
+
+const port = Number(process.env.PORT ?? 3030);
+app.listen(port, () => {
+  // eslint-disable-next-line no-console
+  console.log(`node-openai-decisions-send-to-sentry listening on port ${port}`);
+});

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/src/instrument.ts

@@ -0,0 +1,7 @@
+import * as Sentry from '@sentry/node';
+
+Sentry.init({
+  dsn: process.env.E2E_TEST_DSN,
+  environment: 'qa', // dynamic sampling bias to keep transactions
+  tracesSampleRate: 1,
+});

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tests/send-to-sentry.test.ts

@@ -0,0 +1,159 @@
+import { expect, test } from '@playwright/test';
+import {
+  SEARCH_GEN_AI__INPUT__MESSAGES,
+  SEARCH_GEN_AI__OPERATION__NAME,
+  SEARCH_GEN_AI__OUTPUT__MESSAGES,
+  SEARCH_GEN_AI__PROVIDER__NAME,
+  SEARCH_GEN_AI__REQUEST__MODEL,
+  SEARCH_GEN_AI__RESPONSE__MODEL,
+  SEARCH_GEN_AI__USAGE__INPUT_TOKENS,
+  SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS,
+  SEARCH_GEN_AI__USAGE__TOTAL_TOKENS,
+  SEARCH_ORIGIN,
+  SEARCH_SPAN__OP,
+  SEARCH_SPAN__STATUS,
+} from '@sentry/conventions/attributes/search';
+import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op';
+import {
+  EVENT_POLLING_OPTIONS,
+  fetchSpanAttributes,
+  fetchTrace,
+  findSpanInTrace,
+  flattenTrace,
+  traceTarget,
+} from '@sentry-internal/test-utils/cli';
+
+// Mirrors src/app.ts.
+const MODEL = 'gpt-6-luna';
+
+/**
+ * What every decide span carries once Sentry has stored it, whatever the model answers. The keys are the
+ * search names, e.g. `span.op` and `origin` rather than the `sentry.*` attributes the SDK sends.
+ * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new.
+ */
+const DECIDE_SPAN_ATTRIBUTES = {
+  'span.name': `decide ${MODEL}`,
+  [SEARCH_SPAN__OP]: GEN_AI_DECIDE,
+  [SEARCH_SPAN__STATUS]: 'ok',
+  [SEARCH_ORIGIN]: 'auto.ai.openai',
+  [SEARCH_GEN_AI__OPERATION__NAME]: 'decide',
+  [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai',
+  [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL,
+  [SEARCH_GEN_AI__RESPONSE__MODEL]: MODEL,
+};
+
+test('records a text decision with its state, questions and answers', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/decide`);
+  const body = await response.json();
+  expect(response.status, JSON.stringify(body)).toBe(200);
+  const { traceId }: { traceId: string } = body;
+
+  console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
+
+  let spanId: string | undefined;
+  await expect
+    .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS)
+    .toBeDefined();
+
+  let attributes: Record<string, unknown> | undefined;
+  await expect
+    .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS)
+    .toMatchObject(DECIDE_SPAN_ATTRIBUTES);
+
+  // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens.
+  expect(Number(attributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBe(0);
+  expect(Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0);
+  expect(Number(attributes![SEARCH_GEN_AI__USAGE__TOTAL_TOKENS])).toBe(
+    Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS]),
+  );
+
+  // The request span is the segment; it ends last, so it can land after the decide span.
+  await expect
+    .poll(() => {
+      const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER);
+      return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId);
+    }, EVENT_POLLING_OPTIONS)
+    .toBe(true);
+
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([
+    {
+      type: 'evaluation',
+      state: 'I was charged twice this month and want a refund.',
+      questions: {
+        wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' },
+        department: {
+          type: 'choice',
+          instructions: 'Which team should handle this?',
+          criteria: { billing: 'Charges and refunds', other: 'Anything else' },
+        },
+        urgency: {
+          type: 'score',
+          instructions: 'How urgent is this ticket?',
+          criteria: ['low', 'high'],
+        },
+      },
+    },
+  ]);
+
+  // The answers depend on the model, the shape does not.
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([
+    {
+      type: 'evaluation',
+      answers: {
+        wants_refund: { type: 'predicate', probability: expect.any(Number) },
+        department: {
+          type: 'choice',
+          choice: expect.stringMatching(/^(billing|other)$/),
+          probabilities: { billing: expect.any(Number), other: expect.any(Number) },
+          confidence: expect.any(Number),
+        },
+        urgency: {
+          type: 'score',
+          score: expect.any(Number),
+          probabilities: { 0: expect.any(Number), 1: expect.any(Number) },
+          legend: { 0: 'low', 1: 'high' },
+          confidence: expect.any(Number),
+        },
+      },
+    },
+  ]);
+});
+
+test('records an image decision with its image data', async ({ baseURL }) => {
+  const response = await fetch(`${baseURL}/decide-image`);
+  const body = await response.json();
+  expect(response.status, JSON.stringify(body)).toBe(200);
+  const { traceId }: { traceId: string } = body;
+
+  console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);
+
+  let spanId: string | undefined;
+  await expect
+    .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS)
+    .toBeDefined();
+
+  let attributes: Record<string, unknown> | undefined;
+  await expect
+    .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS)
+    .toMatchObject(DECIDE_SPAN_ATTRIBUTES);
+
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([
+    {
+      type: 'evaluation',
+      state: [
+        {
+          role: 'user',
+          content: [
+            { type: 'input_text', text: 'Is this image mostly red?' },
+            { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) },
+          ],
+        },
+      ],
+      questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } },
+    },
+  ]);
+
+  expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([
+    { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } },
+  ]);
+});

dev-packages/e2e-tests/test-applications/node-openai-decisions-send-to-sentry/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
+    "outDir": "dist",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "types": ["node"]
+  },
+  "include": ["src/**/*.ts"]
+}