|
| 1 | +import { expect, test } from '@playwright/test'; |
| 2 | +import { |
| 3 | + SEARCH_GEN_AI__INPUT__MESSAGES, |
| 4 | + SEARCH_GEN_AI__OPERATION__NAME, |
| 5 | + SEARCH_GEN_AI__OUTPUT__MESSAGES, |
| 6 | + SEARCH_GEN_AI__PROVIDER__NAME, |
| 7 | + SEARCH_GEN_AI__REQUEST__MODEL, |
| 8 | + SEARCH_GEN_AI__RESPONSE__MODEL, |
| 9 | + SEARCH_GEN_AI__USAGE__INPUT_TOKENS, |
| 10 | + SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS, |
| 11 | + SEARCH_GEN_AI__USAGE__TOTAL_TOKENS, |
| 12 | + SEARCH_ORIGIN, |
| 13 | + SEARCH_SPAN__OP, |
| 14 | + SEARCH_SPAN__STATUS, |
| 15 | +} from '@sentry/conventions/attributes/search'; |
| 16 | +import { GEN_AI_DECIDE, HTTP_SERVER } from '@sentry/conventions/op'; |
| 17 | +import { |
| 18 | + EVENT_POLLING_OPTIONS, |
| 19 | + fetchSpanAttributes, |
| 20 | + fetchTrace, |
| 21 | + findSpanInTrace, |
| 22 | + flattenTrace, |
| 23 | + traceTarget, |
| 24 | +} from '@sentry-internal/test-utils/cli'; |
| 25 | + |
| 26 | +// Mirrors src/app.ts. |
| 27 | +const MODEL = 'gpt-6-luna'; |
| 28 | + |
| 29 | +/** |
| 30 | + * What every decide span carries once Sentry has stored it, whatever the model answers. The keys are the |
| 31 | + * search names, e.g. `span.op` and `origin` rather than the `sentry.*` attributes the SDK sends. |
| 32 | + * `span.description` stays empty: ingest only derives it for ops it knows, and `gen_ai.decide` is new. |
| 33 | + */ |
| 34 | +const DECIDE_SPAN_ATTRIBUTES = { |
| 35 | + 'span.name': `decide ${MODEL}`, |
| 36 | + [SEARCH_SPAN__OP]: GEN_AI_DECIDE, |
| 37 | + [SEARCH_SPAN__STATUS]: 'ok', |
| 38 | + [SEARCH_ORIGIN]: 'auto.ai.openai', |
| 39 | + [SEARCH_GEN_AI__OPERATION__NAME]: 'decide', |
| 40 | + [SEARCH_GEN_AI__PROVIDER__NAME]: 'openai', |
| 41 | + [SEARCH_GEN_AI__REQUEST__MODEL]: MODEL, |
| 42 | + [SEARCH_GEN_AI__RESPONSE__MODEL]: MODEL, |
| 43 | +}; |
| 44 | + |
| 45 | +test('records a text decision with its state, questions and answers', async ({ baseURL }) => { |
| 46 | + const response = await fetch(`${baseURL}/decide`); |
| 47 | + const body = await response.json(); |
| 48 | + expect(response.status, JSON.stringify(body)).toBe(200); |
| 49 | + const { traceId }: { traceId: string } = body; |
| 50 | + |
| 51 | + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); |
| 52 | + |
| 53 | + let spanId: string | undefined; |
| 54 | + await expect |
| 55 | + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) |
| 56 | + .toBeDefined(); |
| 57 | + |
| 58 | + let attributes: Record<string, unknown> | undefined; |
| 59 | + await expect |
| 60 | + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) |
| 61 | + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); |
| 62 | + |
| 63 | + // Integers come back as strings. Decisions bills input tokens only, so there are no output tokens. |
| 64 | + expect(Number(attributes![SEARCH_GEN_AI__USAGE__OUTPUT_TOKENS])).toBe(0); |
| 65 | + expect(Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS])).toBeGreaterThan(0); |
| 66 | + expect(Number(attributes![SEARCH_GEN_AI__USAGE__TOTAL_TOKENS])).toBe( |
| 67 | + Number(attributes![SEARCH_GEN_AI__USAGE__INPUT_TOKENS]), |
| 68 | + ); |
| 69 | + |
| 70 | + // The request span is the segment; it ends last, so it can land after the decide span. |
| 71 | + await expect |
| 72 | + .poll(() => { |
| 73 | + const requestSpan = flattenTrace(fetchTrace(traceId)).find(item => item.op === HTTP_SERVER); |
| 74 | + return flattenTrace(requestSpan?.children ?? []).some(item => item.event_id === spanId); |
| 75 | + }, EVENT_POLLING_OPTIONS) |
| 76 | + .toBe(true); |
| 77 | + |
| 78 | + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ |
| 79 | + { |
| 80 | + type: 'evaluation', |
| 81 | + state: 'I was charged twice this month and want a refund.', |
| 82 | + questions: { |
| 83 | + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, |
| 84 | + department: { |
| 85 | + type: 'choice', |
| 86 | + instructions: 'Which team should handle this?', |
| 87 | + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, |
| 88 | + }, |
| 89 | + urgency: { |
| 90 | + type: 'score', |
| 91 | + instructions: 'How urgent is this ticket?', |
| 92 | + criteria: ['low', 'high'], |
| 93 | + }, |
| 94 | + }, |
| 95 | + }, |
| 96 | + ]); |
| 97 | + |
| 98 | + // The answers depend on the model, the shape does not. |
| 99 | + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ |
| 100 | + { |
| 101 | + type: 'evaluation', |
| 102 | + answers: { |
| 103 | + wants_refund: { type: 'predicate', probability: expect.any(Number) }, |
| 104 | + department: { |
| 105 | + type: 'choice', |
| 106 | + choice: expect.stringMatching(/^(billing|other)$/), |
| 107 | + probabilities: { billing: expect.any(Number), other: expect.any(Number) }, |
| 108 | + confidence: expect.any(Number), |
| 109 | + }, |
| 110 | + urgency: { |
| 111 | + type: 'score', |
| 112 | + score: expect.any(Number), |
| 113 | + probabilities: { 0: expect.any(Number), 1: expect.any(Number) }, |
| 114 | + legend: { 0: 'low', 1: 'high' }, |
| 115 | + confidence: expect.any(Number), |
| 116 | + }, |
| 117 | + }, |
| 118 | + }, |
| 119 | + ]); |
| 120 | +}); |
| 121 | + |
| 122 | +test('records an image decision with its image data', async ({ baseURL }) => { |
| 123 | + const response = await fetch(`${baseURL}/decide-image`); |
| 124 | + const body = await response.json(); |
| 125 | + expect(response.status, JSON.stringify(body)).toBe(200); |
| 126 | + const { traceId }: { traceId: string } = body; |
| 127 | + |
| 128 | + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); |
| 129 | + |
| 130 | + let spanId: string | undefined; |
| 131 | + await expect |
| 132 | + .poll(() => (spanId = findSpanInTrace(traceId, GEN_AI_DECIDE)?.event_id), EVENT_POLLING_OPTIONS) |
| 133 | + .toBeDefined(); |
| 134 | + |
| 135 | + let attributes: Record<string, unknown> | undefined; |
| 136 | + await expect |
| 137 | + .poll(() => (attributes = fetchSpanAttributes(traceId, spanId!)), EVENT_POLLING_OPTIONS) |
| 138 | + .toMatchObject(DECIDE_SPAN_ATTRIBUTES); |
| 139 | + |
| 140 | + expect(JSON.parse(attributes![SEARCH_GEN_AI__INPUT__MESSAGES] as string)).toEqual([ |
| 141 | + { |
| 142 | + type: 'evaluation', |
| 143 | + state: [ |
| 144 | + { |
| 145 | + role: 'user', |
| 146 | + content: [ |
| 147 | + { type: 'input_text', text: 'Is this image mostly red?' }, |
| 148 | + { type: 'input_image', image_url: expect.stringMatching(/^data:image\/png;base64,/) }, |
| 149 | + ], |
| 150 | + }, |
| 151 | + ], |
| 152 | + questions: { is_red: { type: 'predicate', instructions: 'Is the image mostly red?' } }, |
| 153 | + }, |
| 154 | + ]); |
| 155 | + |
| 156 | + expect(JSON.parse(attributes![SEARCH_GEN_AI__OUTPUT__MESSAGES] as string)).toEqual([ |
| 157 | + { type: 'evaluation', answers: { is_red: { type: 'predicate', probability: expect.any(Number) } } }, |
| 158 | + ]); |
| 159 | +}); |
0 commit comments