|
| 1 | +import { expect, test } from '@playwright/test'; |
| 2 | +import { |
| 3 | + GEN_AI_CONVERSATION_ID, |
| 4 | + GEN_AI_PROVIDER_NAME, |
| 5 | + GEN_AI_REQUEST_MODEL, |
| 6 | + GEN_AI_RESPONSE_FINISH_REASONS, |
| 7 | + GEN_AI_TOOL_CALL_ARGUMENTS, |
| 8 | + GEN_AI_TOOL_CALL_RESULT, |
| 9 | + GEN_AI_TOOL_NAME, |
| 10 | + GEN_AI_USAGE_INPUT_TOKENS, |
| 11 | + GEN_AI_USAGE_OUTPUT_TOKENS, |
| 12 | + SENTRY_ORIGIN, |
| 13 | + SERVER_ADDRESS, |
| 14 | +} from '@sentry/conventions/attributes'; |
| 15 | +import { DB_QUERY, GEN_AI_CHAT, GEN_AI_INVOKE_AGENT, HTTP_CLIENT, HTTP_SERVER } from '@sentry/conventions/op'; |
| 16 | +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; |
| 17 | +import { newAgentId, prompt, submitPrompt, waitForOperation } from './utils'; |
| 18 | + |
| 19 | +const APP = 'cloudflare-pi-durable'; |
| 20 | + |
| 21 | +test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => { |
| 22 | + // A run is its own trace, and its `invoke_agent` segment ends last. Only this test calls get_weather. |
| 23 | + const spansPromise = collectStreamedSpans( |
| 24 | + APP, |
| 25 | + spansOfTrace => |
| 26 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && |
| 27 | + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'get_weather'), |
| 28 | + ); |
| 29 | + |
| 30 | + const answer = await prompt( |
| 31 | + baseURL!, |
| 32 | + newAgentId('weather'), |
| 33 | + 'What is the weather in Vienna? Use the get_weather tool.', |
| 34 | + ); |
| 35 | + expect(answer.status, answer.reason).toBe('done'); |
| 36 | + |
| 37 | + const spans = await spansPromise; |
| 38 | + const agent = spans.find(span => span.is_segment)!; |
| 39 | + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); |
| 40 | + // The manual span of the tool is the one way to pick the call that ran, should the model call twice. |
| 41 | + const manualSpan = spans.find(span => span.name === 'resolve-weather')!; |
| 42 | + const tool = spans.find(span => span.span_id === manualSpan.parent_span_id)!; |
| 43 | + const providerCalls = spans.filter(span => getSpanOp(span) === HTTP_CLIENT); |
| 44 | + |
| 45 | + expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); |
| 46 | + // The PiHarness root session is pi conversation 1, in a Harness of its own per Durable Object. |
| 47 | + expect(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).toMatch(/^[0-9a-f]{32}:1$/); |
| 48 | + // The request that submitted the prompt is a trace of its own: the scheduler runs the work. |
| 49 | + expect(spans.some(span => getSpanOp(span) === HTTP_SERVER)).toBe(false); |
| 50 | + // pi-durable keeps its state in the `pi_` tables of `PiHarness`, whose statements start no span. |
| 51 | + expect(spans.filter(span => getSpanOp(span) === DB_QUERY && /\bpi_/.test(String(span.name)))).toEqual([]); |
| 52 | + // pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so |
| 53 | + // each request is reported once. |
| 54 | + expect(spans.filter(span => String(span.attributes[SENTRY_ORIGIN]?.value).startsWith('auto.ai.'))).toEqual( |
| 55 | + spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable'), |
| 56 | + ); |
| 57 | + |
| 58 | + // One tool-calling response, then the answer. |
| 59 | + expect(chats.length).toBeGreaterThanOrEqual(2); |
| 60 | + for (const chat of chats) { |
| 61 | + expect(chat.parent_span_id).toBe(agent.span_id); |
| 62 | + expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); |
| 63 | + expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('anthropic/claude-haiku-4.5'); |
| 64 | + expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(agent.attributes[GEN_AI_CONVERSATION_ID]?.value); |
| 65 | + // pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span |
| 66 | + // of its own to assert on. |
| 67 | + if (chat.status === 'ok') { |
| 68 | + expect(typeof chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe('number'); |
| 69 | + expect(typeof chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe('number'); |
| 70 | + expect(providerCalls.some(providerCall => providerCall.parent_span_id === chat.span_id)).toBe(true); |
| 71 | + } |
| 72 | + } |
| 73 | + |
| 74 | + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('get_weather'); |
| 75 | + expect(tool.parent_span_id).toBe(agent.span_id); |
| 76 | + expect(tool.status).toBe('ok'); |
| 77 | + expect(tool.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toContain('Vienna'); |
| 78 | + expect(tool.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('21 degrees and sunny in'); |
| 79 | + |
| 80 | + // The provider's HTTP calls nest inside the `chat` span that sent them. |
| 81 | + expect(providerCalls.length).toBeGreaterThan(0); |
| 82 | + for (const providerCall of providerCalls) { |
| 83 | + expect(chats.map(chat => chat.span_id)).toContain(providerCall.parent_span_id); |
| 84 | + expect(providerCall.attributes[SERVER_ADDRESS]?.value).toBe('openrouter.ai'); |
| 85 | + } |
| 86 | +}); |
| 87 | + |
| 88 | +test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => { |
| 89 | + const errorPromise = waitForError( |
| 90 | + APP, |
| 91 | + event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure', |
| 92 | + ); |
| 93 | + const spansPromise = collectStreamedSpans( |
| 94 | + APP, |
| 95 | + spansOfTrace => |
| 96 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && |
| 97 | + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'fail_now'), |
| 98 | + ); |
| 99 | + |
| 100 | + const answer = await prompt(baseURL!, newAgentId('fail'), 'Call the fail_now tool, then tell me what happened.'); |
| 101 | + expect(answer.status, answer.reason).toBe('done'); |
| 102 | + |
| 103 | + const [error, spans] = await Promise.all([errorPromise, spansPromise]); |
| 104 | + // The error is reported on the span of the call that threw. |
| 105 | + const tool = spans.find(span => span.span_id === error.contexts?.trace?.span_id)!; |
| 106 | + |
| 107 | + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('fail_now'); |
| 108 | + expect(tool.status).toBe('error'); |
| 109 | + expect(tool.trace_id).toBe(error.contexts?.trace?.trace_id); |
| 110 | + expect(error.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); |
| 111 | +}); |
| 112 | + |
| 113 | +test('resumes a run in a new trace after the Durable Object resets during a tool call', async ({ baseURL }) => { |
| 114 | + const agentId = newAgentId('reset'); |
| 115 | + const operationId = crypto.randomUUID(); |
| 116 | + // The tool span of the reset object never ends, so a finished `crash_once` span can only come |
| 117 | + // from the rerun after the restart. |
| 118 | + const spansPromise = collectStreamedSpans( |
| 119 | + APP, |
| 120 | + spansOfTrace => |
| 121 | + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && |
| 122 | + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once' && span.status === 'ok'), |
| 123 | + ); |
| 124 | + |
| 125 | + // The reset fails the request that submitted the prompt, but pi has stored the input already. |
| 126 | + await submitPrompt( |
| 127 | + baseURL!, |
| 128 | + agentId, |
| 129 | + 'Run the crash_once tool for job "nightly", then reply with the word DONE.', |
| 130 | + operationId, |
| 131 | + ); |
| 132 | + const answer = await waitForOperation(baseURL!, agentId, operationId); |
| 133 | + expect(answer.status, answer.reason).toBe('done'); |
| 134 | + |
| 135 | + const spans = await spansPromise; |
| 136 | + const agent = spans.find(span => span.is_segment)!; |
| 137 | + const tool = spans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once')!; |
| 138 | + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); |
| 139 | + const finalAnswer = chats.find(span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]'); |
| 140 | + |
| 141 | + expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT); |
| 142 | + expect(tool.parent_span_id).toBe(agent.span_id); |
| 143 | + expect(finalAnswer?.parent_span_id).toBe(agent.span_id); |
| 144 | + // The request that called the tool ran before the reset, so this trace starts with the rerun of the |
| 145 | + // tool. A run that never reset would start with that request. |
| 146 | + expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true); |
| 147 | +}); |
0 commit comments