diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/scenario-tool-runner.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/scenario-tool-runner.mjs new file mode 100644 index 000000000000..851c8cd4aee5 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/scenario-tool-runner.mjs @@ -0,0 +1,136 @@ +import Anthropic from '@anthropic-ai/sdk'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +const MODEL = 'claude-3-haiku-20240307'; + +function toolUseEvents() { + const events = [ + { + type: 'message_start', + message: { + id: 'msg_tools', + type: 'message', + role: 'assistant', + model: MODEL, + content: [], + usage: { input_tokens: 10 }, + }, + }, + ]; + [ + { id: 'toolu_weather', name: 'weather', input: { city: 'Vienna' } }, + { id: 'toolu_explode', name: 'explode', input: {} }, + ].forEach((call, index) => { + events.push( + { + type: 'content_block_start', + index, + content_block: { type: 'tool_use', id: call.id, name: call.name, input: {} }, + }, + { + type: 'content_block_delta', + index, + delta: { type: 'input_json_delta', partial_json: JSON.stringify(call.input) }, + }, + { type: 'content_block_stop', index }, + ); + }); + events.push( + { type: 'message_delta', delta: { stop_reason: 'tool_use', stop_sequence: null }, usage: { output_tokens: 20 } }, + { type: 'message_stop' }, + ); + return events; +} + +function finalEvents() { + return [ + { + type: 'message_start', + message: { + id: 'msg_final', + type: 'message', + role: 'assistant', + model: MODEL, + content: [], + usage: { input_tokens: 30 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Sunny.' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } }, + { type: 'message_stop' }, + ]; +} + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + let requests = 0; + app.post('/anthropic/v1/messages', (_req, res) => { + res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' }); + + const events = requests++ === 0 ? toolUseEvents() : finalEvents(); + events.forEach((event, index) => { + setTimeout(() => { + res.write(`event: ${event.type}\n`); + res.write(`data: ${JSON.stringify(event)}\n\n`); + if (index === events.length - 1) { + res.end(); + } + }, index * 10); + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); + + const runner = client.beta.messages.toolRunner({ + model: MODEL, + max_tokens: 100, + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + tools: [ + { + name: 'weather', + description: 'Get the weather by city', + input_schema: { type: 'object', properties: { city: { type: 'string' } } }, + run: () => 'sunny', + }, + { + name: 'explode', + description: 'Always fails', + input_schema: { type: 'object', properties: {} }, + run: () => { + throw new Error('Tool exploded'); + }, + }, + ], + stream: true, + runToolsEagerly: true, + }); + for await (const stream of runner) { + await stream.finalMessage(); + } + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 05eac304d0e1..bc4c9fef4d35 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -1,5 +1,13 @@ -import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING, SENTRY_OP } from '@sentry/conventions/attributes'; +import type { Event, SerializedStreamedSpanContainer } from '@sentry/core'; +import { + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_TOOL_NAME, + SENTRY_OP, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; // The attribute extraction is version-independent and covered by the suite pinned to 0.63. What a @@ -72,4 +80,61 @@ describe('Anthropic integration (0.129)', () => { }, }, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tool-runner.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('emits an execute_tool span for each tool the eager runner starts', async () => { + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + await createRunner() + .unordered() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; + }, + }) + .start() + .completed(); + + const main = receivedSpans.find(span => span.is_segment && span.name === 'main')!; + const chat = receivedSpans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_tools')!; + const weather = receivedSpans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'weather')!; + const explode = receivedSpans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'explode')!; + + expect(weather).toMatchObject({ + name: 'execute_tool weather', + parent_span_id: main.span_id, + status: 'ok', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: GEN_AI_EXECUTE_TOOL }, + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: { type: 'string', value: 'toolu_weather' }, + }), + }); + // The eager runner starts the first tool while the reply that called it is still streaming. + expect(weather.start_timestamp).toBeLessThan(chat.end_timestamp); + + expect(explode).toMatchObject({ name: 'execute_tool explode', parent_span_id: main.span_id, status: 'error' }); + expect(receivedEvent).toMatchObject({ + exception: { + values: [{ value: 'Tool exploded', mechanism: { type: 'auto.ai.anthropic', handled: true } }], + }, + }); + }); + }, + { + additionalDependencies: { + '@anthropic-ai/sdk': '0.129.0', + }, + }, + ); }); diff --git a/packages/server-utils/src/ai/anthropic-ai/constants.ts b/packages/server-utils/src/ai/anthropic-ai/constants.ts index 13cc9fea5d4e..4d19d399021b 100644 --- a/packages/server-utils/src/ai/anthropic-ai/constants.ts +++ b/packages/server-utils/src/ai/anthropic-ai/constants.ts @@ -9,3 +9,5 @@ export const ANTHROPIC_METHOD_REGISTRY = { 'completions.create': { operation: 'chat' }, 'beta.messages.create': { operation: 'chat' }, } as const satisfies InstrumentedMethodRegistry; + +export const ANTHROPIC_AI_ORIGIN = 'auto.ai.anthropic'; diff --git a/packages/server-utils/src/ai/anthropic-ai/index.ts b/packages/server-utils/src/ai/anthropic-ai/index.ts index 173ff3c638d4..199649884f62 100644 --- a/packages/server-utils/src/ai/anthropic-ai/index.ts +++ b/packages/server-utils/src/ai/anthropic-ai/index.ts @@ -25,8 +25,9 @@ import { resolveAIRecordingOptions, wrapPromiseWithMethods, } from '../core/utils'; -import { ANTHROPIC_METHOD_REGISTRY } from './constants'; +import { ANTHROPIC_AI_ORIGIN, ANTHROPIC_METHOD_REGISTRY } from './constants'; import { instrumentAsyncIterableStream, instrumentMessageStream } from './streaming'; +import { instrumentToolRunnerMethod } from './tool-runner'; import type { AnthropicAiOptions, AnthropicAiResponse, AnthropicAiStreamingEvent, ContentBlock } from './types'; import { handleResponseError, messagesFromParams, setMessagesAttribute } from './utils'; @@ -50,7 +51,7 @@ export function extractRequestAttributes( const attributes: Record = { [GEN_AI_PROVIDER_NAME]: 'anthropic', [GEN_AI_OPERATION_NAME]: operationName, - [SENTRY_ORIGIN]: 'auto.ai.anthropic', + [SENTRY_ORIGIN]: ANTHROPIC_AI_ORIGIN, }; if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) { @@ -350,6 +351,11 @@ function instrumentClientInPlace(client: T, options: Anthropic owner[methodName] = instrumented; } + const betaMessages = (client as { beta?: { messages?: object } }).beta?.messages; + if (betaMessages) { + instrumentToolRunnerMethod(betaMessages, options); + } + return client; } diff --git a/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts b/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts new file mode 100644 index 000000000000..795b17cffef1 --- /dev/null +++ b/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts @@ -0,0 +1,139 @@ +import type { Span, WrappedFunction } from '@sentry/core'; +import { + captureException, + fill, + getOriginalFunction, + handleCallbackErrors, + SPAN_STATUS_ERROR, + startSpan, + stringify, +} from '@sentry/core'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; +import { resolveAIRecordingOptions } from '../core/utils'; +import { ANTHROPIC_AI_ORIGIN } from './constants'; +import type { AnthropicAiOptions } from './types'; + +type AnyFunction = (...args: unknown[]) => unknown; +type RunnableTool = { name: string; description?: unknown }; +type RunContext = { toolUse?: { id?: unknown }; toolUseBlock?: { id?: unknown }; signal?: AbortSignal } | undefined; +type ToolRunner = { params?: { tools?: unknown } }; + +/** Wrap `beta.messages.toolRunner()` so the runners it returns trace their tool calls. */ +export function instrumentToolRunnerMethod(messages: object, options: AnthropicAiOptions): void { + wrapInPlace(messages, 'toolRunner', (toolRunner, thisArg, args) => { + const runner = toolRunner.apply(thisArg, args); + instrumentToolRunner(runner, options); + return runner; + }); +} + +/** Give each tool call of a tool runner a `gen_ai.execute_tool` span. */ +export function instrumentToolRunner(runner: unknown, options: AnthropicAiOptions): void { + if (!runner || typeof runner !== 'object') { + return; + } + + instrumentTools((runner as ToolRunner).params?.tools, options); + + // `addTools()` keeps its tools out of `params`. + wrapInPlace(runner, 'addTools', (addTools, thisArg, tools) => { + instrumentTools(tools, options); + return addTools.apply(thisArg, tools); + }); + wrapInPlace(runner, 'setMessagesParams', (setMessagesParams, thisArg, args) => { + const result = setMessagesParams.apply(thisArg, args); + instrumentTools((runner as ToolRunner).params?.tools, options); + return result; + }); +} + +function instrumentTools(tools: unknown, options: AnthropicAiOptions): void { + if (!Array.isArray(tools)) { + return; + } + + for (const tool of tools) { + // Toolsets are nameless entries whose `run` takes `(ctx, toolUse)`, so only named tools are wrapped. + if (tool && typeof tool === 'object' && typeof (tool as RunnableTool).name === 'string') { + wrapInPlace(tool, 'run', (run, thisArg, args) => runWithSpan(tool as RunnableTool, run, thisArg, args, options)); + } + } +} + +function wrapInPlace( + owner: object, + key: string, + call: (original: AnyFunction, thisArg: unknown, args: unknown[]) => unknown, +): void { + const current = (owner as Record)[key]; + if (typeof current === 'function' && !getOriginalFunction(current as WrappedFunction)) { + fill( + owner, + key, + (original: AnyFunction) => + function (this: unknown, ...args: unknown[]): unknown { + return call(original, this, args); + }, + ); + } +} + +function runWithSpan( + tool: RunnableTool, + run: AnyFunction, + thisArg: unknown, + args: unknown[], + options: AnthropicAiOptions, +): unknown { + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const [input, context] = args as [unknown, RunContext]; + // SDK versions before `toolUse` existed pass only `toolUseBlock`. + const callId = (context?.toolUse ?? context?.toolUseBlock)?.id; + + return startSpan( + { + name: `execute_tool ${tool.name}`, + op: GEN_AI_EXECUTE_TOOL, + attributes: { + [SENTRY_ORIGIN]: ANTHROPIC_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: tool.name, + ...(typeof tool.description === 'string' ? { [GEN_AI_TOOL_DESCRIPTION]: tool.description } : {}), + ...(typeof callId === 'string' ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: callId } : {}), + ...(recordInputs ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(input) } : {}), + }, + }, + span => + handleCallbackErrors( + () => run.apply(thisArg, args), + error => handleToolError(span, error, context), + undefined, + result => recordOutputs && span.setAttribute(GEN_AI_TOOL_CALL_RESULT, stringify(result)), + ), + ); +} + +function handleToolError(span: Span, error: unknown, context: RunContext): void { + if (context?.signal?.aborted) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + return; + } + + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + + // Only a runner passes `toolUse`/`toolUseBlock` (SDK 0.85+), and it hands the throw to the model as a result. + // A direct call rethrows to its caller, and a `ToolError` is sent on purpose, so neither is captured. + const fromRunner = !!(context?.toolUse ?? context?.toolUseBlock); + if (fromRunner && (error as Error | undefined)?.name !== 'ToolError') { + captureException(error, { mechanism: { handled: true, type: ANTHROPIC_AI_ORIGIN } }); + } +} diff --git a/packages/server-utils/src/integrations/anthropic.ts b/packages/server-utils/src/integrations/anthropic.ts index a45bbf02496f..f2905c397abf 100644 --- a/packages/server-utils/src/integrations/anthropic.ts +++ b/packages/server-utils/src/integrations/anthropic.ts @@ -12,7 +12,9 @@ import { import { getGenAiSpanOp, resolveAIRecordingOptions } from '../ai/core/utils'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { addPrivateRequestAttributes, addResponseAttributes, extractRequestAttributes } from '../ai/anthropic-ai'; +import { ANTHROPIC_AI_ORIGIN } from '../ai/anthropic-ai/constants'; import { instrumentAsyncIterableStream, instrumentMessageStream } from '../ai/anthropic-ai/streaming'; +import { instrumentToolRunner } from '../ai/anthropic-ai/tool-runner'; import type { AnthropicAiOptions, AnthropicAiResponse } from '../ai/anthropic-ai/types'; import type { OrchestrionChannelContext } from '../orchestrion/types'; import { CHANNELS } from '../orchestrion/channels'; @@ -24,8 +26,6 @@ import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation // integration is deduplicated out of the default set. const INTEGRATION_NAME = 'Anthropic_AI' as const; -const ORIGIN = 'auto.ai.anthropic'; - // `stream` determines how the span is ended const INSTRUMENTED_CHANNELS = [ { channel: CHANNELS.ANTHROPIC_CHAT, operation: 'chat', stream: 'async-iterable' }, @@ -74,6 +74,12 @@ function instrumentAnthropic(options: AnthropicAiOptions): void { }, ); } + + diagnosticsChannel + .tracingChannel(CHANNELS.ANTHROPIC_TOOL_RUNNER) + .end.subscribe(message => { + instrumentToolRunner((message as OrchestrionChannelContext).result, options); + }); } /** @@ -110,7 +116,7 @@ function createGenAiSpan( const attributes = extractRequestAttributes(args, operation, recordInputs); const model = (attributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; - attributes[SENTRY_ORIGIN] = ORIGIN; + attributes[SENTRY_ORIGIN] = ANTHROPIC_AI_ORIGIN; const client = getClient(); const span = startInactiveSpan({ diff --git a/packages/server-utils/src/orchestrion/config/anthropic-ai.ts b/packages/server-utils/src/orchestrion/config/anthropic-ai.ts index 0f530e1db778..214b73a09fc0 100644 --- a/packages/server-utils/src/orchestrion/config/anthropic-ai.ts +++ b/packages/server-utils/src/orchestrion/config/anthropic-ai.ts @@ -26,6 +26,11 @@ export const anthropicAiConfig = [ module: { name: '@anthropic-ai/sdk', versionRange: '>=0.19.2 <1', filePath }, functionQuery: { className: 'Messages', methodName: 'stream', kind: 'Sync' as const }, })), + ...['resources/beta/messages/messages.js', 'resources/beta/messages/messages.mjs'].map(filePath => ({ + channelName: 'tool-runner', + module: { name: '@anthropic-ai/sdk', versionRange: '>=0.63.0 <1', filePath }, + functionQuery: { className: 'Messages', methodName: 'toolRunner', kind: 'Sync' as const }, + })), ] satisfies InstrumentationConfig[]; export const anthropicAiModuleNames = getModuleNames(anthropicAiConfig); @@ -33,4 +38,5 @@ export const anthropicAiModuleNames = getModuleNames(anthropicAiConfig); export const anthropicAiChannels = { ANTHROPIC_CHAT: 'orchestrion:@anthropic-ai/sdk:chat', ANTHROPIC_MESSAGES_STREAM: 'orchestrion:@anthropic-ai/sdk:messages-stream', + ANTHROPIC_TOOL_RUNNER: 'orchestrion:@anthropic-ai/sdk:tool-runner', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts new file mode 100644 index 000000000000..4c99901f21a5 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts @@ -0,0 +1,266 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ErrorEvent, Span } from '@sentry/core'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON, startSpan } from '@sentry/core'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; +import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../src/ai/core/gen-ai-attributes'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +type Tool = { name: string; description?: string; input_schema: object; run: (...args: unknown[]) => unknown }; +type Params = { tools?: unknown[] }; + +function tool(name: string, run: (...args: unknown[]) => unknown): Tool { + return { name, description: `${name} tool`, input_schema: { type: 'object' }, run }; +} + +// Mirrors the SDK runner: `params` reads the live params, `addTools()` keeps its tools out of them. +class FakeRunner { + public added: unknown[] = []; + public constructor(public params: Params) {} + public addTools(...tools: unknown[]): void { + this.added.push(...tools); + } + public setMessagesParams(paramsOrMutator: Params | ((prev: Params) => Params)): void { + this.params = typeof paramsOrMutator === 'function' ? paramsOrMutator(this.params) : paramsOrMutator; + } +} + +function fakeClient() { + return { + beta: { + messages: { + create: vi.fn(), + toolRunner: vi.fn((params: Params) => { + if (!params) { + throw new TypeError('params required'); + } + return new FakeRunner(params); + }), + }, + }, + }; +} + +describe('instrumentAnthropicAiClient tool runner', () => { + let endedSpans: Span[]; + let events: ErrorEvent[]; + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + endedSpans = []; + events = []; + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + beforeSend: event => { + events.push(event); + return null; + }, + }), + ); + setCurrentClient(client); + client.init(); + client.on('spanEnd', span => endedSpans.push(span)); + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function toolSpans() { + return endedSpans.map(span => spanToStaticSpanJSON(span)).filter(span => span.op === GEN_AI_EXECUTE_TOOL); + } + + it('wraps each tool call of the runner in an execute_tool span under the active span', async () => { + const weather = tool('weather', async () => 'sunny'); + const serverTool = { type: 'web_search_20250305', name: 'web_search' }; + const serialized = JSON.stringify([weather, serverTool]); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [weather, serverTool] }); + const parent = await startSpan({ name: 'agent' }, async span => { + await weather.run({ city: 'Vienna' }, { toolUse: { id: 'toolu_1' } }); + return spanToStaticSpanJSON(span); + }); + + expect(JSON.stringify([weather, serverTool])).toBe(serialized); + expect(toolSpans()).toEqual([ + expect.objectContaining({ + description: 'execute_tool weather', + parent_span_id: parent.span_id, + origin: 'auto.ai.anthropic', + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: 'weather', + [GEN_AI_TOOL_DESCRIPTION]: 'weather tool', + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'toolu_1', + [GEN_AI_TOOL_CALL_ARGUMENTS]: '{"city":"Vienna"}', + [GEN_AI_TOOL_CALL_RESULT]: 'sunny', + }), + }), + ]); + }); + + it('reads the call id from toolUseBlock on SDKs that pass only that', () => { + const weather = tool('weather', () => 'sunny'); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [weather] }); + weather.run({}, { toolUseBlock: { id: 'toolu_old' } }); + + expect(toolSpans()[0]!.data[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toBe('toolu_old'); + }); + + it('leaves out the attributes a call does not have', () => { + const bare = { name: 'bare', input_schema: {}, run: () => undefined }; + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [bare] }); + bare.run({}); + + const { data } = toolSpans()[0]!; + for (const key of [GEN_AI_TOOL_DESCRIPTION, GEN_AI_TOOL_CALL_ID_ATTRIBUTE, GEN_AI_TOOL_CALL_RESULT]) { + expect(data).not.toHaveProperty([key]); + } + }); + + it('keeps a synchronous run synchronous', () => { + const weather = tool('weather', () => 'sunny'); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [weather] }); + + expect(weather.run({})).toBe('sunny'); + expect(toolSpans()).toHaveLength(1); + }); + + it.each([ + ['async', async () => Promise.reject(new Error('boom'))], + [ + 'sync', + () => { + throw new Error('boom'); + }, + ], + ])('marks a failed %s tool call and captures the error the runner swallows', async (_, run) => { + const explode = tool('explode', run); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [explode] }); + await expect(async () => explode.run({}, { toolUse: { id: 'toolu_1' } })).rejects.toThrow('boom'); + + expect(toolSpans()[0]!.status).toBe('internal_error'); + await vi.waitFor(() => expect(events).toHaveLength(1)); + expect(events[0]!.exception?.values?.[0]).toMatchObject({ + value: 'boom', + mechanism: { type: 'auto.ai.anthropic', handled: true }, + }); + }); + + it('marks but does not capture a throw from a direct call, which reaches its caller', async () => { + const explode = tool('explode', async () => { + throw new Error('boom'); + }); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [explode] }); + await expect(explode.run({})).rejects.toThrow('boom'); + + expect(toolSpans()[0]!.status).toBe('internal_error'); + expect(events).toHaveLength(0); + }); + + it('does not capture a ToolError or a call aborted by the runner', async () => { + const toolError = Object.assign(new Error('bad input'), { name: 'ToolError' }); + const refuses = tool('refuses', async () => { + throw toolError; + }); + const aborted = tool('aborted', async () => { + throw new Error('aborted'); + }); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [refuses, aborted] }); + await expect(refuses.run({}, { toolUse: { id: 'toolu_1' } })).rejects.toBe(toolError); + await expect(aborted.run({}, { toolUse: { id: 'toolu_2' }, signal: AbortSignal.abort() })).rejects.toThrow( + 'aborted', + ); + + expect(toolSpans().map(span => span.status)).toEqual(['internal_error', 'cancelled']); + expect(events).toHaveLength(0); + }); + + it('wraps tools added through addTools() and setMessagesParams()', () => { + const added = tool('added', () => 'a'); + const fromParams = tool('fromParams', () => 'b'); + const fromMutator = tool('fromMutator', () => 'c'); + + const runner = instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [] }); + runner.addTools(added); + runner.setMessagesParams({ tools: [fromParams] }); + runner.setMessagesParams(prev => ({ ...prev, tools: [fromMutator] })); + + added.run({}); + fromParams.run({}); + fromMutator.run({}); + + expect(toolSpans().map(span => span.description)).toEqual([ + 'execute_tool added', + 'execute_tool fromParams', + 'execute_tool fromMutator', + ]); + }); + + it('wraps a tool once when it is shared between runners or instrumented twice', () => { + const weather = tool('weather', () => 'sunny'); + + const instrumented = instrumentAnthropicAiClient(instrumentAnthropicAiClient(fakeClient())); + instrumented.beta.messages.toolRunner({ tools: [weather] }); + instrumented.beta.messages.toolRunner({ tools: [weather] }); + weather.run({}); + + expect(toolSpans()).toHaveLength(1); + }); + + it('leaves toolsets alone, since their run takes (ctx, toolUse)', async () => { + const plainRun = vi.fn(async () => { + throw new Error('aborted by user'); + }); + const plainToolset = { type: 'browser_toolset_20260801', run: plainRun }; + class ComputerToolset { + public readonly type = 'computer_toolset_20260801'; + public readonly toolsetName = 'computer'; + public async run(): Promise { + return []; + } + } + const classToolset = new ComputerToolset(); + const classRun = classToolset.run; + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [plainToolset, classToolset] }); + const toolUse = { id: 'toolu_1', name: 'navigate', toolset_name: 'browser', input: {} }; + await expect(plainToolset.run({ toolUse, signal: AbortSignal.abort() }, toolUse)).rejects.toThrow(); + await classToolset.run(); + + expect(plainToolset.run).toBe(plainRun); + expect(classToolset.run).toBe(classRun); + expect(toolSpans()).toHaveLength(0); + expect(events).toHaveLength(0); + }); + + it('runs a frozen tool without a span', () => { + const frozen = Object.freeze(tool('frozen', () => 'ok')); + + instrumentAnthropicAiClient(fakeClient()).beta.messages.toolRunner({ tools: [frozen] }); + + expect(frozen.run({})).toBe('ok'); + expect(toolSpans()).toHaveLength(0); + }); + + it('passes a toolRunner() error through', () => { + const instrumented = instrumentAnthropicAiClient(fakeClient()); + + expect(() => instrumented.beta.messages.toolRunner(undefined as unknown as Params)).toThrow('params required'); + }); +});