From a028ebaf48bfca464f80b975708235e70362eca6 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 13:40:25 +0200 Subject: [PATCH 1/2] feat(server-utils): Record Workers AI Jev and Clef calls as gen_ai.decide spans Co-Authored-By: Claude Opus 5.5 --- .../suites/tracing/workers-ai/test.ts | 12 +++++----- .../tests/send-to-sentry.test.ts | 16 +++++++------- .../server-utils/src/ai/workers-ai/utils.ts | 22 +++++++++---------- .../test/ai/lib/tracing/workers-ai.test.ts | 8 +++---- .../ai/lib/utils/workers-ai-utils.test.ts | 18 +++++++-------- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index 600062d6eb0a..767698b4a6e7 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -149,14 +149,14 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si expect(genAiSpans[0]).toEqual( expect.objectContaining({ - name: 'evaluate typesafe/jev', + name: 'decide typesafe/jev', status: 'ok', is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' }, + [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, - [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, + [GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' }, [GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13.0', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 426, type: 'integer' }, @@ -195,14 +195,14 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) expect(genAiSpans[0]).toEqual( expect.objectContaining({ - name: 'evaluate @cf/cloudflare/clef', + name: 'decide @cf/cloudflare/clef', status: 'ok', is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' }, + [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, - [GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' }, + [GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' }, [GEN_AI_RESPONSE_MODEL]: { value: 'clef', type: 'string' }, [GEN_AI_USAGE_INPUT_TOKENS]: { value: 412, type: 'integer' }, diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts index f5ce99ab2a0f..fd79fa5b7743 100644 --- a/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts +++ b/dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts @@ -94,20 +94,20 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => { }); }); -test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => { +test('Sends a Workers AI gen_ai.decide span for a TypeSafe Jev call to Sentry', async () => { const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200)); - console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`); + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); let spanId: string | undefined; await expect - .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS) + .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS) .toBeDefined(); await expect .poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS) .toMatchObject({ - 'gen_ai.operation.name': 'evaluate', + 'gen_ai.operation.name': 'decide', 'gen_ai.request.model': 'typesafe/jev', 'gen_ai.response.model': expect.stringMatching(/^jev-/), 'gen_ai.input.messages': expect.stringContaining('My payouts have been failing'), @@ -115,20 +115,20 @@ test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry' }); }); -test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => { +test('Sends a Workers AI gen_ai.decide span for a Clef call to Sentry', async () => { const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200)); - console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`); + console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`); let spanId: string | undefined; await expect - .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS) + .poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS) .toBeDefined(); await expect .poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS) .toMatchObject({ - 'gen_ai.operation.name': 'evaluate', + 'gen_ai.operation.name': 'decide', 'gen_ai.request.model': '@cf/cloudflare/clef', 'gen_ai.response.model': expect.stringContaining('clef'), 'gen_ai.input.messages': expect.stringContaining('Checkout has been failing'), diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 73da7883b12e..29db4d2cd136 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -16,7 +16,7 @@ import { GEN_AI_SYSTEM_INSTRUCTIONS, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_DECIDE } from '@sentry/conventions/op'; import { isObjectLike, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; @@ -26,32 +26,32 @@ import { getTokenUsageAttributes, setOutputMessagesAttribute, } from '../core/utils'; -import { addResponseAttributes as addEvaluateResponseAttributes } from '../typesafe'; +import { addResponseAttributes as addDecideResponseAttributes } from '../typesafe'; // Re-exported so `workers-ai/streaming.ts` keeps importing it from this module. export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; import type { WorkersAiInput, WorkersAiOutput } from './types'; -export type WorkersAiOperationName = 'chat' | 'embeddings' | 'evaluate'; +export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide'; export const WORKERS_AI_OPERATION_SPAN_OPS: Record = { chat: GEN_AI_CHAT, embeddings: GEN_AI_EMBEDDINGS, - evaluate: GEN_AI_EVALUATE, + decide: GEN_AI_DECIDE, }; /** - * Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `evaluate` spans. + * Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `decide` spans. */ -const EVALUATE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef']; +const DECIDE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef']; /** * Determine the gen_ai operation name from the model and inputs passed to `AI.run`. * Workers AI exposes a single `run` method, so we infer the operation from the model ID and the input shape. */ export function getOperationName(model: unknown, inputs: unknown): WorkersAiOperationName { - if (typeof model === 'string' && EVALUATE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) { - return 'evaluate'; + if (typeof model === 'string' && DECIDE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) { + return 'decide'; } if (inputs && typeof inputs === 'object') { if ('messages' in inputs || 'prompt' in inputs) { @@ -118,7 +118,7 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName: } const params = inputs as WorkersAiInput; - if (operationName === 'evaluate') { + if (operationName === 'decide') { span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(inputs as Record)); return; } @@ -159,10 +159,10 @@ export function addResponseAttributes( recordOutputs: boolean, operationName?: WorkersAiOperationName, ): void { - if (operationName === 'evaluate') { + if (operationName === 'decide') { // The binding wraps the TypeSafe response as `{ state, result }`, but the docs show it unwrapped. const body = isObjectLike(result) && isObjectLike(result.result) ? result.result : result; - addEvaluateResponseAttributes(span, body, recordOutputs); + addDecideResponseAttributes(span, body, recordOutputs); return; } diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index f2784635c5f2..e30bcf47e1fe 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -164,7 +164,7 @@ describe('instrumentWorkersAiClient', () => { }); }); - it('records TypeSafe models as evaluate spans, like the TypeSafe integration', async () => { + it('records TypeSafe models as decide spans, like the TypeSafe integration', async () => { const client = new TestClient( getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), ); @@ -185,14 +185,14 @@ describe('instrumentWorkersAiClient', () => { await instrumentWorkersAiClient(ai).run('typesafe/jev', { state: 'Help!', questions }); const span = spanToStaticSpanJSON(endedSpans[0]!); - expect(span.description).toBe('evaluate typesafe/jev'); + expect(span.description).toBe('decide typesafe/jev'); expect(span.data).toEqual({ [SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai', - [SENTRY_OP]: 'gen_ai.evaluate', + [SENTRY_OP]: 'gen_ai.decide', [SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1, [SENTRY_SEGMENT_NAME_SOURCE]: 'custom', [GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai', - [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_OPERATION_NAME]: 'decide', [GEN_AI_REQUEST_MODEL]: 'typesafe/jev', [GEN_AI_RESPONSE_MODEL]: 'jev-1.13.0', [GEN_AI_USAGE_INPUT_TOKENS]: 426, diff --git a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts index 40abfadf8f2f..85d690e01421 100644 --- a/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts +++ b/packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts @@ -60,20 +60,20 @@ describe('workers-ai utils', () => { expect(getOperationName(MODEL, { text: 'embed me' })).toBe('embeddings'); }); - it('returns "evaluate" for TypeSafe Jev models', () => { - expect(getOperationName('typesafe/jev', { state: 'Help!', questions: {} })).toBe('evaluate'); - expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('evaluate'); + it('returns "decide" for TypeSafe Jev models', () => { + expect(getOperationName('typesafe/jev', { state: 'Help!', questions: {} })).toBe('decide'); + expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('decide'); }); - it('returns "evaluate" for Cloudflare Clef models', () => { - expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('evaluate'); + it('returns "decide" for Cloudflare Clef models', () => { + expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('decide'); }); - it('does not return "evaluate" for other TypeSafe models', () => { + it('does not return "decide" for other TypeSafe models', () => { expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat'); }); - it('does not return "evaluate" for other models with state and questions inputs', () => { + it('does not return "decide" for other models with state and questions inputs', () => { expect(getOperationName(MODEL, { state: 'Help!', questions: {} })).toBe('chat'); }); @@ -206,7 +206,7 @@ describe('workers-ai utils', () => { it.each([ { name: 'wrapped in { state, result }', wrap: (body: object) => ({ state: 'Completed', result: body }) }, { name: 'unwrapped', wrap: (body: object) => body }, - ])('records evaluate responses $name', ({ wrap }) => { + ])('records decide responses $name', ({ wrap }) => { const { span, attributes } = createMockSpan(); const answers = { is_urgent: { type: 'noul', noul: 0.95 } }; @@ -214,7 +214,7 @@ describe('workers-ai utils', () => { span, wrap({ model: 'jev-1.13.0', answers, usage: { input_tokens: 366, output_tokens: 50 } }), true, - 'evaluate', + 'decide', ); expect(attributes).toEqual({ From b9212873722c3ffd3f1ed09c3827eb53821f3a0a Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Mon, 12 Oct 2026 13:42:12 +0200 Subject: [PATCH 2/2] Use GEN_AI_DECIDE constant in Workers AI test assertions --- .../suites/tracing/workers-ai/test.ts | 5 +++-- packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index 767698b4a6e7..aca3281b19c2 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -21,6 +21,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { SDK_VERSION } from '@sentry/core'; import { expect, it } from 'vitest'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; @@ -154,7 +155,7 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, + [SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' }, @@ -200,7 +201,7 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) is_segment: false, attributes: { [SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' }, - [SENTRY_OP]: { value: 'gen_ai.decide', type: 'string' }, + [SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' }, [GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' }, [GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' }, [GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' }, diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index e30bcf47e1fe..5b1f8738b7a3 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -14,6 +14,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getMainCarrier, @@ -188,7 +189,7 @@ describe('instrumentWorkersAiClient', () => { expect(span.description).toBe('decide typesafe/jev'); expect(span.data).toEqual({ [SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai', - [SENTRY_OP]: 'gen_ai.decide', + [SENTRY_OP]: GEN_AI_DECIDE, [SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1, [SENTRY_SEGMENT_NAME_SOURCE]: 'custom', [GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai',