Skip to content

feat(server-utils): Record Workers AI Jev and Clef calls as gen_ai.decide spans #25272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andreiborza merged 2 commits into from
Oct 12, 2026
Merged

feat(server-utils): Record Workers AI Jev and Clef calls as gen_ai.decide spans #25272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a028eba
feat(server-utils): Record Workers AI Jev and Clef calls as gen_ai.de…
andreiborza Oct 12, 2026
b921287
Use GEN_AI_DECIDE constant in Workers AI test assertions
andreiborza Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
import { SDK_VERSION } from '@sentry/core';
import { expect, it } from 'vitest';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
Expand Down Expand Up @@ -149,14 +150,14 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si

expect(genAiSpans[0]).toEqual(
expect.objectContaining({
name: 'evaluate typesafe/jev',
name: 'decide typesafe/jev',
status: 'ok',
is_segment: false,
attributes: {
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' },
[GEN_AI_RESPONSE_MODEL]: { value: 'jev-1.13.0', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 426, type: 'integer' },
Expand Down Expand Up @@ -195,14 +196,14 @@ it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal })

expect(genAiSpans[0]).toEqual(
expect.objectContaining({
name: 'evaluate @cf/cloudflare/clef',
name: 'decide @cf/cloudflare/clef',
status: 'ok',
is_segment: false,
attributes: {
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
[SENTRY_OP]: { value: GEN_AI_DECIDE, type: 'string' },
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
[GEN_AI_OPERATION_NAME]: { value: 'decide', type: 'string' },
[GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' },
[GEN_AI_RESPONSE_MODEL]: { value: 'clef', type: 'string' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 412, type: 'integer' },
Expand Down
16 changes: 8 additions & 8 deletions ...2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,41 +94,41 @@ test('Sends a Workers AI gen_ai span to Sentry', async () => {
});
});

test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry', async () => {
test('Sends a Workers AI gen_ai.decide span for a TypeSafe Jev call to Sentry', async () => {
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-jev`, 200));

console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);

let spanId: string | undefined;
await expect
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
.toBeDefined();

await expect
.poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
.toMatchObject({
'gen_ai.operation.name': 'evaluate',
'gen_ai.operation.name': 'decide',
'gen_ai.request.model': 'typesafe/jev',
'gen_ai.response.model': expect.stringMatching(/^jev-/),
'gen_ai.input.messages': expect.stringContaining('My payouts have been failing'),
'gen_ai.output.messages': expect.stringContaining('"is_urgent"'),
});
});

test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => {
test('Sends a Workers AI gen_ai.decide span for a Clef call to Sentry', async () => {
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200));

console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
console.log(`Polling for gen_ai.decide span: sentry trace view ${traceTarget(traceId)}`);

let spanId: string | undefined;
await expect
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.decide')?.event_id), EVENT_POLLING_OPTIONS)
.toBeDefined();

await expect
.poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
.toMatchObject({
'gen_ai.operation.name': 'evaluate',
'gen_ai.operation.name': 'decide',
'gen_ai.request.model': '@cf/cloudflare/clef',
'gen_ai.response.model': expect.stringContaining('clef'),
'gen_ai.input.messages': expect.stringContaining('Checkout has been failing'),
Expand Down
22 changes: 11 additions & 11 deletions packages/server-utils/src/ai/workers-ai/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
GEN_AI_SYSTEM_INSTRUCTIONS,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EVALUATE } from '@sentry/conventions/op';
import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_DECIDE } from '@sentry/conventions/op';
import { isObjectLike, stringify } from '@sentry/core';
import type { Span, SpanAttributeValue } from '@sentry/core';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes';
Expand All @@ -26,32 +26,32 @@ import {
getTokenUsageAttributes,
setOutputMessagesAttribute,
} from '../core/utils';
import { addResponseAttributes as addEvaluateResponseAttributes } from '../typesafe';
import { addResponseAttributes as addDecideResponseAttributes } from '../typesafe';
// Re-exported so `workers-ai/streaming.ts` keeps importing it from this module.
export { setOutputMessagesAttribute };
import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants';
import type { WorkersAiInput, WorkersAiOutput } from './types';

export type WorkersAiOperationName = 'chat' | 'embeddings' | 'evaluate';
export type WorkersAiOperationName = 'chat' | 'embeddings' | 'decide';

export const WORKERS_AI_OPERATION_SPAN_OPS: Record<WorkersAiOperationName, string> = {
chat: GEN_AI_CHAT,
embeddings: GEN_AI_EMBEDDINGS,
evaluate: GEN_AI_EVALUATE,
decide: GEN_AI_DECIDE,
};

/**
* Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `evaluate` spans.
* Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `decide` spans.
*/
const EVALUATE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef'];
const DECIDE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef'];

/**
* Determine the gen_ai operation name from the model and inputs passed to `AI.run`.
* Workers AI exposes a single `run` method, so we infer the operation from the model ID and the input shape.
*/
export function getOperationName(model: unknown, inputs: unknown): WorkersAiOperationName {
if (typeof model === 'string' && EVALUATE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
return 'evaluate';
if (typeof model === 'string' && DECIDE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
return 'decide';
}
if (inputs && typeof inputs === 'object') {
if ('messages' in inputs || 'prompt' in inputs) {
Expand Down Expand Up @@ -118,7 +118,7 @@ export function addRequestAttributes(span: Span, inputs: unknown, operationName:
}
const params = inputs as WorkersAiInput;

if (operationName === 'evaluate') {
if (operationName === 'decide') {
span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(inputs as Record<string, unknown>));
return;
}
Expand Down Expand Up @@ -159,10 +159,10 @@ export function addResponseAttributes(
recordOutputs: boolean,
operationName?: WorkersAiOperationName,
): void {
if (operationName === 'evaluate') {
if (operationName === 'decide') {
// The binding wraps the TypeSafe response as `{ state, result }`, but the docs show it unwrapped.
const body = isObjectLike(result) && isObjectLike(result.result) ? result.result : result;
addEvaluateResponseAttributes(span, body, recordOutputs);
addDecideResponseAttributes(span, body, recordOutputs);
return;
}

Expand Down
9 changes: 5 additions & 4 deletions packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
getMainCarrier,
Expand Down Expand Up @@ -164,7 +165,7 @@ describe('instrumentWorkersAiClient', () => {
});
});

it('records TypeSafe models as evaluate spans, like the TypeSafe integration', async () => {
it('records TypeSafe models as decide spans, like the TypeSafe integration', async () => {
const client = new TestClient(
getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }),
);
Expand All @@ -185,14 +186,14 @@ describe('instrumentWorkersAiClient', () => {
await instrumentWorkersAiClient(ai).run('typesafe/jev', { state: 'Help!', questions });

const span = spanToStaticSpanJSON(endedSpans[0]!);
expect(span.description).toBe('evaluate typesafe/jev');
expect(span.description).toBe('decide typesafe/jev');
expect(span.data).toEqual({
[SENTRY_ORIGIN]: 'auto.ai.cloudflare.workers_ai',
[SENTRY_OP]: 'gen_ai.evaluate',
[SENTRY_OP]: GEN_AI_DECIDE,
[SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE]: 1,
[SENTRY_SEGMENT_NAME_SOURCE]: 'custom',
[GEN_AI_PROVIDER_NAME]: 'cloudflare.workers_ai',
[GEN_AI_OPERATION_NAME]: 'evaluate',
[GEN_AI_OPERATION_NAME]: 'decide',
[GEN_AI_REQUEST_MODEL]: 'typesafe/jev',
[GEN_AI_RESPONSE_MODEL]: 'jev-1.13.0',
[GEN_AI_USAGE_INPUT_TOKENS]: 426,
Expand Down
18 changes: 9 additions & 9 deletions packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,20 +60,20 @@ describe('workers-ai utils', () => {
expect(getOperationName(MODEL, { text: 'embed me' })).toBe('embeddings');
});

it('returns "evaluate" for TypeSafe Jev models', () => {
expect(getOperationName('typesafe/jev', { state: 'Help!', questions: {} })).toBe('evaluate');
expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('evaluate');
it('returns "decide" for TypeSafe Jev models', () => {
expect(getOperationName('typesafe/jev', { state: 'Help!', questions: {} })).toBe('decide');
expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('decide');
});

it('returns "evaluate" for Cloudflare Clef models', () => {
expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('evaluate');
it('returns "decide" for Cloudflare Clef models', () => {
expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('decide');
});

it('does not return "evaluate" for other TypeSafe models', () => {
it('does not return "decide" for other TypeSafe models', () => {
expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat');
});

it('does not return "evaluate" for other models with state and questions inputs', () => {
it('does not return "decide" for other models with state and questions inputs', () => {
expect(getOperationName(MODEL, { state: 'Help!', questions: {} })).toBe('chat');
});

Expand Down Expand Up @@ -206,15 +206,15 @@ describe('workers-ai utils', () => {
it.each([
{ name: 'wrapped in { state, result }', wrap: (body: object) => ({ state: 'Completed', result: body }) },
{ name: 'unwrapped', wrap: (body: object) => body },
])('records evaluate responses $name', ({ wrap }) => {
])('records decide responses $name', ({ wrap }) => {
const { span, attributes } = createMockSpan();
const answers = { is_urgent: { type: 'noul', noul: 0.95 } };

addResponseAttributes(
span,
wrap({ model: 'jev-1.13.0', answers, usage: { input_tokens: 366, output_tokens: 50 } }),
true,
'evaluate',
'decide',
);

expect(attributes).toEqual({
Expand Down
Loading