Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/
import { newAgentId, prompt, submitPrompt, waitForOperation } from './utils';

const APP = 'cloudflare-pi-durable';
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;

test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => {
// A run is its own trace, and its `invoke_agent` segment ends last. Only this test calls get_weather.
Expand All @@ -43,8 +44,8 @@ test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and prov
const providerCalls = spans.filter(span => getSpanOp(span) === HTTP_CLIENT);

expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable');
// The PiHarness root session is pi conversation 1, in a Harness of its own per Durable Object.
expect(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).toMatch(/^[0-9a-f]{32}:1$/);
// The provider session id pi-durable keeps with the conversation.
expect(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).toMatch(UUID);
// The request that submitted the prompt is a trace of its own: the scheduler runs the work.
expect(spans.some(span => getSpanOp(span) === HTTP_SERVER)).toBe(false);
// pi-durable keeps its state in the `pi_` tables of `PiHarness`, whose statements start no span.
Expand Down Expand Up @@ -141,6 +142,9 @@ test('resumes a run in a new trace after the Durable Object resets during a tool
expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT);
expect(tool.parent_span_id).toBe(agent.span_id);
expect(finalAnswer?.parent_span_id).toBe(agent.span_id);
// The restarted object reads the conversation id from storage, also when the run resumes with a tool call.
expect(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).toMatch(UUID);
expect(tool.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(agent.attributes[GEN_AI_CONVERSATION_ID]?.value);
// The request that called the tool ran before the reset, so this trace starts with the rerun of the
// tool. A run that never reset would start with that request.
expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
defineExtension,
defineTool,
Harness,
ProviderDoc,
section,
} from '@earendil-works/pi-durable';
import { openNodeSqliteStorage } from '@earendil-works/pi-durable/storage/sqlite/node';
Expand Down Expand Up @@ -125,7 +126,9 @@ async function handle(req, res) {
{ ownership: { kind: 'ownerless' }, agent: { model: MODEL } },
context,
);
return { id: conversation.id };
// The SDK reports the provider session id as `gen_ai.conversation.id`.
const provider = await harness.snapshot(ProviderDoc, conversation.id, context);
return { id: conversation.id, sessionId: provider?.sessionId };
}

// Returns as soon as the input is admitted. The run continues in the background, possibly in a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,13 @@ import { createConversation, sendMessage } from './utils';
const APP = 'node-pi-durable';

test('traces a run as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => {
const conversationId = await createConversation(baseURL!);
const { id: conversationId, sessionId } = await createConversation(baseURL!);
// A run is its own trace, and its `invoke_agent` segment ends last.
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(
span =>
span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`),
) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'get_weather'),
spansOfTrace.some(span => span.is_segment && span.attributes[GEN_AI_CONVERSATION_ID]?.value === sessionId) &&
spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'get_weather'),
);

const settled = await sendMessage(
Expand Down Expand Up @@ -102,18 +100,16 @@ test('traces a run as invoke_agent with chat, execute_tool and provider spans',
});

test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => {
const conversationId = await createConversation(baseURL!);
const { id: conversationId, sessionId } = await createConversation(baseURL!);
const errorPromise = waitForError(
APP,
event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure',
);
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(
span =>
span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`),
) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'fail_now'),
spansOfTrace.some(span => span.is_segment && span.attributes[GEN_AI_CONVERSATION_ID]?.value === sessionId) &&
spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'fail_now'),
);

const settled = await sendMessage(baseURL!, conversationId, 'Call the fail_now tool, then tell me what happened.');
Expand All @@ -130,14 +126,12 @@ test('reports a throwing tool as an error on its execute_tool span', async ({ ba
});

test('nests a subagent run under the tool call that delegated to it', async ({ baseURL }) => {
const conversationId = await createConversation(baseURL!);
const { id: conversationId, sessionId } = await createConversation(baseURL!);
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(
span =>
span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`),
) && spansOfTrace.some(span => !span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT),
spansOfTrace.some(span => span.is_segment && span.attributes[GEN_AI_CONVERSATION_ID]?.value === sessionId) &&
spansOfTrace.some(span => !span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT),
);

const settled = await sendMessage(
Expand All @@ -157,24 +151,20 @@ test('nests a subagent run under the tool call that delegated to it', async ({ b
expect(subagent.parent_span_id).toBe(delegate.span_id);
expect(subagentChat).toBeDefined();

// The subagent has a conversation of its own, in the same Harness.
const [agentHarness, agentConversation] = String(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).split(':');
const [subagentHarness, subagentConversation] = String(subagent.attributes[GEN_AI_CONVERSATION_ID]?.value).split(':');
expect(subagentHarness).toBe(agentHarness);
expect(subagentConversation).not.toBe(agentConversation);
// The subagent has a conversation of its own.
expect(subagent.attributes[GEN_AI_CONVERSATION_ID]?.value).toEqual(expect.any(String));
expect(subagent.attributes[GEN_AI_CONVERSATION_ID]?.value).not.toBe(sessionId);
});

test('resumes a run in a new trace after the server crashes during a tool call', async ({ baseURL }) => {
const conversationId = await createConversation(baseURL!);
const { id: conversationId, sessionId } = await createConversation(baseURL!);
// The tool span of the crashed process never ends, so a finished `crash_once` span can only come
// from the rerun in the restarted server.
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(
span =>
span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`),
) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once' && span.status === 'ok'),
spansOfTrace.some(span => span.is_segment && span.attributes[GEN_AI_CONVERSATION_ID]?.value === sessionId) &&
spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once' && span.status === 'ok'),
);

const settled = await sendMessage(
Expand All @@ -193,6 +183,8 @@ test('resumes a run in a new trace after the server crashes during a tool call',
expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT);
expect(tool.parent_span_id).toBe(agent.span_id);
expect(answer?.parent_span_id).toBe(agent.span_id);
// The restarted process reads the conversation id from storage, also when the run resumes with a tool call.
expect(tool.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(sessionId);
// The request that called the tool ran in the crashed process, so this trace starts with the rerun
// of the tool. A run that never crashed would start with that request.
expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { expect } from '@playwright/test';

export async function createConversation(baseURL: string): Promise<number> {
/** Create a conversation. `sessionId` is the id the SDK reports as `gen_ai.conversation.id`. */
export async function createConversation(baseURL: string): Promise<{ id: number; sessionId: string }> {
const res = await fetch(`${baseURL}/conversations`, { method: 'POST' });
expect(res.status).toBe(200);
return ((await res.json()) as { id: number }).id;
return (await res.json()) as { id: number; sessionId: string };
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const PI_DURABLE_DEPENDENCIES = {
},
};

// The provider session id that pi-durable stores with each conversation, a UUIDv7.
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;

const CONTENT_ATTRIBUTES = [
GEN_AI_INPUT_MESSAGES,
GEN_AI_OUTPUT_MESSAGES,
Expand Down Expand Up @@ -84,9 +87,7 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable');
expect(agent.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('invoke_agent');
const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value;
// The root conversation's id is `1` in every pi-durable storage, so it is prefixed
// with a per-Harness id.
expect(conversationId).toMatch(/^[0-9a-f]{32}:1$/);
expect(conversationId).toMatch(UUID);

const chats = spans
.filter(span => span.name === 'chat faux-model')
Expand Down Expand Up @@ -399,7 +400,7 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
.start()
.completed();

expect(runConversationId).toMatch(/^[0-9a-f]{32}:1$/);
expect(runConversationId).toMatch(UUID);
expect(compactionConversationId).toBe(runConversationId);
});
},
Expand Down Expand Up @@ -427,13 +428,14 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
expect(delegate.parent_span_id).toBe(parent.span_id);

// The subagent's run is a child of the delegating call, in the same trace, on its own
// conversation of the same Harness.
// conversation.
const child = spans.find(span => span.name === 'invoke_agent' && !span.is_segment)!;
expect(child.parent_span_id).toBe(delegate.span_id);
const parentConversation = String(parent.attributes[GEN_AI_CONVERSATION_ID]?.value);
const childConversation = String(child.attributes[GEN_AI_CONVERSATION_ID]?.value);
expect(parentConversation).toMatch(UUID);
expect(childConversation).toMatch(UUID);
expect(childConversation).not.toBe(parentConversation);
expect(childConversation.split(':')[0]).toBe(parentConversation.split(':')[0]);

const childChat = spans.find(span => span.parent_span_id === child.span_id)!;
expect(childChat.name).toBe('chat faux-model');
Expand Down Expand Up @@ -646,7 +648,7 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
);
// The run's own conversation id, not the one the request set on its scope.
const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value;
expect(conversationId).toMatch(/^[0-9a-f]{32}:\d+$/);
expect(conversationId).toMatch(UUID);
for (const span of spans) {
expect(span.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId);
expect(span.attributes['request']).toBeUndefined();
Expand All @@ -667,7 +669,7 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
'request B',
);
const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value;
expect(conversationId).toMatch(/^[0-9a-f]{32}:\d+$/);
expect(conversationId).toMatch(UUID);
for (const span of spans) {
expect(span.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId);
expect(span.attributes['request']).toBeUndefined();
Expand Down
3 changes: 3 additions & 0 deletions packages/server-utils/src/ai/pi-durable/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ export const PI_TASK = {
/** Kind of the built-in document that holds a conversation's run control. */
export const PI_LIVE_DOC_KIND = 'pi.live';

/** Kind of the built-in document that holds a conversation's provider session id. */
export const PI_PROVIDER_DOC_KIND = 'pi.provider';

/** Kind of the transcript entry that holds the tool result the model receives. */
export const PI_TOOL_RESULT_ENTRY_KIND = 'pi.tool-result';

Expand Down
17 changes: 15 additions & 2 deletions packages/server-utils/src/ai/pi-durable/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ import {
startNewTrace,
withActiveSpan,
} from '@sentry/core';
import { GEN_AI_CONVERSATION_ID } from '@sentry/conventions/attributes';
import type { GenAiOptions } from '../core/utils';
import { skipPiAiProviderIntegrations } from '../pi-ai/providers';
import { PI_DURABLE_ORIGIN, PI_TASK } from './constants';
import { instrumentPiModels } from './models';
import type { PiRuns } from './runs';
import { createRuns, endRun, observeRunControl, startRun, toSentryConversationId } from './runs';
import { createRuns, endRun, observeRunControl, readProviderSessionId, startRun, toSentryConversationId } from './runs';
import { instrumentTool, instrumentToolPhase } from './tools';
import type {
PiHarness,
Expand Down Expand Up @@ -137,6 +138,7 @@ function instrumentRegistry(registry: PiRegistryReader, options: PiDurableOption
definition.name,
observed => phase.call(receiver, taskRecord, observed, context),
runtime,
context,
runs,
wrapTool,
isAbort,
Expand Down Expand Up @@ -191,6 +193,7 @@ function runPhase(
kind: string,
phase: (runtime: PiTaskRuntime) => Promise<unknown>,
runtime: PiTaskRuntime,
context: unknown,
runs: PiRuns,
wrapTool: (tool: PiTool) => PiTool,
isAbort: boolean,
Expand All @@ -216,7 +219,17 @@ function runPhase(
}

const runInScope = async (scope: Scope): Promise<unknown> => {
scope.setConversationId(toSentryConversationId(runs, conversationId));
// Read at the first phase of a run, which is a `pi.tool` phase when a restarted Harness reruns a
// tool call.
let sessionId = run?.sessionId;
if (!sessionId && (run || kind === PI_TASK.COMPACTION)) {
sessionId = await readProviderSessionId(runtime, context);
if (run && sessionId) {
run.sessionId = sessionId;
run.span.setAttribute(GEN_AI_CONVERSATION_ID, sessionId);
}
}
scope.setConversationId(sessionId ?? toSentryConversationId(runs, conversationId));
try {
return await phase(observed);
} catch (error) {
Expand Down
40 changes: 34 additions & 6 deletions packages/server-utils/src/ai/pi-durable/runs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,17 @@ import {
import { GEN_AI_CONVERSATION_ID, GEN_AI_OPERATION_NAME, SENTRY_ORIGIN } from '@sentry/conventions/attributes';
import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
import { DEBUG_BUILD } from '../../debug-build';
import { MAX_TRACKED_PI_RUNS, PI_DURABLE_ORIGIN, PI_LIVE_DOC_KIND } from './constants';
import type { PiCommitChange, PiDocToken, PiLiveState, PiSettlement, PiTaskRuntime } from './types';
import { MAX_TRACKED_PI_RUNS, PI_DURABLE_ORIGIN, PI_LIVE_DOC_KIND, PI_PROVIDER_DOC_KIND } from './constants';
import type { PiCommitChange, PiDocToken, PiLiveState, PiProviderState, PiSettlement, PiTaskRuntime } from './types';
import { withCleanScopes } from '../../utils/withCleanScopes';
import { bound } from './utils';

// The definition of `pi.provider` as pi-durable checks it on a read: the address, the version and the
// persisted semantics of the document.
const PROVIDER_DOC = {
definition: { kind: PI_PROVIDER_DOC_KIND, version: 1, scope: 'conversation', history: 'latest', fork: 'initial' },
};

// Unanswered reasons that mean the run was stopped, not that it failed.
const CANCELLED_REASONS = new Set(['aborted', 'reset']);

Expand All @@ -32,6 +38,8 @@ export interface PiRun {
isolationScope: Scope;
/** The run's first input submission, which identifies the run in `pi.live`. */
firstInput?: unknown;
/** The conversation's provider session id, read by the first phase of the run. */
sessionId?: string;
/**
* The status the run ends with, set while the commit that ends it is in flight. `harness.close()`
* can begin in that gap, because `wait()` resolves as soon as the commit settles the inputs.
Expand All @@ -42,9 +50,9 @@ export interface PiRun {
/** Open runs of one Harness, keyed by the pi-durable conversation id. */
export interface PiRuns {
/**
* Random id of this Harness, prefixed to `gen_ai.conversation.id`. pi-durable numbers
* conversations per storage from 1, so the bare id would merge the root conversations of every
* Harness into one Sentry conversation.
* Random id of this Harness, prefixed to `gen_ai.conversation.id` until the provider session id of
* the conversation is known. pi-durable numbers conversations per storage from 1, so the bare id
* would merge the root conversations of every Harness into one Sentry conversation.
*/
harnessId: string;
active: Map<unknown, PiRun>;
Expand Down Expand Up @@ -96,7 +104,27 @@ export function createRuns(): PiRuns {
}

export function toSentryConversationId(runs: PiRuns, conversationId: unknown): string {
return `${runs.harnessId}:${String(conversationId)}`;
return runs.active.get(conversationId)?.sessionId ?? `${runs.harnessId}:${String(conversationId)}`;
}

/**
* Read the provider session id of the conversation, or `undefined` when pi-durable keeps none.
*
* pi-durable 1.0.2 and later keep a UUIDv7 per conversation in the `pi.provider` document, stored with
* the conversation. Unlike the harness prefix, it stays the same when a Harness opens the storage
* again, for example after a Durable Object restarts, and it is unique across storages.
*/
export async function readProviderSessionId(runtime: PiTaskRuntime, context: unknown): Promise<string | undefined> {
try {
const provider = (await runtime.snapshot?.(PROVIDER_DOC, runtime.conversationId, context)) as
| PiProviderState
| undefined;
return typeof provider?.sessionId === 'string' && provider.sessionId ? provider.sessionId : undefined;
} catch (error) {
// A later version of the document fails the read. The harness prefix is used then.
DEBUG_BUILD && debug.warn('[pi-durable] failed to read the provider session id', error);
return undefined;
}
}

/** Start a run of `conversationId`: a new trace, or a child of the tool call that owns the conversation. */
Expand Down
6 changes: 6 additions & 0 deletions packages/server-utils/src/ai/pi-durable/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ export interface PiLiveState {
run?: { inputs?: unknown[] };
}

/** The `pi.provider` document, in pi-durable 1.0.2 and later. */
export interface PiProviderState {
sessionId?: string;
}

export type PiCommitChange = (tx: object, current: unknown) => unknown;

/** pi-durable ids (conversations, tasks, submissions) are numbers, typed as `unknown` here. */
Expand All @@ -108,6 +113,7 @@ export interface PiTaskRuntime {
readonly signal?: AbortSignal;
agent(context: unknown): Promise<PiAgent>;
commit(change: PiCommitChange, context: unknown): Promise<void>;
snapshot?(token: PiDocToken, ownerId: unknown, ...rest: unknown[]): Promise<unknown>;
}

export type PiPhaseHandler = (task: unknown, runtime: PiTaskRuntime, context: unknown) => Promise<unknown>;
Expand Down
Loading
Loading